feat(prometheus): the Prometheus dashboard widget, target + template (#5691)

#5691 "Generic Prometheus dashboard widget: pick a target and a template",
step 3 of milestone 508 "Prometheus widget templates".

- /plugins/prometheus/widget draws one target with one template in widget
  shape: the verdict (led by "isn't answering" when the target is down),
  then each block with problems first, figures beside a share bar or the
  last 24h of bars, and the healthy rest folding to "+N more healthy" past
  six. A missing target or template, or one that no longer checks, says so.
- library.py: bundled templates from plugins/prometheus/widgets/*.yaml,
  the picker's target and template lists, and the default title.
- widgets/fabled_archives.yaml: the first bundled template, against the
  series Archives reports (refined in #5693).
- Dashboard: a widget setting may take `options_from` (read when the picker
  opens) with an `empty` sentence; `default_title`; a Details link may name
  settings ("/plugins/prometheus/targets/{target}"). Prometheus is grouped
  with the integrations.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

plugins/prometheus/__init__.py

@@ -12,6 +12,34 @@ def setup(app: "Quart") -> None:
     global _app
     _app = app
     from .models import PrometheusTarget  # noqa: F401 — registers the table
+    register_widgets()
+
+
+def register_widgets() -> None:
+    """One dashboard widget for every target and template (System 181's
+    ruling). Its target and template lists are read when the widget picker
+    opens, so a new target or template appears without a restart."""
+    from steward.core.widgets import register_widget
+
+    from . import library
+    register_widget({
+        "key": "prometheus",
+        "label": "Prometheus",
+        "description": "One scrape target drawn by a widget template: Fabled Archives, or one you copy in",
+        "hx_url": "/plugins/prometheus/widget",
+        "detail_url": "/plugins/prometheus/targets/{target}",
+        "plugin": "prometheus",
+        "poll": True,
+        "default_title": library.default_title,
+        "params": [
+            {"key": "target", "label": "Target", "type": "select", "default": "",
+             "options_from": library.target_options,
+             "empty": "No targets yet: add one under Prometheus first."},
+            {"key": "template", "label": "Template", "type": "select", "default": "",
+             "options_from": library.template_options,
+             "empty": "No templates yet."},
+        ],
+    })
 
 
 def get_scheduled_tasks() -> list:

plugins/prometheus/library.py

@@ -0,0 +1,71 @@
+"""The widget templates a Prometheus widget can draw with.
+
+Bundled templates ship in plugins/prometheus/widgets/*.yaml, read once and
+read-only; their key is "bundled:<file name>". Templates an admin copies in
+join them with the template library (#5692). Steward fetches none from
+anywhere (System 181's ruling).
+"""
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from functools import lru_cache
+from pathlib import Path
+
+from sqlalchemy import select
+
+from .models import PrometheusTarget
+from .template_format import Problem, Template, TemplateError, check
+
+BUNDLED = Path(__file__).parent / "widgets"
+
+
+@dataclass
+class Entry:
+    key: str
+    name: str
+    bundled: bool
+    text: str
+    template: Template | None = None
+    problems: list[Problem] = field(default_factory=list)
+
+
+def _entry(key: str, fallback_name: str, text: str, bundled: bool) -> Entry:
+    try:
+        template = check(text)
+    except TemplateError as exc:
+        return Entry(key, fallback_name, bundled, text, None, exc.problems)
+    return Entry(key, template.name, bundled, text, template)
+
+
+@lru_cache(maxsize=1)
+def bundled() -> tuple[Entry, ...]:
+    return tuple(_entry(f"bundled:{p.stem}", p.stem, p.read_text(encoding="utf-8"), True)
+                 for p in sorted(BUNDLED.glob("*.yaml")))
+
+
+async def entries(db) -> list[Entry]:
+    """Every template, by name."""
+    return sorted(bundled(), key=lambda e: e.name.lower())
+
+
+async def find(db, key: str) -> Entry | None:
+    return next((e for e in await entries(db) if e.key == key), None)
+
+
+# ── The widget's settings ────────────────────────────────────────────────────
+
+async def target_options(db) -> list[dict]:
+    rows = (await db.execute(select(PrometheusTarget.id, PrometheusTarget.name)
+                             .order_by(PrometheusTarget.name))).all()
+    return [{"value": tid, "label": name} for tid, name in rows]
+
+
+async def template_options(db) -> list[dict]:
+    return [{"value": e.key, "label": f"{e.name} (built in)" if e.bundled else e.name}
+            for e in await entries(db) if e.template is not None]
+
+
+async def default_title(db, config: dict) -> str | None:
+    """A new widget is named after its template: "Fabled Archives"."""
+    entry = await find(db, str(config.get("template") or ""))
+    return entry.name if entry and entry.template else None

plugins/prometheus/routes.py

@@ -308,3 +308,57 @@ async def series_detail(target_id: str):
         "prometheus/series.html", target=target, resource=resource, metric=metric, labels=labels,
         now=shown_value(metric, now_row.value if now_row else None), hours=hours, ranges=RANGES,
         points_json=json.dumps([(t.isoformat(), v) for t, v in series]), stamp_axis=stamp_axis)
+
+
+# ── The dashboard widget ─────────────────────────────────────────────────────
+
+SPARK_HOURS = 24
+
+
+def spark_heights(values: list[float]) -> list[float]:
+    """Bar heights in % of the series' own peak over the window (System
+    154's ruling on small history bars); a flat zero draws flat."""
+    peak = max((abs(v) for v in values), default=0.0)
+    return [round(100.0 * abs(v) / peak, 1) if peak else 0.0 for v in values]
+
+
+@prometheus_bp.get("/widget")
+@require_role(UserRole.viewer)
+async def widget():
+    """The Prometheus dashboard widget: one target drawn by one template
+    (System 181's ruling), in widget shape. Reads stored values only."""
+    from . import library
+    from .template_view import render as draw
+
+    target_id = request.args.get("target") or ""
+    key = request.args.get("template") or ""
+    now = datetime.now(timezone.utc)
+    async with current_app.db_sessionmaker() as db:
+        target = await db.get(PrometheusTarget, target_id) if target_id else None
+        entry = await library.find(db, key) if key else None
+        problem = ("This widget has no target; remove it and add it again with one." if not target_id
+                   else "The target this widget showed has been deleted." if target is None
+                   else "This widget has no template; remove it and add it again with one." if not key
+                   else "The template this widget used is gone." if entry is None
+                   else f"The template “{entry.name}” has a problem: {entry.problems[0]}"
+                   if entry.template is None else "")
+        if problem:
+            return await render_template("prometheus/widget.html", problem=problem, target=None,
+                                         view=None)
+        latest = await latest_values(db, SOURCE, target.name, include_children=True)
+        view = draw(entry.template, target.name, latest, now)
+        if view.wants:
+            since = now - timedelta(hours=SPARK_HOURS)
+            history = {}
+            for resource, metric in view.wants:
+                got = await points(db, SOURCE, resource, metric, since, target=SPARK_HOURS)
+                history[(resource, metric)] = spark_heights([v for _, v in got])
+            view = draw(entry.template, target.name, latest, now, history=history)
+    return await render_template("prometheus/widget.html", problem="", view=view,
+                                 target=view_target(target, now), spark_hours=SPARK_HOURS)
+
+
+def view_target(target: PrometheusTarget, now: datetime) -> dict:
+    state = target_state(target, now)
+    return {"id": target.id, "name": target.name, "reach": state.reach, "says": state.says,
+            "ok_at": target.last_ok_at, "scraped_at": target.last_scraped_at}

plugins/prometheus/templates/prometheus/widget.html

@@ -0,0 +1,70 @@
+{# prometheus/widget.html — the Prometheus dashboard widget (#5691): one
+   target drawn by one template (template_view.render). A verdict naming the
+   worst thing, then each block: its problems first with why, figures in their
+   units beside a bar or the last day's bars, and the healthy rest packed after,
+   past REST rows as "+N more healthy". #}
+{% from "_macros.html" import status_dot %}
+{% set REST = 6 %}
+{% set dot = {"crit": "crit", "warn": "warn", "ok": "ok", "dim": "paused"} %}
+{% set tone = {"crit": "tone-crit", "warn": "tone-warn", "ok": "tone-ok", "dim": "tone-dim"} %}
+{% macro shown(v) -%}
+  {%- if v.moment %}{{ v.moment|ago }}{% elif v.text %}{{ v.text }}{% else %}—{% endif -%}
+{%- endmacro %}
+{% if problem %}
+<div class="w-verdict tone-warn">{{ problem }}</div>
+{% else %}
+{% set down = target.reach in ("down", "stale") %}
+<div class="w-verdict {{ 'tone-crit' if target.reach == 'down' else tone[view.tone] }}">
+  {%- if target.reach == "down" %}{{ target.name }} isn't answering{% elif down %}{{ target.name }} isn't being scraped{% endif %}
+  {%- if down %}{% if target.ok_at %}; these values are from {{ target.ok_at|ago }}{% endif %}. {% endif -%}
+  {{ view.verdict }}
+</div>
+{% for b in view.blocks %}
+<div class="w-section">{{ b.title }}</div>
+{% if b.note %}
+<div class="w-note tone-warn">{{ b.note }}</div>
+{% elif b.kind == "info" %}
+<div class="w-note">{% for r in b.rows %}{{ r.name }}{% if not loop.last %}; {% endif %}{% endfor %}</div>
+{% elif b.kind == "table" %}
+<table class="w-table">
+  <thead><tr><th></th>{% for c in b.columns %}<th>{{ c }}</th>{% endfor %}</tr></thead>
+  <tbody>
+  {% for r in b.rows %}
+  <tr><td class="w-name" title="{{ r.name }}">{{ r.name }}</td>{% for c in r.cells %}<td>{{ shown(c) }}</td>{% endfor %}</tr>
+  {% endfor %}
+  </tbody>
+</table>
+{% else %}
+{% set problems = b.rows | selectattr("tone", "in", ["crit", "warn"]) | list %}
+{% set rest = b.rows | rejectattr("tone", "in", ["crit", "warn"]) | list %}
+{% set listed = problems + rest[:REST] %}
+<table class="w-table">
+  {% if b.columns %}<thead><tr><th></th><th>Now</th>{% for c in b.columns %}<th class="w-wide-only">{{ c }}</th>{% endfor %}</tr></thead>{% endif %}
+  <tbody>
+  {% for r in listed %}
+  <tr>
+    <td class="w-name" title="{{ r.name }}">
+      {{- status_dot(dot[r.tone]) -}}
+      {{ r.name }}
+      {%- if r.why %}<span class="w-sub {{ tone[r.tone] }}">{{ r.why }}</span>{% endif %}
+    </td>
+    <td class="{{ tone[r.tone] if r.tone in ('crit', 'warn') }}">
+      {%- if r.history %}<span class="w-bars w-wide-only" aria-hidden="true" title="The last {{ spark_hours }}h, scaled to its own peak">
+        {%- for h in r.history %}<i style="height:{{ [h, 4]|max }}%"></i>{% endfor -%}
+      </span>{% endif -%}
+      {{ shown(r) }}
+      {%- if r.bar is not none %}<div class="meter w-wide-only" title="{{ (r.bar * 100)|round|int }}%"><div class="meter-fill meter-{{ r.tone if r.tone in ('crit', 'warn') else 'ok' }}" style="width:{{ (r.bar * 100)|round(1) }}%"></div></div>{% endif -%}
+    </td>
+    {% for c in r.cells %}<td class="w-wide-only">{{ shown(c) }}</td>{% endfor %}
+  </tr>
+  {% endfor %}
+  </tbody>
+</table>
+{% if rest|length > REST %}<div class="w-more">+{{ rest|length - REST }} more healthy</div>{% endif %}
+{% endif %}
+{% endfor %}
+<div class="w-note">
+  {%- if session.user_id %}<a href="/plugins/prometheus/targets/{{ target.id }}">{{ target.name }}</a>{% else %}{{ target.name }}{% endif %}
+  {%- if target.scraped_at %}, scraped {{ target.scraped_at|ago }}{% endif %}.
+</div>
+{% endif %}

plugins/prometheus/widgets/fabled_archives.yaml

@@ -0,0 +1,55 @@
+# Fabled Archives, from its /metrics on the LAN-only backup listener (:8081).
+# Written against what Archives reports; see docs/plugins/prometheus.md.
+name: Fabled Archives
+description: Backup targets, replication jobs, storage, integrity and the version running.
+version: 1
+expects: fabledarchives_
+blocks:
+  - states:
+      title: Backup targets
+      metric: fabledarchives_backup_target_state
+      name: target
+      healthy: [fresh]
+      quiet: [disabled, waiting]
+      warning: [late]
+      columns:
+        - {label: Last backup, metric: fabledarchives_backup_target_last_backup_timestamp_seconds}
+        - {label: Stored, metric: fabledarchives_backup_target_stored_bytes}
+  - states:
+      title: Replication
+      metric: fabledarchives_replication_job_state
+      name: job
+      healthy: [ok, running]
+      warning: [late, warning, never]
+      columns:
+        - {label: Next run, metric: fabledarchives_replication_job_next_run_timestamp_seconds}
+  - values:
+      title: Storage and integrity
+      items:
+        - label: Disk free
+          metric: fabledarchives_data_disk_free_bytes
+          of: fabledarchives_data_disk_size_bytes
+          warn_below: 10%
+          crit_below: 5%
+          sparkline: true
+        - label: Files checked
+          metric: fabledarchives_files_checked
+          of: fabledarchives_files
+          unit: count
+        - label: Malware held
+          metric: fabledarchives_malware_held_files
+          unit: count
+          warn_above: 0
+        - label: Notification channels failing
+          metric: fabledarchives_notification_channels_failing
+          unit: count
+          warn_above: 0
+  - table:
+      title: Stored per person
+      metric: fabledarchives_person_stored_bytes
+      rows: person
+      columns: kind
+  - info:
+      title: Running
+      metric: fabledarchives_build_info
+      labels: [version, channel, commit]