feat(prometheus): Fabled Archives template v2 against the live series; a state past its deadline is overdue (#5693)

Read from what Steward stored for Archives on 2026-10-14: the nightly
Restic replication job had said "running" since the first scrape two days
earlier, with no good copy yet and past both its late and missing times.
Archives ranks "running" above lateness, so the widget called it healthy.

- states blocks take warn_past / crit_past: a deadline metric per row (a
  *_timestamp_seconds). Once it has passed, a healthy row reads as a
  warning or a problem: "FullNightly-Restic is running, 11h 53m overdue".
  Quiet and already-failing rows keep their own reading.
- The Archives template (version 2) uses the job's late_at / missing_at
  deadlines and shows its last good copy beside the next run. It says why
  section_up, the queue and the HTTP/audit counters are left out.
- tests/plugins/prometheus/archives_live.py is the series as captured
  live, and test_archives_template.py renders the template over them.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

plugins/prometheus/template_format.py

@@ -284,7 +284,8 @@ class _Check:
 
 # Each block: the keys it takes, those it needs, and its title when none is given.
 _BLOCKS = {
-    "states": (("title", "metric", "where", "name", "healthy", "quiet", "warning", "columns"),
+    "states": (("title", "metric", "where", "name", "healthy", "quiet", "warning", "warn_past",
+                "crit_past", "columns"),
                ("metric", "healthy"), "States"),
     "values": (("title", "items"), ("items",), "Figures"),
     "times": (("title", "items"), ("items",), "Times"),
@@ -301,7 +302,12 @@ _COLUMN = ("label", "metric", "unit")
 def _states(c: _Check, m: _Map, ctx: str) -> dict:
     spec = {"metric": c.metric(m, "metric", ctx), "where": c.where(m, ctx), "name": c.label(m, "name", ctx),
             "healthy": c.words(m, "healthy", ctx), "quiet": c.words(m, "quiet", ctx),
-            "warning": c.words(m, "warning", ctx), "columns": []}
+            "warning": c.words(m, "warning", ctx), "columns": [],
+            # A deadline per row (a *_timestamp_seconds): once it has passed, a
+            # healthy row reads as a warning or a problem. A job that says
+            # "running" for days is still overdue.
+            "warn_past": c.metric(m, "warn_past", ctx) if "warn_past" in m else "",
+            "crit_past": c.metric(m, "crit_past", ctx) if "crit_past" in m else ""}
     if "healthy" in m and not spec["healthy"] and isinstance(m["healthy"], list):
         c.add(m.lines["healthy"], f"`healthy` in {ctx} needs at least one state.")
     seen: dict[str, str] = {}

plugins/prometheus/template_view.py

@@ -104,6 +104,29 @@ def _ordered(rows: list[Row]) -> list[Row]:
 
 # ── Blocks ───────────────────────────────────────────────────────────────────
 
+def _same_row(index, metric: str, labels: str, name_label: str, name: str) -> list[Sample]:
+    """The series of `metric` belonging to a state set's row: the same labels,
+    or else the same value of the label that names it."""
+    match = [s for s in index.get(metric, []) if s.labels == labels]
+    if not match and name:
+        match = [s for s in index.get(metric, []) if s.pairs.get(name_label) == name]
+    return match
+
+
+def _overdue(spec: dict, index, labels: str, name: str, now: datetime) -> tuple[str, float] | None:
+    """(tone, seconds past) when the row's crit_past or warn_past deadline has passed."""
+    for level in ("crit", "warn"):
+        metric = spec[f"{level}_past"]
+        if not metric:
+            continue
+        match = _same_row(index, metric, labels, spec["name"], name)
+        if len(match) == 1 and match[0].value > 0:
+            past = now.timestamp() - match[0].value
+            if past > 0:
+                return level, past
+    return None
+
+
 def _states(block: Block, index, now) -> BlockView:
     spec = block.spec
     view = BlockView("states", block.title, columns=[c["label"] for c in spec["columns"]])
@@ -130,13 +153,14 @@ def _states(block: Block, index, now) -> BlockView:
             else:
                 tones.append("crit")
         tone = (min(tones, key=TONE_ORDER.__getitem__) if tones else "warn")
-        row = Row(name=name or f["labels"] or block.title, tone=tone, text=f["state"] or "—",
-                  why="" if tone in ("ok", "dim") else (f"is {f['state']}" if current else "reports no state"),
+        why = "" if tone in ("ok", "dim") else (f"is {f['state']}" if current else "reports no state")
+        overdue = _overdue(spec, index, f["labels"], name, now) if tone == "ok" else None
+        if overdue:
+            tone, why = overdue[0], f"is {f['state']}, {human_duration(overdue[1])} overdue"
+        row = Row(name=name or f["labels"] or block.title, tone=tone, text=f["state"] or "—", why=why,
                   resource=f["resource"], metric=spec["metric"])
         for col in spec["columns"]:
-            match = [s for s in index.get(col["metric"], []) if s.labels == f["labels"]]
-            if not match and name:
-                match = [s for s in index.get(col["metric"], []) if s.pairs.get(spec["name"]) == name]
+            match = _same_row(index, col["metric"], f["labels"], spec["name"], name)
             row.cells.append(shown(col["metric"], col["unit"], match[0].value if len(match) == 1 else None))
         view.rows.append(row)
     view.rows = _ordered(view.rows)

plugins/prometheus/widgets/fabled_archives.yaml

@@ -1,8 +1,16 @@
 # Fabled Archives, from its /metrics on the LAN-only backup listener (:8081).
-# Written against what Archives reports; see docs/plugins/prometheus.md.
+# Written against the series Steward stores for Archives (#5693); see
+# docs/plugins/prometheus.md.
+#
+# Left out on purpose:
+# - fabledarchives_section_up: a section Archives can't read stops reporting
+#   its series, and the rows here that read them then say "not reported".
+# - the background queue (fabledarchives_jobs, *_job_attempts_*), HTTP
+#   requests and the audit counters: they belong on charts and alert rules,
+#   not on a glance.
 name: Fabled Archives
 description: Backup targets, replication jobs, storage, integrity and the version running.
-version: 1
+version: 2
 expects: fabledarchives_
 blocks:
   - states:
@@ -19,9 +27,14 @@ blocks:
       title: Replication
       metric: fabledarchives_replication_job_state
       name: job
+      # Archives ranks "running" above lateness: a run that never ends still
+      # reads "running". Its own deadlines say when it is overdue.
       healthy: [ok, running]
       warning: [late, warning, never]
+      warn_past: fabledarchives_replication_job_late_at_timestamp_seconds
+      crit_past: fabledarchives_replication_job_missing_at_timestamp_seconds
       columns:
+        - {label: Last good copy, metric: fabledarchives_replication_job_last_success_timestamp_seconds}
         - {label: Next run, metric: fabledarchives_replication_job_next_run_timestamp_seconds}
   - values:
       title: Storage and integrity