From 41d9d0021611e03e5e68f7f542cd1a02c632f915 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Wed, 14 Oct 2026 09:43:38 -0400 Subject: [PATCH] feat(prometheus): Fabled Archives template v2 against the live series; a state past its deadline is overdue (#5693) Read from what Steward stored for Archives on 2026-10-14: the nightly Restic replication job had said "running" since the first scrape two days earlier, with no good copy yet and past both its late and missing times. Archives ranks "running" above lateness, so the widget called it healthy. - states blocks take warn_past / crit_past: a deadline metric per row (a *_timestamp_seconds). Once it has passed, a healthy row reads as a warning or a problem: "FullNightly-Restic is running, 11h 53m overdue". Quiet and already-failing rows keep their own reading. - The Archives template (version 2) uses the job's late_at / missing_at deadlines and shows its last good copy beside the next run. It says why section_up, the queue and the HTTP/audit counters are left out. - tests/plugins/prometheus/archives_live.py is the series as captured live, and test_archives_template.py renders the template over them. Co-Authored-By: Claude Opus 5.5 --- plugins/prometheus/template_format.py | 10 ++- plugins/prometheus/template_view.py | 34 ++++++++-- .../prometheus/widgets/fabled_archives.yaml | 17 ++++- tests/plugins/prometheus/archives_live.py | 62 +++++++++++++++++++ .../prometheus/test_archives_template.py | 47 ++++++++++++++ tests/plugins/prometheus/test_templates.py | 35 ++++++++++- 6 files changed, 195 insertions(+), 10 deletions(-) create mode 100644 tests/plugins/prometheus/archives_live.py create mode 100644 tests/plugins/prometheus/test_archives_template.py diff --git a/plugins/prometheus/template_format.py b/plugins/prometheus/template_format.py index b257c85..87b3459 100644 --- a/plugins/prometheus/template_format.py +++ b/plugins/prometheus/template_format.py @@ -284,7 +284,8 @@ class _Check: # Each block: the keys it takes, those it needs, and its title when none is given. _BLOCKS = { - "states": (("title", "metric", "where", "name", "healthy", "quiet", "warning", "columns"), + "states": (("title", "metric", "where", "name", "healthy", "quiet", "warning", "warn_past", + "crit_past", "columns"), ("metric", "healthy"), "States"), "values": (("title", "items"), ("items",), "Figures"), "times": (("title", "items"), ("items",), "Times"), @@ -301,7 +302,12 @@ _COLUMN = ("label", "metric", "unit") def _states(c: _Check, m: _Map, ctx: str) -> dict: spec = {"metric": c.metric(m, "metric", ctx), "where": c.where(m, ctx), "name": c.label(m, "name", ctx), "healthy": c.words(m, "healthy", ctx), "quiet": c.words(m, "quiet", ctx), - "warning": c.words(m, "warning", ctx), "columns": []} + "warning": c.words(m, "warning", ctx), "columns": [], + # A deadline per row (a *_timestamp_seconds): once it has passed, a + # healthy row reads as a warning or a problem. A job that says + # "running" for days is still overdue. + "warn_past": c.metric(m, "warn_past", ctx) if "warn_past" in m else "", + "crit_past": c.metric(m, "crit_past", ctx) if "crit_past" in m else ""} if "healthy" in m and not spec["healthy"] and isinstance(m["healthy"], list): c.add(m.lines["healthy"], f"`healthy` in {ctx} needs at least one state.") seen: dict[str, str] = {} diff --git a/plugins/prometheus/template_view.py b/plugins/prometheus/template_view.py index 1644d54..cce3cc8 100644 --- a/plugins/prometheus/template_view.py +++ b/plugins/prometheus/template_view.py @@ -104,6 +104,29 @@ def _ordered(rows: list[Row]) -> list[Row]: # ── Blocks ─────────────────────────────────────────────────────────────────── +def _same_row(index, metric: str, labels: str, name_label: str, name: str) -> list[Sample]: + """The series of `metric` belonging to a state set's row: the same labels, + or else the same value of the label that names it.""" + match = [s for s in index.get(metric, []) if s.labels == labels] + if not match and name: + match = [s for s in index.get(metric, []) if s.pairs.get(name_label) == name] + return match + + +def _overdue(spec: dict, index, labels: str, name: str, now: datetime) -> tuple[str, float] | None: + """(tone, seconds past) when the row's crit_past or warn_past deadline has passed.""" + for level in ("crit", "warn"): + metric = spec[f"{level}_past"] + if not metric: + continue + match = _same_row(index, metric, labels, spec["name"], name) + if len(match) == 1 and match[0].value > 0: + past = now.timestamp() - match[0].value + if past > 0: + return level, past + return None + + def _states(block: Block, index, now) -> BlockView: spec = block.spec view = BlockView("states", block.title, columns=[c["label"] for c in spec["columns"]]) @@ -130,13 +153,14 @@ def _states(block: Block, index, now) -> BlockView: else: tones.append("crit") tone = (min(tones, key=TONE_ORDER.__getitem__) if tones else "warn") - row = Row(name=name or f["labels"] or block.title, tone=tone, text=f["state"] or "—", - why="" if tone in ("ok", "dim") else (f"is {f['state']}" if current else "reports no state"), + why = "" if tone in ("ok", "dim") else (f"is {f['state']}" if current else "reports no state") + overdue = _overdue(spec, index, f["labels"], name, now) if tone == "ok" else None + if overdue: + tone, why = overdue[0], f"is {f['state']}, {human_duration(overdue[1])} overdue" + row = Row(name=name or f["labels"] or block.title, tone=tone, text=f["state"] or "—", why=why, resource=f["resource"], metric=spec["metric"]) for col in spec["columns"]: - match = [s for s in index.get(col["metric"], []) if s.labels == f["labels"]] - if not match and name: - match = [s for s in index.get(col["metric"], []) if s.pairs.get(spec["name"]) == name] + match = _same_row(index, col["metric"], f["labels"], spec["name"], name) row.cells.append(shown(col["metric"], col["unit"], match[0].value if len(match) == 1 else None)) view.rows.append(row) view.rows = _ordered(view.rows) diff --git a/plugins/prometheus/widgets/fabled_archives.yaml b/plugins/prometheus/widgets/fabled_archives.yaml index e38fb12..6c6df9f 100644 --- a/plugins/prometheus/widgets/fabled_archives.yaml +++ b/plugins/prometheus/widgets/fabled_archives.yaml @@ -1,8 +1,16 @@ # Fabled Archives, from its /metrics on the LAN-only backup listener (:8081). -# Written against what Archives reports; see docs/plugins/prometheus.md. +# Written against the series Steward stores for Archives (#5693); see +# docs/plugins/prometheus.md. +# +# Left out on purpose: +# - fabledarchives_section_up: a section Archives can't read stops reporting +# its series, and the rows here that read them then say "not reported". +# - the background queue (fabledarchives_jobs, *_job_attempts_*), HTTP +# requests and the audit counters: they belong on charts and alert rules, +# not on a glance. name: Fabled Archives description: Backup targets, replication jobs, storage, integrity and the version running. -version: 1 +version: 2 expects: fabledarchives_ blocks: - states: @@ -19,9 +27,14 @@ blocks: title: Replication metric: fabledarchives_replication_job_state name: job + # Archives ranks "running" above lateness: a run that never ends still + # reads "running". Its own deadlines say when it is overdue. healthy: [ok, running] warning: [late, warning, never] + warn_past: fabledarchives_replication_job_late_at_timestamp_seconds + crit_past: fabledarchives_replication_job_missing_at_timestamp_seconds columns: + - {label: Last good copy, metric: fabledarchives_replication_job_last_success_timestamp_seconds} - {label: Next run, metric: fabledarchives_replication_job_next_run_timestamp_seconds} - values: title: Storage and integrity diff --git a/tests/plugins/prometheus/archives_live.py b/tests/plugins/prometheus/archives_live.py new file mode 100644 index 0000000..b4e2df2 --- /dev/null +++ b/tests/plugins/prometheus/archives_live.py @@ -0,0 +1,62 @@ +"""Fabled Archives' series as Steward stored them for the Archives target, +read from the live instance at 2026-10-14 13:40:14 UTC (#5693): every series +the bundled Fabled Archives template reads, and a few it leaves out on +purpose (section_up, the target and job info series). Values are as stored: +labels after the target's name, timestamps in seconds. + +At that moment the nightly Restic replication job had reported "running" +since Steward's first scrape two days before, with no good copy yet, and was +past both its late and missing times; Archives ranks "running" above +lateness, so its state alone says nothing is wrong. +""" +from datetime import datetime, timezone +from types import SimpleNamespace as NS + +TARGET = "Archives" +AT = datetime.fromtimestamp(1791985214.02, timezone.utc) # fabledarchives_read_timestamp_seconds + +_STEWARD = "target=Steward,target_id=44571987-c446-4f95-8ee5-306995e5e568" +_JOB = "job=FullNightly-Restic,job_id=1e8ee176-22dc-4518-8784-8aa5cbfabb40" +_ME = "person=Bryan Van Deusen,person_id=d6f3deff-f4df-4c09-a618-7f6124531298" + +SERIES: list[tuple[str, str, float]] = [ + ("", "fabledarchives_data_disk_free_bytes", 17800573747200.0), + ("", "fabledarchives_data_disk_size_bytes", 46059916492800.0), + ("", "fabledarchives_files", 100462.0), + ("", "fabledarchives_files_checked", 100462.0), + ("", "fabledarchives_malware_held_files", 2.0), + ("", "fabledarchives_notification_channels", 0.0), + ("", "fabledarchives_notification_channels_failing", 0.0), + ("", "fabledarchives_read_timestamp_seconds", 1791985214.02), + ("", "fabledarchives_video_copies_bytes", 0.0), + *((f"{_STEWARD},state={s}", "fabledarchives_backup_target_state", float(s == "fresh")) + for s in ("disabled", "fresh", "late", "missing", "waiting")), + (_STEWARD, "fabledarchives_backup_target_expected_interval_seconds", 86400.0), + (_STEWARD, "fabledarchives_backup_target_last_backup_timestamp_seconds", 1791947022.37), + (_STEWARD, "fabledarchives_backup_target_late_at_timestamp_seconds", 1792055022.37), + (_STEWARD, "fabledarchives_backup_target_missing_at_timestamp_seconds", 1792119822.37), + (_STEWARD, "fabledarchives_backup_target_stored_bytes", 389120.0), + (f"{_STEWARD},slug=steward,owner=Bryan Van Deusen", "fabledarchives_backup_target_info", 1.0), + *((f"{_JOB},state={s}", "fabledarchives_replication_job_state", float(s == "running")) + for s in ("check_failed", "failing", "late", "missing", "never", "ok", "running", "warning")), + (_JOB, "fabledarchives_replication_job_enabled", 1.0), + (_JOB, "fabledarchives_replication_job_expected_interval_seconds", 86400.0), + (_JOB, "fabledarchives_replication_job_late_at_timestamp_seconds", 1791942396.87), + (_JOB, "fabledarchives_replication_job_missing_at_timestamp_seconds", 1791942396.87), + (_JOB, "fabledarchives_replication_job_next_run_timestamp_seconds", 1792047600.0), + (f"{_JOB},method=restic,source=everyone,destination=Hetzner-storage-box-1 Restic," + "schedule=CRON_TZ=America/New_York 0 3 * * *", "fabledarchives_replication_job_info", 1.0), + (f"{_ME},kind=files", "fabledarchives_person_stored_bytes", 473016059695.0), + (f"{_ME},kind=trash", "fabledarchives_person_stored_bytes", 0.0), + (f"{_ME},kind=versions", "fabledarchives_person_stored_bytes", 0.0), + (_ME, "fabledarchives_person_counted_bytes", 473016059695.0), + *((f"section={s}", "fabledarchives_section_up", 1.0) + for s in ("disk", "notifications", "people", "queue", "replication", "sign_ins", "storage", "targets")), + ("version=2026.10.12.1717,channel=dev,commit=3605d0a", "fabledarchives_build_info", 1.0), +] + + +def latest(series=SERIES) -> list: + """PluginMetricLatest-like rows for the Archives target.""" + return [NS(resource_name=f"{TARGET}:{labels}" if labels else TARGET, metric_name=metric, value=value, + recorded_at=AT) for labels, metric, value in series] diff --git a/tests/plugins/prometheus/test_archives_template.py b/tests/plugins/prometheus/test_archives_template.py new file mode 100644 index 0000000..0539d02 --- /dev/null +++ b/tests/plugins/prometheus/test_archives_template.py @@ -0,0 +1,47 @@ +"""The bundled Fabled Archives template (#5693) over the series Archives +actually reported (archives_live.py): what the widget said about it.""" +from plugins.prometheus import library +from plugins.prometheus.template_view import render +from tests.plugins.prometheus.archives_live import AT, SERIES, TARGET, latest + +ARCHIVES = library.bundled()[0].template + + +def _rows(view, title): + return {r.name: r for b in view.blocks if b.title == title for r in b.rows} + + +def test_a_replication_that_never_finishes_reads_as_overdue_not_running(): + view = render(ARCHIVES, TARGET, latest(), AT) + assert view.tone == "crit" and view.note == "" + assert view.verdict == "2 need a look. Replication: FullNightly-Restic is running, 11h 53m overdue." + job = _rows(view, "Replication")["FullNightly-Restic"] + assert (job.tone, job.text) == ("crit", "running") + last_good, next_run = job.cells + assert last_good["text"] == "—" and next_run["moment"] is not None # no good copy yet + assert _rows(view, "Storage and integrity")["Malware held"].why == "above 0" + + +def test_everything_else_archives_reported_reads_as_well(): + view = render(ARCHIVES, TARGET, latest(), AT) + assert all(not b.note for b in view.blocks), [b.note for b in view.blocks] + steward = _rows(view, "Backup targets")["Steward"] + assert (steward.tone, steward.text) == ("ok", "fresh") and steward.cells[0]["moment"] is not None + storage = _rows(view, "Storage and integrity") + assert storage["Disk free"].tone == "ok" and "of" in storage["Disk free"].text + assert (storage["Files checked"].text, storage["Files checked"].tone) == ("100,462 of 100,462", "ok") + assert storage["Notification channels failing"].tone == "ok" + assert list(_rows(view, "Stored per person")) == ["Bryan Van Deusen"] + assert list(_rows(view, "Running")) == ["version 2026.10.12.1717 · channel dev · commit 3605d0a"] + + +def test_a_finished_copy_clears_it(): + """The next night's run succeeds: Archives reports ok with a last good + copy and deadlines a day out, and the widget says so.""" + later = AT.timestamp() + 86400 + done = [(labels, metric, (float(labels.endswith("state=ok")) if metric.endswith("job_state") else value)) + for labels, metric, value in SERIES] + done = [(lb, m, later + 30 * 3600 if m.endswith(("late_at_timestamp_seconds", "missing_at_timestamp_seconds")) + and lb.startswith("job=") else v) for lb, m, v in done] + job = _rows(render(ARCHIVES, TARGET, latest(done), AT), "Replication")["FullNightly-Restic"] + assert (job.tone, job.text, job.why) == ("ok", "ok", "") diff --git a/tests/plugins/prometheus/test_templates.py b/tests/plugins/prometheus/test_templates.py index dc891fd..e4bceb2 100644 --- a/tests/plugins/prometheus/test_templates.py +++ b/tests/plugins/prometheus/test_templates.py @@ -120,7 +120,7 @@ blocks: "Line 2: `version` is the template's own version: a whole number from 1. " "Raise it when you change a template you have shared.", "Line 8: Block 1 (states) has no `colour`. It takes: title, metric, where, name, healthy, " - "quiet, warning, columns.", + "quiet, warning, warn_past, crit_past, columns.", "Line 9: Block 2 is “gauge”; the kinds are: states, values, times, table, info.", "Line 12: `metric` in an item of block 3 (values) should be a metric name as Steward stores it, " "like fabledarchives_files (a counter is stored as _per_sec).", @@ -236,3 +236,36 @@ def test_all_well(): ("", "fa_malware_held_files"): 0.0} view = _view(healthy) assert (view.tone, view.verdict) == ("ok", "All well.") + + +def test_a_healthy_state_past_its_deadline_is_overdue(): + """warn_past / crit_past (#5693): a row whose state is healthy but whose + deadline has passed reads as a warning or a problem; a quiet or already + failing row keeps its own reading.""" + text = """\ +name: Jobs +version: 1 +blocks: + - states: + title: Jobs + metric: j_state + name: job + healthy: [ok, running] + quiet: [paused] + warn_past: j_late_at_timestamp_seconds + crit_past: j_missing_at_timestamp_seconds +""" + series = {} + for job, state, late, missing in (("A", "running", before(hours=2), before(hours=1) + 7200), + ("B", "running", before(hours=5), before(hours=3)), + ("C", "ok", before(hours=-1), before(hours=-2)), + ("D", "paused", before(hours=9), before(hours=8))): + for s in ("ok", "running", "paused"): + series[(f"job={job},state={s}", "j_state")] = float(s == state) + series[(f"job={job}", "j_late_at_timestamp_seconds")] = late + series[(f"job={job}", "j_missing_at_timestamp_seconds")] = missing + rows = {r.name: r for r in render(check(text), T, latest(series), NOW).blocks[0].rows} + assert (rows["A"].tone, rows["A"].why) == ("warn", "is running, 2h overdue") + assert (rows["B"].tone, rows["B"].why) == ("crit", "is running, 3h overdue") + assert (rows["C"].tone, rows["C"].why) == ("ok", "") + assert rows["D"].tone == "dim"