diff --git a/plugins/prometheus/template_format.py b/plugins/prometheus/template_format.py index b257c85..87b3459 100644 --- a/plugins/prometheus/template_format.py +++ b/plugins/prometheus/template_format.py @@ -284,7 +284,8 @@ class _Check: # Each block: the keys it takes, those it needs, and its title when none is given. _BLOCKS = { - "states": (("title", "metric", "where", "name", "healthy", "quiet", "warning", "columns"), + "states": (("title", "metric", "where", "name", "healthy", "quiet", "warning", "warn_past", + "crit_past", "columns"), ("metric", "healthy"), "States"), "values": (("title", "items"), ("items",), "Figures"), "times": (("title", "items"), ("items",), "Times"), @@ -301,7 +302,12 @@ _COLUMN = ("label", "metric", "unit") def _states(c: _Check, m: _Map, ctx: str) -> dict: spec = {"metric": c.metric(m, "metric", ctx), "where": c.where(m, ctx), "name": c.label(m, "name", ctx), "healthy": c.words(m, "healthy", ctx), "quiet": c.words(m, "quiet", ctx), - "warning": c.words(m, "warning", ctx), "columns": []} + "warning": c.words(m, "warning", ctx), "columns": [], + # A deadline per row (a *_timestamp_seconds): once it has passed, a + # healthy row reads as a warning or a problem. A job that says + # "running" for days is still overdue. + "warn_past": c.metric(m, "warn_past", ctx) if "warn_past" in m else "", + "crit_past": c.metric(m, "crit_past", ctx) if "crit_past" in m else ""} if "healthy" in m and not spec["healthy"] and isinstance(m["healthy"], list): c.add(m.lines["healthy"], f"`healthy` in {ctx} needs at least one state.") seen: dict[str, str] = {} diff --git a/plugins/prometheus/template_view.py b/plugins/prometheus/template_view.py index 1644d54..cce3cc8 100644 --- a/plugins/prometheus/template_view.py +++ b/plugins/prometheus/template_view.py @@ -104,6 +104,29 @@ def _ordered(rows: list[Row]) -> list[Row]: # ── Blocks ─────────────────────────────────────────────────────────────────── +def _same_row(index, metric: str, labels: str, name_label: str, name: str) -> list[Sample]: + """The series of `metric` belonging to a state set's row: the same labels, + or else the same value of the label that names it.""" + match = [s for s in index.get(metric, []) if s.labels == labels] + if not match and name: + match = [s for s in index.get(metric, []) if s.pairs.get(name_label) == name] + return match + + +def _overdue(spec: dict, index, labels: str, name: str, now: datetime) -> tuple[str, float] | None: + """(tone, seconds past) when the row's crit_past or warn_past deadline has passed.""" + for level in ("crit", "warn"): + metric = spec[f"{level}_past"] + if not metric: + continue + match = _same_row(index, metric, labels, spec["name"], name) + if len(match) == 1 and match[0].value > 0: + past = now.timestamp() - match[0].value + if past > 0: + return level, past + return None + + def _states(block: Block, index, now) -> BlockView: spec = block.spec view = BlockView("states", block.title, columns=[c["label"] for c in spec["columns"]]) @@ -130,13 +153,14 @@ def _states(block: Block, index, now) -> BlockView: else: tones.append("crit") tone = (min(tones, key=TONE_ORDER.__getitem__) if tones else "warn") - row = Row(name=name or f["labels"] or block.title, tone=tone, text=f["state"] or "—", - why="" if tone in ("ok", "dim") else (f"is {f['state']}" if current else "reports no state"), + why = "" if tone in ("ok", "dim") else (f"is {f['state']}" if current else "reports no state") + overdue = _overdue(spec, index, f["labels"], name, now) if tone == "ok" else None + if overdue: + tone, why = overdue[0], f"is {f['state']}, {human_duration(overdue[1])} overdue" + row = Row(name=name or f["labels"] or block.title, tone=tone, text=f["state"] or "—", why=why, resource=f["resource"], metric=spec["metric"]) for col in spec["columns"]: - match = [s for s in index.get(col["metric"], []) if s.labels == f["labels"]] - if not match and name: - match = [s for s in index.get(col["metric"], []) if s.pairs.get(spec["name"]) == name] + match = _same_row(index, col["metric"], f["labels"], spec["name"], name) row.cells.append(shown(col["metric"], col["unit"], match[0].value if len(match) == 1 else None)) view.rows.append(row) view.rows = _ordered(view.rows) diff --git a/plugins/prometheus/widgets/fabled_archives.yaml b/plugins/prometheus/widgets/fabled_archives.yaml index e38fb12..6c6df9f 100644 --- a/plugins/prometheus/widgets/fabled_archives.yaml +++ b/plugins/prometheus/widgets/fabled_archives.yaml @@ -1,8 +1,16 @@ # Fabled Archives, from its /metrics on the LAN-only backup listener (:8081). -# Written against what Archives reports; see docs/plugins/prometheus.md. +# Written against the series Steward stores for Archives (#5693); see +# docs/plugins/prometheus.md. +# +# Left out on purpose: +# - fabledarchives_section_up: a section Archives can't read stops reporting +# its series, and the rows here that read them then say "not reported". +# - the background queue (fabledarchives_jobs, *_job_attempts_*), HTTP +# requests and the audit counters: they belong on charts and alert rules, +# not on a glance. name: Fabled Archives description: Backup targets, replication jobs, storage, integrity and the version running. -version: 1 +version: 2 expects: fabledarchives_ blocks: - states: @@ -19,9 +27,14 @@ blocks: title: Replication metric: fabledarchives_replication_job_state name: job + # Archives ranks "running" above lateness: a run that never ends still + # reads "running". Its own deadlines say when it is overdue. healthy: [ok, running] warning: [late, warning, never] + warn_past: fabledarchives_replication_job_late_at_timestamp_seconds + crit_past: fabledarchives_replication_job_missing_at_timestamp_seconds columns: + - {label: Last good copy, metric: fabledarchives_replication_job_last_success_timestamp_seconds} - {label: Next run, metric: fabledarchives_replication_job_next_run_timestamp_seconds} - values: title: Storage and integrity diff --git a/tests/plugins/prometheus/archives_live.py b/tests/plugins/prometheus/archives_live.py new file mode 100644 index 0000000..b4e2df2 --- /dev/null +++ b/tests/plugins/prometheus/archives_live.py @@ -0,0 +1,62 @@ +"""Fabled Archives' series as Steward stored them for the Archives target, +read from the live instance at 2026-10-14 13:40:14 UTC (#5693): every series +the bundled Fabled Archives template reads, and a few it leaves out on +purpose (section_up, the target and job info series). Values are as stored: +labels after the target's name, timestamps in seconds. + +At that moment the nightly Restic replication job had reported "running" +since Steward's first scrape two days before, with no good copy yet, and was +past both its late and missing times; Archives ranks "running" above +lateness, so its state alone says nothing is wrong. +""" +from datetime import datetime, timezone +from types import SimpleNamespace as NS + +TARGET = "Archives" +AT = datetime.fromtimestamp(1791985214.02, timezone.utc) # fabledarchives_read_timestamp_seconds + +_STEWARD = "target=Steward,target_id=44571987-c446-4f95-8ee5-306995e5e568" +_JOB = "job=FullNightly-Restic,job_id=1e8ee176-22dc-4518-8784-8aa5cbfabb40" +_ME = "person=Bryan Van Deusen,person_id=d6f3deff-f4df-4c09-a618-7f6124531298" + +SERIES: list[tuple[str, str, float]] = [ + ("", "fabledarchives_data_disk_free_bytes", 17800573747200.0), + ("", "fabledarchives_data_disk_size_bytes", 46059916492800.0), + ("", "fabledarchives_files", 100462.0), + ("", "fabledarchives_files_checked", 100462.0), + ("", "fabledarchives_malware_held_files", 2.0), + ("", "fabledarchives_notification_channels", 0.0), + ("", "fabledarchives_notification_channels_failing", 0.0), + ("", "fabledarchives_read_timestamp_seconds", 1791985214.02), + ("", "fabledarchives_video_copies_bytes", 0.0), + *((f"{_STEWARD},state={s}", "fabledarchives_backup_target_state", float(s == "fresh")) + for s in ("disabled", "fresh", "late", "missing", "waiting")), + (_STEWARD, "fabledarchives_backup_target_expected_interval_seconds", 86400.0), + (_STEWARD, "fabledarchives_backup_target_last_backup_timestamp_seconds", 1791947022.37), + (_STEWARD, "fabledarchives_backup_target_late_at_timestamp_seconds", 1792055022.37), + (_STEWARD, "fabledarchives_backup_target_missing_at_timestamp_seconds", 1792119822.37), + (_STEWARD, "fabledarchives_backup_target_stored_bytes", 389120.0), + (f"{_STEWARD},slug=steward,owner=Bryan Van Deusen", "fabledarchives_backup_target_info", 1.0), + *((f"{_JOB},state={s}", "fabledarchives_replication_job_state", float(s == "running")) + for s in ("check_failed", "failing", "late", "missing", "never", "ok", "running", "warning")), + (_JOB, "fabledarchives_replication_job_enabled", 1.0), + (_JOB, "fabledarchives_replication_job_expected_interval_seconds", 86400.0), + (_JOB, "fabledarchives_replication_job_late_at_timestamp_seconds", 1791942396.87), + (_JOB, "fabledarchives_replication_job_missing_at_timestamp_seconds", 1791942396.87), + (_JOB, "fabledarchives_replication_job_next_run_timestamp_seconds", 1792047600.0), + (f"{_JOB},method=restic,source=everyone,destination=Hetzner-storage-box-1 Restic," + "schedule=CRON_TZ=America/New_York 0 3 * * *", "fabledarchives_replication_job_info", 1.0), + (f"{_ME},kind=files", "fabledarchives_person_stored_bytes", 473016059695.0), + (f"{_ME},kind=trash", "fabledarchives_person_stored_bytes", 0.0), + (f"{_ME},kind=versions", "fabledarchives_person_stored_bytes", 0.0), + (_ME, "fabledarchives_person_counted_bytes", 473016059695.0), + *((f"section={s}", "fabledarchives_section_up", 1.0) + for s in ("disk", "notifications", "people", "queue", "replication", "sign_ins", "storage", "targets")), + ("version=2026.10.12.1717,channel=dev,commit=3605d0a", "fabledarchives_build_info", 1.0), +] + + +def latest(series=SERIES) -> list: + """PluginMetricLatest-like rows for the Archives target.""" + return [NS(resource_name=f"{TARGET}:{labels}" if labels else TARGET, metric_name=metric, value=value, + recorded_at=AT) for labels, metric, value in series] diff --git a/tests/plugins/prometheus/test_archives_template.py b/tests/plugins/prometheus/test_archives_template.py new file mode 100644 index 0000000..0539d02 --- /dev/null +++ b/tests/plugins/prometheus/test_archives_template.py @@ -0,0 +1,47 @@ +"""The bundled Fabled Archives template (#5693) over the series Archives +actually reported (archives_live.py): what the widget said about it.""" +from plugins.prometheus import library +from plugins.prometheus.template_view import render +from tests.plugins.prometheus.archives_live import AT, SERIES, TARGET, latest + +ARCHIVES = library.bundled()[0].template + + +def _rows(view, title): + return {r.name: r for b in view.blocks if b.title == title for r in b.rows} + + +def test_a_replication_that_never_finishes_reads_as_overdue_not_running(): + view = render(ARCHIVES, TARGET, latest(), AT) + assert view.tone == "crit" and view.note == "" + assert view.verdict == "2 need a look. Replication: FullNightly-Restic is running, 11h 53m overdue." + job = _rows(view, "Replication")["FullNightly-Restic"] + assert (job.tone, job.text) == ("crit", "running") + last_good, next_run = job.cells + assert last_good["text"] == "—" and next_run["moment"] is not None # no good copy yet + assert _rows(view, "Storage and integrity")["Malware held"].why == "above 0" + + +def test_everything_else_archives_reported_reads_as_well(): + view = render(ARCHIVES, TARGET, latest(), AT) + assert all(not b.note for b in view.blocks), [b.note for b in view.blocks] + steward = _rows(view, "Backup targets")["Steward"] + assert (steward.tone, steward.text) == ("ok", "fresh") and steward.cells[0]["moment"] is not None + storage = _rows(view, "Storage and integrity") + assert storage["Disk free"].tone == "ok" and "of" in storage["Disk free"].text + assert (storage["Files checked"].text, storage["Files checked"].tone) == ("100,462 of 100,462", "ok") + assert storage["Notification channels failing"].tone == "ok" + assert list(_rows(view, "Stored per person")) == ["Bryan Van Deusen"] + assert list(_rows(view, "Running")) == ["version 2026.10.12.1717 · channel dev · commit 3605d0a"] + + +def test_a_finished_copy_clears_it(): + """The next night's run succeeds: Archives reports ok with a last good + copy and deadlines a day out, and the widget says so.""" + later = AT.timestamp() + 86400 + done = [(labels, metric, (float(labels.endswith("state=ok")) if metric.endswith("job_state") else value)) + for labels, metric, value in SERIES] + done = [(lb, m, later + 30 * 3600 if m.endswith(("late_at_timestamp_seconds", "missing_at_timestamp_seconds")) + and lb.startswith("job=") else v) for lb, m, v in done] + job = _rows(render(ARCHIVES, TARGET, latest(done), AT), "Replication")["FullNightly-Restic"] + assert (job.tone, job.text, job.why) == ("ok", "ok", "") diff --git a/tests/plugins/prometheus/test_templates.py b/tests/plugins/prometheus/test_templates.py index dc891fd..e4bceb2 100644 --- a/tests/plugins/prometheus/test_templates.py +++ b/tests/plugins/prometheus/test_templates.py @@ -120,7 +120,7 @@ blocks: "Line 2: `version` is the template's own version: a whole number from 1. " "Raise it when you change a template you have shared.", "Line 8: Block 1 (states) has no `colour`. It takes: title, metric, where, name, healthy, " - "quiet, warning, columns.", + "quiet, warning, warn_past, crit_past, columns.", "Line 9: Block 2 is “gauge”; the kinds are: states, values, times, table, info.", "Line 12: `metric` in an item of block 3 (values) should be a metric name as Steward stores it, " "like fabledarchives_files (a counter is stored as _per_sec).", @@ -236,3 +236,36 @@ def test_all_well(): ("", "fa_malware_held_files"): 0.0} view = _view(healthy) assert (view.tone, view.verdict) == ("ok", "All well.") + + +def test_a_healthy_state_past_its_deadline_is_overdue(): + """warn_past / crit_past (#5693): a row whose state is healthy but whose + deadline has passed reads as a warning or a problem; a quiet or already + failing row keeps its own reading.""" + text = """\ +name: Jobs +version: 1 +blocks: + - states: + title: Jobs + metric: j_state + name: job + healthy: [ok, running] + quiet: [paused] + warn_past: j_late_at_timestamp_seconds + crit_past: j_missing_at_timestamp_seconds +""" + series = {} + for job, state, late, missing in (("A", "running", before(hours=2), before(hours=1) + 7200), + ("B", "running", before(hours=5), before(hours=3)), + ("C", "ok", before(hours=-1), before(hours=-2)), + ("D", "paused", before(hours=9), before(hours=8))): + for s in ("ok", "running", "paused"): + series[(f"job={job},state={s}", "j_state")] = float(s == state) + series[(f"job={job}", "j_late_at_timestamp_seconds")] = late + series[(f"job={job}", "j_missing_at_timestamp_seconds")] = missing + rows = {r.name: r for r in render(check(text), T, latest(series), NOW).blocks[0].rows} + assert (rows["A"].tone, rows["A"].why) == ("warn", "is running, 2h overdue") + assert (rows["B"].tone, rows["B"].why) == ("crit", "is running, 3h overdue") + assert (rows["C"].tone, rows["C"].why) == ("ok", "") + assert rows["D"].tone == "dim"