bvandeusenandClaude Opus 5.5
41d9d00216
Read from what Steward stored for Archives on 2026-10-14: the nightly Restic replication job had said "running" since the first scrape two days earlier, with no good copy yet and past both its late and missing times. Archives ranks "running" above lateness, so the widget called it healthy. - states blocks take warn_past / crit_past: a deadline metric per row (a *_timestamp_seconds). Once it has passed, a healthy row reads as a warning or a problem: "FullNightly-Restic is running, 11h 53m overdue". Quiet and already-failing rows keep their own reading. - The Archives template (version 2) uses the job's late_at / missing_at deadlines and shows its last good copy beside the next run. It says why section_up, the queue and the HTTP/audit counters are left out. - tests/plugins/prometheus/archives_live.py is the series as captured live, and test_archives_template.py renders the template over them. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
63 lines
3.8 KiB
Python
63 lines
3.8 KiB
Python
"""Fabled Archives' series as Steward stored them for the Archives target,
|
|
read from the live instance at 2026-10-14 13:40:14 UTC (#5693): every series
|
|
the bundled Fabled Archives template reads, and a few it leaves out on
|
|
purpose (section_up, the target and job info series). Values are as stored:
|
|
labels after the target's name, timestamps in seconds.
|
|
|
|
At that moment the nightly Restic replication job had reported "running"
|
|
since Steward's first scrape two days before, with no good copy yet, and was
|
|
past both its late and missing times; Archives ranks "running" above
|
|
lateness, so its state alone says nothing is wrong.
|
|
"""
|
|
from datetime import datetime, timezone
|
|
from types import SimpleNamespace as NS
|
|
|
|
TARGET = "Archives"
|
|
AT = datetime.fromtimestamp(1791985214.02, timezone.utc) # fabledarchives_read_timestamp_seconds
|
|
|
|
_STEWARD = "target=Steward,target_id=44571987-c446-4f95-8ee5-306995e5e568"
|
|
_JOB = "job=FullNightly-Restic,job_id=1e8ee176-22dc-4518-8784-8aa5cbfabb40"
|
|
_ME = "person=Bryan Van Deusen,person_id=d6f3deff-f4df-4c09-a618-7f6124531298"
|
|
|
|
SERIES: list[tuple[str, str, float]] = [
|
|
("", "fabledarchives_data_disk_free_bytes", 17800573747200.0),
|
|
("", "fabledarchives_data_disk_size_bytes", 46059916492800.0),
|
|
("", "fabledarchives_files", 100462.0),
|
|
("", "fabledarchives_files_checked", 100462.0),
|
|
("", "fabledarchives_malware_held_files", 2.0),
|
|
("", "fabledarchives_notification_channels", 0.0),
|
|
("", "fabledarchives_notification_channels_failing", 0.0),
|
|
("", "fabledarchives_read_timestamp_seconds", 1791985214.02),
|
|
("", "fabledarchives_video_copies_bytes", 0.0),
|
|
*((f"{_STEWARD},state={s}", "fabledarchives_backup_target_state", float(s == "fresh"))
|
|
for s in ("disabled", "fresh", "late", "missing", "waiting")),
|
|
(_STEWARD, "fabledarchives_backup_target_expected_interval_seconds", 86400.0),
|
|
(_STEWARD, "fabledarchives_backup_target_last_backup_timestamp_seconds", 1791947022.37),
|
|
(_STEWARD, "fabledarchives_backup_target_late_at_timestamp_seconds", 1792055022.37),
|
|
(_STEWARD, "fabledarchives_backup_target_missing_at_timestamp_seconds", 1792119822.37),
|
|
(_STEWARD, "fabledarchives_backup_target_stored_bytes", 389120.0),
|
|
(f"{_STEWARD},slug=steward,owner=Bryan Van Deusen", "fabledarchives_backup_target_info", 1.0),
|
|
*((f"{_JOB},state={s}", "fabledarchives_replication_job_state", float(s == "running"))
|
|
for s in ("check_failed", "failing", "late", "missing", "never", "ok", "running", "warning")),
|
|
(_JOB, "fabledarchives_replication_job_enabled", 1.0),
|
|
(_JOB, "fabledarchives_replication_job_expected_interval_seconds", 86400.0),
|
|
(_JOB, "fabledarchives_replication_job_late_at_timestamp_seconds", 1791942396.87),
|
|
(_JOB, "fabledarchives_replication_job_missing_at_timestamp_seconds", 1791942396.87),
|
|
(_JOB, "fabledarchives_replication_job_next_run_timestamp_seconds", 1792047600.0),
|
|
(f"{_JOB},method=restic,source=everyone,destination=Hetzner-storage-box-1 Restic,"
|
|
"schedule=CRON_TZ=America/New_York 0 3 * * *", "fabledarchives_replication_job_info", 1.0),
|
|
(f"{_ME},kind=files", "fabledarchives_person_stored_bytes", 473016059695.0),
|
|
(f"{_ME},kind=trash", "fabledarchives_person_stored_bytes", 0.0),
|
|
(f"{_ME},kind=versions", "fabledarchives_person_stored_bytes", 0.0),
|
|
(_ME, "fabledarchives_person_counted_bytes", 473016059695.0),
|
|
*((f"section={s}", "fabledarchives_section_up", 1.0)
|
|
for s in ("disk", "notifications", "people", "queue", "replication", "sign_ins", "storage", "targets")),
|
|
("version=2026.10.12.1717,channel=dev,commit=3605d0a", "fabledarchives_build_info", 1.0),
|
|
]
|
|
|
|
|
|
def latest(series=SERIES) -> list:
|
|
"""PluginMetricLatest-like rows for the Archives target."""
|
|
return [NS(resource_name=f"{TARGET}:{labels}" if labels else TARGET, metric_name=metric, value=value,
|
|
recorded_at=AT) for labels, metric, value in series]
|