bvandeusenandClaude Opus 5.5
c8dbe811f8
#5467 "Steward's own health: metrics, rules can watch them, a page that shows them". The self-monitor tick now records, under source "steward", resource "process": ingest_posts / ingest_failed / ingest_slowest_ms (the host-agent posts since the last tick, tallied in steward/core/health.py), db_roundtrip_ms (a timed SELECT 1), and unwatched_secs (the gap before this tick past watch.GAP_S, from Steward's own last sample). The hourly cleanup records cleanup_secs. The open-file pair is still recorded, and is now only skipped on its own where /proc is absent. All of these are in CORE_METRICS, so an alert rule can watch them under "Steward itself". Settings → System (/settings/system/, admin) shows a verdict first (failed posts, a gap in watching, open files near the limit), then each measure now and over the last day, with a sparkline scaled to its own peak. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
61 lines
2.3 KiB
Python
61 lines
2.3 KiB
Python
"""Settings → System (#5467): Steward's own health, a verdict then each measure
|
||
now and over the day. The verdict names what is wrong, and only that."""
|
||
from datetime import datetime, timezone
|
||
|
||
import pytest
|
||
from quart import render_template, session
|
||
|
||
from steward.core import health
|
||
|
||
AT = datetime(2026, 10, 9, 12, 0, tzinfo=timezone.utc)
|
||
|
||
|
||
def _previews(**over):
|
||
base = {m: {"now": 0.0, "at": AT, "series": [0.0, 0.0], "low": 0.0, "high": 0.0}
|
||
for m in health.PROCESS_METRICS}
|
||
base["db_roundtrip_ms"] = {"now": 4.0, "at": AT, "series": [3.0, 5.0], "low": 2.0, "high": 9.0}
|
||
for m, v in over.items():
|
||
base[m] = {**base[m], **v}
|
||
return base
|
||
|
||
|
||
@pytest.fixture
|
||
def previews(monkeypatch):
|
||
def use(**over):
|
||
got = _previews(**over)
|
||
|
||
async def fake(db, source, resource, metric, now=None):
|
||
assert (source, resource) == ("steward", "process")
|
||
return got[metric]
|
||
monkeypatch.setattr("steward.alerts.service.metric_preview", fake)
|
||
return use
|
||
|
||
|
||
async def test_a_healthy_steward_has_no_problems(previews):
|
||
previews()
|
||
got = await health.process_health(None)
|
||
assert got["seen"] and got["problems"] == []
|
||
assert [r["metric"] for r in got["rows"]] == health.PROCESS_METRICS
|
||
|
||
|
||
async def test_failures_and_gaps_are_named(previews):
|
||
previews(ingest_failed={"now": 2.0, "high": 2.0}, unwatched_secs={"high": 420.0},
|
||
open_fds_pct={"now": 85.0})
|
||
problems = (await health.process_health(None))["problems"]
|
||
assert problems[0] == "2 agent posts failed in the last minute"
|
||
assert "wasn't watching for up to 7m" in problems[1]
|
||
assert problems[2] == "open files at 85% of the limit"
|
||
|
||
|
||
async def test_the_page_reads_as_a_verdict_then_each_measure(app, previews):
|
||
previews(unwatched_secs={"high": 420.0})
|
||
got = await health.process_health(None)
|
||
for r in got["rows"]:
|
||
r["spark"] = ""
|
||
async with app.test_request_context("/settings/system/"):
|
||
session["user_role"] = "admin"
|
||
page = " ".join((await render_template("settings/system.html", **got)).split())
|
||
assert '<span class="tone-warn">Steward wasn\'t watching for up to 7m' in page
|
||
assert "Database round trip" in page and "4 ms" in page and "2 ms – 9 ms" in page
|
||
assert page.index("page-verdict") < page.index("Agent posts per minute")
|