feat(health): Steward records its own health and Settings → System shows it (#5467)

#5467 "Steward's own health: metrics, rules can watch them, a page that shows them".

The self-monitor tick now records, under source "steward", resource "process":
ingest_posts / ingest_failed / ingest_slowest_ms (the host-agent posts since the
last tick, tallied in steward/core/health.py), db_roundtrip_ms (a timed SELECT 1),
and unwatched_secs (the gap before this tick past watch.GAP_S, from Steward's own
last sample). The hourly cleanup records cleanup_secs. The open-file pair is still
recorded, and is now only skipped on its own where /proc is absent. All of these
are in CORE_METRICS, so an alert rule can watch them under "Steward itself".

Settings → System (/settings/system/, admin) shows a verdict first (failed posts,
a gap in watching, open files near the limit), then each measure now and over the
last day, with a sparkline scaled to its own peak.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

tests/core/test_self_monitor.py

@@ -1,4 +1,4 @@
-"""Unit tests for the self-fd watchdog (no DB, no network).
+"""Unit tests for the self-monitor: open files and Steward's own health (no DB, no network).
 
 Exercises the fd accounting, the percentage/warning floor, and that the
 recorder degrades to a no-op where fd accounting isn't available. The DB session
@@ -36,9 +36,15 @@ class _Ctx:
 
 
 class _FakeSession(_Ctx):
+    last = None   # the latest steward sample the unwatched gap is counted from
+
     def begin(self):
         return _Ctx()
 
+    async def execute(self, stmt):
+        last = self.last
+        return type("R", (), {"scalar": staticmethod(lambda: last)})()
+
 
 class _FakeApp:
     def db_sessionmaker(self):
@@ -104,12 +110,37 @@ def test_no_warning_below_floor(monkeypatch, caplog):
     assert not any("soft limit" in r.message for r in caplog.records)
 
 
-def test_noop_when_fd_count_unavailable(monkeypatch):
-    # /proc absent → must return before touching the DB.
+def test_without_fd_accounting_the_rest_is_still_recorded(monkeypatch):
+    # /proc absent: no open-file pair, but Steward's other health still lands.
+    recorded = _patch_recorder(monkeypatch)
     monkeypatch.setattr(sm, "count_open_fds", lambda: None)
 
-    class _Boom:
-        def db_sessionmaker(self):
-            raise AssertionError("must not open a session when fds are unknown")
+    asyncio.run(sm.record_self_metrics(_FakeApp()))
 
-    asyncio.run(sm.record_self_metrics(_Boom()))   # no raise
+    names = {m for (_s, _r, m, _v) in recorded}
+    assert "db_roundtrip_ms" in names and not names & {"open_fds", "open_fds_pct"}
+
+
+def test_the_minute_s_ingests_are_recorded_and_the_tally_starts_again(monkeypatch):
+    """#5467: posts, failures and the slowest post since the last tick."""
+    from steward.core import health
+    recorded = _patch_recorder(monkeypatch)
+    monkeypatch.setattr(sm, "count_open_fds", lambda: 50)
+    monkeypatch.setattr(health, "ingest", health.IngestTally())
+    for ms, ok in ((300.0, True), (9800.0, False), (450.0, True)):
+        health.ingest.done(ms, ok)
+
+    asyncio.run(sm.record_self_metrics(_FakeApp()))
+
+    got = {m: v for (s, r, m, v) in recorded if (s, r) == ("steward", "process")}
+    assert (got["ingest_posts"], got["ingest_failed"], got["ingest_slowest_ms"]) == (3.0, 1.0, 9800.0)
+    assert got["unwatched_secs"] == 0.0 and got["db_roundtrip_ms"] >= 0
+    assert health.ingest.take() == health.IngestTally()   # taken: the next minute starts at zero
+
+
+def test_unwatched_counts_only_a_real_gap():
+    from datetime import datetime, timedelta, timezone
+    now = datetime(2026, 10, 9, 12, 0, tzinfo=timezone.utc)
+    assert sm.unwatched_secs(None, now) == 0.0                          # first ever tick
+    assert sm.unwatched_secs(now - timedelta(seconds=61), now) == 0.0   # an ordinary minute
+    assert sm.unwatched_secs(now - timedelta(minutes=7), now) == 420.0  # a deploy or outage

tests/core/test_steward_health.py

@@ -0,0 +1,60 @@
+"""Settings → System (#5467): Steward's own health, a verdict then each measure
+now and over the day. The verdict names what is wrong, and only that."""
+from datetime import datetime, timezone
+
+import pytest
+from quart import render_template, session
+
+from steward.core import health
+
+AT = datetime(2026, 10, 9, 12, 0, tzinfo=timezone.utc)
+
+
+def _previews(**over):
+    base = {m: {"now": 0.0, "at": AT, "series": [0.0, 0.0], "low": 0.0, "high": 0.0}
+            for m in health.PROCESS_METRICS}
+    base["db_roundtrip_ms"] = {"now": 4.0, "at": AT, "series": [3.0, 5.0], "low": 2.0, "high": 9.0}
+    for m, v in over.items():
+        base[m] = {**base[m], **v}
+    return base
+
+
+@pytest.fixture
+def previews(monkeypatch):
+    def use(**over):
+        got = _previews(**over)
+
+        async def fake(db, source, resource, metric, now=None):
+            assert (source, resource) == ("steward", "process")
+            return got[metric]
+        monkeypatch.setattr("steward.alerts.service.metric_preview", fake)
+    return use
+
+
+async def test_a_healthy_steward_has_no_problems(previews):
+    previews()
+    got = await health.process_health(None)
+    assert got["seen"] and got["problems"] == []
+    assert [r["metric"] for r in got["rows"]] == health.PROCESS_METRICS
+
+
+async def test_failures_and_gaps_are_named(previews):
+    previews(ingest_failed={"now": 2.0, "high": 2.0}, unwatched_secs={"high": 420.0},
+             open_fds_pct={"now": 85.0})
+    problems = (await health.process_health(None))["problems"]
+    assert problems[0] == "2 agent posts failed in the last minute"
+    assert "wasn't watching for up to 7m" in problems[1]
+    assert problems[2] == "open files at 85% of the limit"
+
+
+async def test_the_page_reads_as_a_verdict_then_each_measure(app, previews):
+    previews(unwatched_secs={"high": 420.0})
+    got = await health.process_health(None)
+    for r in got["rows"]:
+        r["spark"] = ""
+    async with app.test_request_context("/settings/system/"):
+        session["user_role"] = "admin"
+        page = " ".join((await render_template("settings/system.html", **got)).split())
+    assert '<span class="tone-warn">Steward wasn\'t watching for up to 7m' in page
+    assert "Database round trip" in page and "4 ms" in page and "2 ms – 9 ms" in page
+    assert page.index("page-verdict") < page.index("Agent posts per minute")