feat(health): Steward records its own health and Settings → System shows it (#5467)

#5467 "Steward's own health: metrics, rules can watch them, a page that shows them".

The self-monitor tick now records, under source "steward", resource "process":
ingest_posts / ingest_failed / ingest_slowest_ms (the host-agent posts since the
last tick, tallied in steward/core/health.py), db_roundtrip_ms (a timed SELECT 1),
and unwatched_secs (the gap before this tick past watch.GAP_S, from Steward's own
last sample). The hourly cleanup records cleanup_secs. The open-file pair is still
recorded, and is now only skipped on its own where /proc is absent. All of these
are in CORE_METRICS, so an alert rule can watch them under "Steward itself".

Settings → System (/settings/system/, admin) shows a verdict first (failed posts,
a gap in watching, open files near the limit), then each measure now and over the
last day, with a sparkline scaled to its own peak.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

steward/core/formatting.py

@@ -276,6 +276,12 @@ METRICS: dict[str, tuple[str, str]] = {
     "is_up": ("Up or down", "whether the check passed"),
     "is_healthy": ("Health check", "the container's own HEALTHCHECK result"),
     "response_ms": ("Response time", "how long the check took"),
+    "ingest_posts": ("Agent posts per minute", "host-agent reports Steward took in the last minute"),
+    "ingest_failed": ("Failed agent posts", "reports in the last minute that didn't land"),
+    "ingest_slowest_ms": ("Slowest agent post", "the longest report in the last minute; agents give up at 10 s"),
+    "db_roundtrip_ms": ("Database round trip", "a trivial query to Steward's database, connection included"),
+    "unwatched_secs": ("Time not watching", "how long Steward wasn't recording before this minute; 0 when unbroken"),
+    "cleanup_secs": ("Hourly cleanup took", "how long the last retention and rollup run took"),
     "open_fds": ("Open files", "files Steward itself has open"),
     "open_fds_pct": ("Open files (% of limit)", "files Steward has open, against its limit"),
     "request_rate": ("Requests per second", "requests through this router"),

steward/core/health.py

@@ -0,0 +1,69 @@
+"""Steward's own health between self-monitor ticks (#5467).
+
+The host-agent ingest reports each post here as it finishes; the self-monitor
+tick (core/self_monitor.py) takes the minute's totals and records them as
+metrics under source "steward", resource "process", beside the open-file
+counts, where alert rules can watch them like any other metric. One process
+serves Steward, so a module-level counter is the whole of it.
+"""
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+# Steward's own metrics, in the order the Settings → System page lists them.
+PROCESS_METRICS = ["ingest_posts", "ingest_failed", "ingest_slowest_ms", "db_roundtrip_ms",
+                   "unwatched_secs", "cleanup_secs", "open_fds", "open_fds_pct"]
+
+
+@dataclass
+class IngestTally:
+    posts: int = 0
+    failed: int = 0
+    slowest_ms: float = 0.0
+
+    def done(self, ms: float, ok: bool = True) -> None:
+        """One ingest post finished in `ms`; `ok` False when it didn't land
+        (the agent gave up, or the handler raised)."""
+        self.posts += 1
+        if not ok:
+            self.failed += 1
+        self.slowest_ms = max(self.slowest_ms, ms)
+
+    def take(self) -> "IngestTally":
+        """The totals since the last take, starting the next minute afresh."""
+        got = IngestTally(self.posts, self.failed, self.slowest_ms)
+        self.posts, self.failed, self.slowest_ms = 0, 0, 0.0
+        return got
+
+
+ingest = IngestTally()
+
+
+async def process_health(db) -> dict:
+    """What Settings → System shows (#5467): each of Steward's own metrics now
+    and over the last day, and the problems among them, worst first.
+
+    Returns {"rows": [{metric, now, at, low, high, series}], "problems": [str],
+    "seen": bool}. Reads plugin_metric_latest and a day of each series (the
+    covering series index makes that cheap, #5490)."""
+    from steward.alerts.service import metric_preview
+    from steward.core.formatting import human_duration
+    rows = []
+    for metric in PROCESS_METRICS:
+        p = await metric_preview(db, "steward", "process", metric)
+        rows.append({"metric": metric, **(p or {"now": None, "at": None, "series": [],
+                                               "low": None, "high": None})})
+    by = {r["metric"]: r for r in rows}
+    problems = []
+    if (by["ingest_failed"]["now"] or 0) > 0:
+        problems.append(f"{by['ingest_failed']['now']:.0f} agent posts failed in the last minute")
+    elif (by["ingest_failed"]["high"] or 0) > 0:
+        problems.append(f"agent posts failed in the last day, up to {by['ingest_failed']['high']:.0f} "
+                        "in a minute")
+    if (by["unwatched_secs"]["high"] or 0) > 0:
+        problems.append(f"Steward wasn't watching for up to {human_duration(by['unwatched_secs']['high'])} "
+                        "in the last day (a deploy or an outage)")
+    from steward.core.self_monitor import FD_WARN_PCT
+    if (by["open_fds_pct"]["now"] or 0) >= FD_WARN_PCT:
+        problems.append(f"open files at {by['open_fds_pct']['now']:.0f}% of the limit")
+    return {"rows": rows, "problems": problems, "seen": any(r["at"] for r in rows)}

steward/core/self_monitor.py

@@ -1,4 +1,4 @@
-"""Self-monitoring: record Steward's own open file-descriptor usage.
+"""Self-monitoring: record Steward's own health each minute.
 
 A leaked socket/file handle in a poll loop (see the SNMP and UniFi fd-leak
 issues) used to fail silently until the process hit its fd ceiling and
@@ -16,10 +16,25 @@ Both flow through the normal alert pipeline, so the operator can attach an alert
 rule to either via the existing alert-rules UI. As a zero-config floor we also
 log a WARNING once usage crosses ``FD_WARN_PCT`` — a leak becomes visible even
 before any rule is set up.
+
+The same tick records the rest of Steward's health (#5467), under the same
+source and resource:
+
+  • ``ingest_posts`` / ``ingest_failed`` / ``ingest_slowest_ms`` — the host-agent
+    posts since the last tick (core/health.py), how many didn't land, the slowest
+  • ``db_roundtrip_ms`` — a timed ``SELECT 1``, connection checkout included
+  • ``unwatched_secs`` — how long Steward wasn't recording before this tick (a
+    deploy, an outage, a stalled loop), from its own last sample; 0 when unbroken
+
+``cleanup_secs`` is recorded by the hourly cleanup itself when it finishes.
 """
 from __future__ import annotations
 import logging
 import os
+import time
+from datetime import datetime, timezone
+
+from sqlalchemy import func
 
 logger = logging.getLogger(__name__)
 
@@ -67,40 +82,57 @@ def fd_soft_limit() -> int | None:
     return soft
 
 
-async def record_self_metrics(app) -> None:
-    """Record open-fd usage as Steward's own metrics; warn past the floor.
+def unwatched_secs(last: datetime | None, now: datetime) -> float:
+    """How long Steward wasn't recording before `now`, given its last sample:
+    0 when the ticks are unbroken (a gap no longer than core.watch.GAP_S)."""
+    from .watch import GAP_S
+    if last is None:
+        return 0.0
+    gap = (now - last).total_seconds()
+    return gap if gap > GAP_S else 0.0
 
-    No-op (logged at debug) when fd accounting isn't available on this platform,
-    so it's safe to schedule unconditionally.
+
+async def record_self_metrics(app) -> None:
+    """Record Steward's own health as metrics; warn past the open-file floor.
+
+    The open-file pair is skipped (logged at debug) where fd accounting isn't
+    available, so it's safe to schedule unconditionally.
     """
     global _warned
 
+    from sqlalchemy import select, text
+    from steward.models.metrics import PluginMetricLatest
+    from .alerts import MetricBatch
+    from .health import ingest
+
     fds = count_open_fds()
     if fds is None:
         logger.debug("self_monitor: /proc/self/fd unavailable — skipping fd metrics")
-        return
-
-    from .alerts import MetricBatch
-
     soft = fd_soft_limit()
-    pct = (fds / soft * 100.0) if soft else None
+    pct = (fds / soft * 100.0) if (fds is not None and soft) else None
+    tally = ingest.take()
 
+    now = datetime.now(timezone.utc)
+    started = time.monotonic()
     async with app.db_sessionmaker() as session:
         async with session.begin():
+            await session.execute(text("SELECT 1"))
+            roundtrip_ms = (time.monotonic() - started) * 1000.0
+            # The last tick's sample, whatever restarts came between.
+            last = (await session.execute(select(func.max(PluginMetricLatest.recorded_at)).where(
+                PluginMetricLatest.source_module == "steward",
+                PluginMetricLatest.resource_name == "process",
+                PluginMetricLatest.metric_name == "db_roundtrip_ms"))).scalar()
             batch = MetricBatch()
-            batch.add(
-                source_module="steward",
-                resource_name="process",
-                metric_name="open_fds",
-                value=float(fds),
-            )
-            if pct is not None:
-                batch.add(
-                    source_module="steward",
-                    resource_name="process",
-                    metric_name="open_fds_pct",
-                    value=pct,
-                )
+            # Literal names, so tests/core/test_metric_catalog.py sees each one.
+            for metric_name, value in [
+                    ("ingest_posts", tally.posts), ("ingest_failed", tally.failed),
+                    ("ingest_slowest_ms", tally.slowest_ms), ("db_roundtrip_ms", roundtrip_ms),
+                    ("unwatched_secs", unwatched_secs(last, now)),
+                    ("open_fds", fds), ("open_fds_pct", pct)]:
+                if value is not None:
+                    batch.add(source_module="steward", resource_name="process",
+                              metric_name=metric_name, value=value, recorded_at=now)
             await batch.record(session)
 
     if pct is not None and pct >= FD_WARN_PCT:
@@ -113,4 +145,16 @@ async def record_self_metrics(app) -> None:
     else:
         _warned = False  # recovered — re-arm the warning for the next breach
 
-    logger.debug("self_monitor: open_fds=%d soft_limit=%s pct=%s", fds, soft, pct)
+    logger.debug("self_monitor: open_fds=%s soft_limit=%s pct=%s ingest=%s db=%.0fms",
+                 fds, soft, pct, tally, roundtrip_ms)
+
+
+async def record_cleanup_secs(app, secs: float) -> None:
+    """How long the hourly cleanup just took (#5467), as Steward's own metric."""
+    from .alerts import MetricBatch
+    async with app.db_sessionmaker() as session:
+        async with session.begin():
+            batch = MetricBatch()
+            batch.add(source_module="steward", resource_name="process",
+                      metric_name="cleanup_secs", value=secs)
+            await batch.record(session)