feat(health): Steward records its own health and Settings → System shows it (#5467)

#5467 "Steward's own health: metrics, rules can watch them, a page that shows them".

The self-monitor tick now records, under source "steward", resource "process":
ingest_posts / ingest_failed / ingest_slowest_ms (the host-agent posts since the
last tick, tallied in steward/core/health.py), db_roundtrip_ms (a timed SELECT 1),
and unwatched_secs (the gap before this tick past watch.GAP_S, from Steward's own
last sample). The hourly cleanup records cleanup_secs. The open-file pair is still
recorded, and is now only skipped on its own where /proc is absent. All of these
are in CORE_METRICS, so an alert rule can watch them under "Steward itself".

Settings → System (/settings/system/, admin) shows a verdict first (failed posts,
a gap in watching, open files near the limit), then each measure now and over the
last day, with a sparkline scaled to its own peak.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

steward/core/self_monitor.py

@@ -1,4 +1,4 @@
-"""Self-monitoring: record Steward's own open file-descriptor usage.
+"""Self-monitoring: record Steward's own health each minute.
 
 A leaked socket/file handle in a poll loop (see the SNMP and UniFi fd-leak
 issues) used to fail silently until the process hit its fd ceiling and
@@ -16,10 +16,25 @@ Both flow through the normal alert pipeline, so the operator can attach an alert
 rule to either via the existing alert-rules UI. As a zero-config floor we also
 log a WARNING once usage crosses ``FD_WARN_PCT`` — a leak becomes visible even
 before any rule is set up.
+
+The same tick records the rest of Steward's health (#5467), under the same
+source and resource:
+
+  • ``ingest_posts`` / ``ingest_failed`` / ``ingest_slowest_ms`` — the host-agent
+    posts since the last tick (core/health.py), how many didn't land, the slowest
+  • ``db_roundtrip_ms`` — a timed ``SELECT 1``, connection checkout included
+  • ``unwatched_secs`` — how long Steward wasn't recording before this tick (a
+    deploy, an outage, a stalled loop), from its own last sample; 0 when unbroken
+
+``cleanup_secs`` is recorded by the hourly cleanup itself when it finishes.
 """
 from __future__ import annotations
 import logging
 import os
+import time
+from datetime import datetime, timezone
+
+from sqlalchemy import func
 
 logger = logging.getLogger(__name__)
 
@@ -67,40 +82,57 @@ def fd_soft_limit() -> int | None:
     return soft
 
 
-async def record_self_metrics(app) -> None:
-    """Record open-fd usage as Steward's own metrics; warn past the floor.
+def unwatched_secs(last: datetime | None, now: datetime) -> float:
+    """How long Steward wasn't recording before `now`, given its last sample:
+    0 when the ticks are unbroken (a gap no longer than core.watch.GAP_S)."""
+    from .watch import GAP_S
+    if last is None:
+        return 0.0
+    gap = (now - last).total_seconds()
+    return gap if gap > GAP_S else 0.0
 
-    No-op (logged at debug) when fd accounting isn't available on this platform,
-    so it's safe to schedule unconditionally.
+
+async def record_self_metrics(app) -> None:
+    """Record Steward's own health as metrics; warn past the open-file floor.
+
+    The open-file pair is skipped (logged at debug) where fd accounting isn't
+    available, so it's safe to schedule unconditionally.
     """
     global _warned
 
+    from sqlalchemy import select, text
+    from steward.models.metrics import PluginMetricLatest
+    from .alerts import MetricBatch
+    from .health import ingest
+
     fds = count_open_fds()
     if fds is None:
         logger.debug("self_monitor: /proc/self/fd unavailable — skipping fd metrics")
-        return
-
-    from .alerts import MetricBatch
-
     soft = fd_soft_limit()
-    pct = (fds / soft * 100.0) if soft else None
+    pct = (fds / soft * 100.0) if (fds is not None and soft) else None
+    tally = ingest.take()
 
+    now = datetime.now(timezone.utc)
+    started = time.monotonic()
     async with app.db_sessionmaker() as session:
         async with session.begin():
+            await session.execute(text("SELECT 1"))
+            roundtrip_ms = (time.monotonic() - started) * 1000.0
+            # The last tick's sample, whatever restarts came between.
+            last = (await session.execute(select(func.max(PluginMetricLatest.recorded_at)).where(
+                PluginMetricLatest.source_module == "steward",
+                PluginMetricLatest.resource_name == "process",
+                PluginMetricLatest.metric_name == "db_roundtrip_ms"))).scalar()
             batch = MetricBatch()
-            batch.add(
-                source_module="steward",
-                resource_name="process",
-                metric_name="open_fds",
-                value=float(fds),
-            )
-            if pct is not None:
-                batch.add(
-                    source_module="steward",
-                    resource_name="process",
-                    metric_name="open_fds_pct",
-                    value=pct,
-                )
+            # Literal names, so tests/core/test_metric_catalog.py sees each one.
+            for metric_name, value in [
+                    ("ingest_posts", tally.posts), ("ingest_failed", tally.failed),
+                    ("ingest_slowest_ms", tally.slowest_ms), ("db_roundtrip_ms", roundtrip_ms),
+                    ("unwatched_secs", unwatched_secs(last, now)),
+                    ("open_fds", fds), ("open_fds_pct", pct)]:
+                if value is not None:
+                    batch.add(source_module="steward", resource_name="process",
+                              metric_name=metric_name, value=value, recorded_at=now)
             await batch.record(session)
 
     if pct is not None and pct >= FD_WARN_PCT:
@@ -113,4 +145,16 @@ async def record_self_metrics(app) -> None:
     else:
         _warned = False  # recovered — re-arm the warning for the next breach
 
-    logger.debug("self_monitor: open_fds=%d soft_limit=%s pct=%s", fds, soft, pct)
+    logger.debug("self_monitor: open_fds=%s soft_limit=%s pct=%s ingest=%s db=%.0fms",
+                 fds, soft, pct, tally, roundtrip_ms)
+
+
+async def record_cleanup_secs(app, secs: float) -> None:
+    """How long the hourly cleanup just took (#5467), as Steward's own metric."""
+    from .alerts import MetricBatch
+    async with app.db_sessionmaker() as session:
+        async with session.begin():
+            batch = MetricBatch()
+            batch.add(source_module="steward", resource_name="process",
+                      metric_name="cleanup_secs", value=secs)
+            await batch.record(session)