feat(health): Steward records its own health and Settings → System shows it (#5467)

#5467 "Steward's own health: metrics, rules can watch them, a page that shows them".

The self-monitor tick now records, under source "steward", resource "process":
ingest_posts / ingest_failed / ingest_slowest_ms (the host-agent posts since the
last tick, tallied in steward/core/health.py), db_roundtrip_ms (a timed SELECT 1),
and unwatched_secs (the gap before this tick past watch.GAP_S, from Steward's own
last sample). The hourly cleanup records cleanup_secs. The open-file pair is still
recorded, and is now only skipped on its own where /proc is absent. All of these
are in CORE_METRICS, so an alert rule can watch them under "Steward itself".

Settings → System (/settings/system/, admin) shows a verdict first (failed posts,
a gap in watching, open files near the limit), then each measure now and over the
last day, with a sparkline scaled to its own peak.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

steward/alerts/service.py

@@ -27,7 +27,8 @@ CORE_METRICS: dict[str, list[str]] = {
     "tcp":     ["is_up", "response_ms"],
     "dns":     ["is_up", "response_ms"],
     "http":    ["is_up", "response_ms"],
-    "steward": ["open_fds", "open_fds_pct"],
+    "steward": ["ingest_posts", "ingest_failed", "ingest_slowest_ms", "db_roundtrip_ms",
+                "unwatched_secs", "cleanup_secs", "open_fds", "open_fds_pct"],
 }
 
 

steward/app.py

@@ -414,8 +414,14 @@ def _register_core_tasks(app: Quart) -> None:
         await run_due_monitors(app)
 
     async def run_cleanup():
+        import time
         from .core.cleanup import run_cleanup as _cleanup
+        from .core.self_monitor import record_cleanup_secs
+        started = time.monotonic()
         await _cleanup(app)
+        # How long it took, as Steward's own metric (#5467); a failed run
+        # raises past this and records nothing.
+        await record_cleanup_secs(app, time.monotonic() - started)
 
     async def run_alert_reconcile():
         from .alerts.service import reconcile_covering_rules

steward/core/formatting.py

@@ -276,6 +276,12 @@ METRICS: dict[str, tuple[str, str]] = {
     "is_up": ("Up or down", "whether the check passed"),
     "is_healthy": ("Health check", "the container's own HEALTHCHECK result"),
     "response_ms": ("Response time", "how long the check took"),
+    "ingest_posts": ("Agent posts per minute", "host-agent reports Steward took in the last minute"),
+    "ingest_failed": ("Failed agent posts", "reports in the last minute that didn't land"),
+    "ingest_slowest_ms": ("Slowest agent post", "the longest report in the last minute; agents give up at 10 s"),
+    "db_roundtrip_ms": ("Database round trip", "a trivial query to Steward's database, connection included"),
+    "unwatched_secs": ("Time not watching", "how long Steward wasn't recording before this minute; 0 when unbroken"),
+    "cleanup_secs": ("Hourly cleanup took", "how long the last retention and rollup run took"),
     "open_fds": ("Open files", "files Steward itself has open"),
     "open_fds_pct": ("Open files (% of limit)", "files Steward has open, against its limit"),
     "request_rate": ("Requests per second", "requests through this router"),

steward/core/health.py

@@ -0,0 +1,69 @@
+"""Steward's own health between self-monitor ticks (#5467).
+
+The host-agent ingest reports each post here as it finishes; the self-monitor
+tick (core/self_monitor.py) takes the minute's totals and records them as
+metrics under source "steward", resource "process", beside the open-file
+counts, where alert rules can watch them like any other metric. One process
+serves Steward, so a module-level counter is the whole of it.
+"""
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+# Steward's own metrics, in the order the Settings → System page lists them.
+PROCESS_METRICS = ["ingest_posts", "ingest_failed", "ingest_slowest_ms", "db_roundtrip_ms",
+                   "unwatched_secs", "cleanup_secs", "open_fds", "open_fds_pct"]
+
+
+@dataclass
+class IngestTally:
+    posts: int = 0
+    failed: int = 0
+    slowest_ms: float = 0.0
+
+    def done(self, ms: float, ok: bool = True) -> None:
+        """One ingest post finished in `ms`; `ok` False when it didn't land
+        (the agent gave up, or the handler raised)."""
+        self.posts += 1
+        if not ok:
+            self.failed += 1
+        self.slowest_ms = max(self.slowest_ms, ms)
+
+    def take(self) -> "IngestTally":
+        """The totals since the last take, starting the next minute afresh."""
+        got = IngestTally(self.posts, self.failed, self.slowest_ms)
+        self.posts, self.failed, self.slowest_ms = 0, 0, 0.0
+        return got
+
+
+ingest = IngestTally()
+
+
+async def process_health(db) -> dict:
+    """What Settings → System shows (#5467): each of Steward's own metrics now
+    and over the last day, and the problems among them, worst first.
+
+    Returns {"rows": [{metric, now, at, low, high, series}], "problems": [str],
+    "seen": bool}. Reads plugin_metric_latest and a day of each series (the
+    covering series index makes that cheap, #5490)."""
+    from steward.alerts.service import metric_preview
+    from steward.core.formatting import human_duration
+    rows = []
+    for metric in PROCESS_METRICS:
+        p = await metric_preview(db, "steward", "process", metric)
+        rows.append({"metric": metric, **(p or {"now": None, "at": None, "series": [],
+                                               "low": None, "high": None})})
+    by = {r["metric"]: r for r in rows}
+    problems = []
+    if (by["ingest_failed"]["now"] or 0) > 0:
+        problems.append(f"{by['ingest_failed']['now']:.0f} agent posts failed in the last minute")
+    elif (by["ingest_failed"]["high"] or 0) > 0:
+        problems.append(f"agent posts failed in the last day, up to {by['ingest_failed']['high']:.0f} "
+                        "in a minute")
+    if (by["unwatched_secs"]["high"] or 0) > 0:
+        problems.append(f"Steward wasn't watching for up to {human_duration(by['unwatched_secs']['high'])} "
+                        "in the last day (a deploy or an outage)")
+    from steward.core.self_monitor import FD_WARN_PCT
+    if (by["open_fds_pct"]["now"] or 0) >= FD_WARN_PCT:
+        problems.append(f"open files at {by['open_fds_pct']['now']:.0f}% of the limit")
+    return {"rows": rows, "problems": problems, "seen": any(r["at"] for r in rows)}

steward/core/self_monitor.py

@@ -1,4 +1,4 @@
-"""Self-monitoring: record Steward's own open file-descriptor usage.
+"""Self-monitoring: record Steward's own health each minute.
 
 A leaked socket/file handle in a poll loop (see the SNMP and UniFi fd-leak
 issues) used to fail silently until the process hit its fd ceiling and
@@ -16,10 +16,25 @@ Both flow through the normal alert pipeline, so the operator can attach an alert
 rule to either via the existing alert-rules UI. As a zero-config floor we also
 log a WARNING once usage crosses ``FD_WARN_PCT`` — a leak becomes visible even
 before any rule is set up.
+
+The same tick records the rest of Steward's health (#5467), under the same
+source and resource:
+
+  • ``ingest_posts`` / ``ingest_failed`` / ``ingest_slowest_ms`` — the host-agent
+    posts since the last tick (core/health.py), how many didn't land, the slowest
+  • ``db_roundtrip_ms`` — a timed ``SELECT 1``, connection checkout included
+  • ``unwatched_secs`` — how long Steward wasn't recording before this tick (a
+    deploy, an outage, a stalled loop), from its own last sample; 0 when unbroken
+
+``cleanup_secs`` is recorded by the hourly cleanup itself when it finishes.
 """
 from __future__ import annotations
 import logging
 import os
+import time
+from datetime import datetime, timezone
+
+from sqlalchemy import func
 
 logger = logging.getLogger(__name__)
 
@@ -67,40 +82,57 @@ def fd_soft_limit() -> int | None:
     return soft
 
 
-async def record_self_metrics(app) -> None:
-    """Record open-fd usage as Steward's own metrics; warn past the floor.
+def unwatched_secs(last: datetime | None, now: datetime) -> float:
+    """How long Steward wasn't recording before `now`, given its last sample:
+    0 when the ticks are unbroken (a gap no longer than core.watch.GAP_S)."""
+    from .watch import GAP_S
+    if last is None:
+        return 0.0
+    gap = (now - last).total_seconds()
+    return gap if gap > GAP_S else 0.0
 
-    No-op (logged at debug) when fd accounting isn't available on this platform,
-    so it's safe to schedule unconditionally.
+
+async def record_self_metrics(app) -> None:
+    """Record Steward's own health as metrics; warn past the open-file floor.
+
+    The open-file pair is skipped (logged at debug) where fd accounting isn't
+    available, so it's safe to schedule unconditionally.
     """
     global _warned
 
+    from sqlalchemy import select, text
+    from steward.models.metrics import PluginMetricLatest
+    from .alerts import MetricBatch
+    from .health import ingest
+
     fds = count_open_fds()
     if fds is None:
         logger.debug("self_monitor: /proc/self/fd unavailable — skipping fd metrics")
-        return
-
-    from .alerts import MetricBatch
-
     soft = fd_soft_limit()
-    pct = (fds / soft * 100.0) if soft else None
+    pct = (fds / soft * 100.0) if (fds is not None and soft) else None
+    tally = ingest.take()
 
+    now = datetime.now(timezone.utc)
+    started = time.monotonic()
     async with app.db_sessionmaker() as session:
         async with session.begin():
+            await session.execute(text("SELECT 1"))
+            roundtrip_ms = (time.monotonic() - started) * 1000.0
+            # The last tick's sample, whatever restarts came between.
+            last = (await session.execute(select(func.max(PluginMetricLatest.recorded_at)).where(
+                PluginMetricLatest.source_module == "steward",
+                PluginMetricLatest.resource_name == "process",
+                PluginMetricLatest.metric_name == "db_roundtrip_ms"))).scalar()
             batch = MetricBatch()
-            batch.add(
-                source_module="steward",
-                resource_name="process",
-                metric_name="open_fds",
-                value=float(fds),
-            )
-            if pct is not None:
-                batch.add(
-                    source_module="steward",
-                    resource_name="process",
-                    metric_name="open_fds_pct",
-                    value=pct,
-                )
+            # Literal names, so tests/core/test_metric_catalog.py sees each one.
+            for metric_name, value in [
+                    ("ingest_posts", tally.posts), ("ingest_failed", tally.failed),
+                    ("ingest_slowest_ms", tally.slowest_ms), ("db_roundtrip_ms", roundtrip_ms),
+                    ("unwatched_secs", unwatched_secs(last, now)),
+                    ("open_fds", fds), ("open_fds_pct", pct)]:
+                if value is not None:
+                    batch.add(source_module="steward", resource_name="process",
+                              metric_name=metric_name, value=value, recorded_at=now)
             await batch.record(session)
 
     if pct is not None and pct >= FD_WARN_PCT:
@@ -113,4 +145,16 @@ async def record_self_metrics(app) -> None:
     else:
         _warned = False  # recovered — re-arm the warning for the next breach
 
-    logger.debug("self_monitor: open_fds=%d soft_limit=%s pct=%s", fds, soft, pct)
+    logger.debug("self_monitor: open_fds=%s soft_limit=%s pct=%s ingest=%s db=%.0fms",
+                 fds, soft, pct, tally, roundtrip_ms)
+
+
+async def record_cleanup_secs(app, secs: float) -> None:
+    """How long the hourly cleanup just took (#5467), as Steward's own metric."""
+    from .alerts import MetricBatch
+    async with app.db_sessionmaker() as session:
+        async with session.begin():
+            batch = MetricBatch()
+            batch.add(source_module="steward", resource_name="process",
+                      metric_name="cleanup_secs", value=secs)
+            await batch.record(session)

steward/settings/routes.py

@@ -623,6 +623,22 @@ async def _reports_page(settings: dict, flash):
                                  shown_day=shown_day, shown_hour=shown_hour, last_sent=last_sent)
 
 
+@settings_bp.get("/system/")
+@require_role(UserRole.admin)
+async def system():
+    """Steward's own health (#5467): its process metrics now and over the day,
+    each one an alert rule can watch (source "Steward itself")."""
+    from steward.core.health import process_health
+    from steward.core.status import sparkline_svg
+    async with current_app.db_sessionmaker() as db:
+        health = await process_health(db)
+    for row in health["rows"]:
+        # From zero to the row's own peak, with a floor (UI & Design ruling 8).
+        row["spark"] = sparkline_svg(row["series"], width=120, height=20,
+                                     stroke="currentColor", floor=1.0) if len(row["series"]) > 1 else ""
+    return await render_template("settings/system.html", **health)
+
+
 @settings_bp.get("/reports/")
 @require_role(UserRole.admin)
 async def reports():

steward/templates/settings/_tabs.html

@@ -10,5 +10,6 @@
                  ("auth", "Authentication", "/settings/auth/"),
                  ("users", "Users", "/settings/users/"),
                  ("ansible", "Ansible", "/settings/ansible/"),
-                 ("plugins", "Plugins", "/settings/plugins/")], active, "Settings") }}
+                 ("plugins", "Plugins", "/settings/plugins/"),
+                 ("system", "System", "/settings/system/")], active, "Settings") }}
 {%- endmacro %}

steward/templates/settings/system.html

@@ -0,0 +1,41 @@
+{# settings/system.html — Steward's own health (#5467): a verdict, then each of
+   its process metrics now and over the last day, from the self-monitor tick.
+   Each is a metric an alert rule can watch under "Steward itself". #}
+{% extends "base.html" %}
+{% from "_macros.html" import crumbs %}
+{% from "settings/_tabs.html" import settings_tabs %}
+{% block title %}Settings — System — Steward{% endblock %}
+{% block section_tabs %}{{ settings_tabs("system") }}{% endblock %}
+{% block breadcrumb %}{{ crumbs([("Settings", "/settings/"), ("System", "")]) }}{% endblock %}
+{% block content %}
+<p class="page-verdict">
+  {%- if not seen %}<span class="tone-dim">No readings yet: the first arrive within a minute of Steward starting.</span>
+  {%- elif problems %}{% set said = problems|join("; ") %}<span class="tone-warn">{{ said[:1]|upper }}{{ said[1:] }}.</span>
+  {%- else %}<span class="tone-ok">Steward is running normally.</span>{% endif -%}
+</p>
+
+<h2 class="section-title" style="margin:1.5rem 0 1rem;">Steward itself</h2>
+<div class="card-flush">
+  <table class="table">
+    <thead><tr><th>Measure</th><th>Now</th><th>Last 24 h</th></tr></thead>
+    <tbody>
+      {% for r in rows %}
+      <tr>
+        <td class="cell-name"><span title="{{ metric_label(r.metric) }}">{{ metric_label(r.metric) }}</span>
+          <div class="cell-sub" title="{{ metric_about(r.metric) }}">{{ metric_about(r.metric) }}</div></td>
+        <td class="nowrap">{{ metric_value(r.metric, r.now) }}
+          {%- if r.at %}<div class="cell-sub" title="{{ r.at|when }}">{{ r.at|ago }}</div>{% endif %}</td>
+        <td class="nowrap">
+          {%- if r.low is not none %}{{ metric_value(r.metric, r.low) }} – {{ metric_value(r.metric, r.high) }}
+          <span class="metric-preview-spark" aria-hidden="true">{{ r.spark|safe }}</span>
+          {%- else %}<span class="tone-dim">—</span>{% endif -%}
+        </td>
+      </tr>
+      {% endfor %}
+    </tbody>
+  </table>
+</div>
+<p class="field-hint" style="margin-top:0.75rem;">
+  Recorded each minute (the cleanup each hour). Any of these can be watched by an
+  <a href="/alerts/rules/new">alert rule</a>: pick the source "Steward itself".</p>
+{% endblock %}