"""Steward's own health between self-monitor ticks (#5467). The host-agent ingest reports each post here as it finishes; the self-monitor tick (core/self_monitor.py) takes the minute's totals and records them as metrics under source "steward", resource "process", beside the open-file counts, where alert rules can watch them like any other metric. One process serves Steward, so a module-level counter is the whole of it. """ from __future__ import annotations from dataclasses import dataclass # Steward's own metrics, in the order the Settings → System page lists them. PROCESS_METRICS = ["ingest_posts", "ingest_failed", "ingest_slowest_ms", "db_roundtrip_ms", "unwatched_secs", "cleanup_secs", "open_fds", "open_fds_pct"] @dataclass class IngestTally: posts: int = 0 failed: int = 0 slowest_ms: float = 0.0 def done(self, ms: float, ok: bool = True) -> None: """One ingest post finished in `ms`; `ok` False when it didn't land (the agent gave up, or the handler raised).""" self.posts += 1 if not ok: self.failed += 1 self.slowest_ms = max(self.slowest_ms, ms) def take(self) -> "IngestTally": """The totals since the last take, starting the next minute afresh.""" got = IngestTally(self.posts, self.failed, self.slowest_ms) self.posts, self.failed, self.slowest_ms = 0, 0, 0.0 return got ingest = IngestTally() async def process_health(db) -> dict: """What Settings → System shows (#5467): each of Steward's own metrics now and over the last day, and the problems among them, worst first. Returns {"rows": [{metric, now, at, low, high, series}], "problems": [str], "seen": bool}. Reads plugin_metric_latest and a day of each series (the covering series index makes that cheap, #5490).""" from steward.alerts.service import metric_preview from steward.core.formatting import human_duration rows = [] for metric in PROCESS_METRICS: p = await metric_preview(db, "steward", "process", metric) rows.append({"metric": metric, **(p or {"now": None, "at": None, "series": [], "low": None, "high": None})}) by = {r["metric"]: r for r in rows} problems = [] if (by["ingest_failed"]["now"] or 0) > 0: problems.append(f"{by['ingest_failed']['now']:.0f} agent posts failed in the last minute") elif (by["ingest_failed"]["high"] or 0) > 0: problems.append(f"agent posts failed in the last day, up to {by['ingest_failed']['high']:.0f} " "in a minute") if (by["unwatched_secs"]["high"] or 0) > 0: problems.append(f"Steward wasn't watching for up to {human_duration(by['unwatched_secs']['high'])} " "in the last day (a deploy or an outage)") from steward.core.self_monitor import FD_WARN_PCT if (by["open_fds_pct"]["now"] or 0) >= FD_WARN_PCT: problems.append(f"open files at {by['open_fds_pct']['now']:.0f}% of the limit") return {"rows": rows, "problems": problems, "seen": any(r["at"] for r in rows)}