bvandeusenandClaude Opus 5.5
c8dbe811f8
#5467 "Steward's own health: metrics, rules can watch them, a page that shows them". The self-monitor tick now records, under source "steward", resource "process": ingest_posts / ingest_failed / ingest_slowest_ms (the host-agent posts since the last tick, tallied in steward/core/health.py), db_roundtrip_ms (a timed SELECT 1), and unwatched_secs (the gap before this tick past watch.GAP_S, from Steward's own last sample). The hourly cleanup records cleanup_secs. The open-file pair is still recorded, and is now only skipped on its own where /proc is absent. All of these are in CORE_METRICS, so an alert rule can watch them under "Steward itself". Settings → System (/settings/system/, admin) shows a verdict first (failed posts, a gap in watching, open files near the limit), then each measure now and over the last day, with a sparkline scaled to its own peak. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
70 lines
3.1 KiB
Python
70 lines
3.1 KiB
Python
"""Steward's own health between self-monitor ticks (#5467).
|
|
|
|
The host-agent ingest reports each post here as it finishes; the self-monitor
|
|
tick (core/self_monitor.py) takes the minute's totals and records them as
|
|
metrics under source "steward", resource "process", beside the open-file
|
|
counts, where alert rules can watch them like any other metric. One process
|
|
serves Steward, so a module-level counter is the whole of it.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
from dataclasses import dataclass
|
|
|
|
# Steward's own metrics, in the order the Settings → System page lists them.
|
|
PROCESS_METRICS = ["ingest_posts", "ingest_failed", "ingest_slowest_ms", "db_roundtrip_ms",
|
|
"unwatched_secs", "cleanup_secs", "open_fds", "open_fds_pct"]
|
|
|
|
|
|
@dataclass
|
|
class IngestTally:
|
|
posts: int = 0
|
|
failed: int = 0
|
|
slowest_ms: float = 0.0
|
|
|
|
def done(self, ms: float, ok: bool = True) -> None:
|
|
"""One ingest post finished in `ms`; `ok` False when it didn't land
|
|
(the agent gave up, or the handler raised)."""
|
|
self.posts += 1
|
|
if not ok:
|
|
self.failed += 1
|
|
self.slowest_ms = max(self.slowest_ms, ms)
|
|
|
|
def take(self) -> "IngestTally":
|
|
"""The totals since the last take, starting the next minute afresh."""
|
|
got = IngestTally(self.posts, self.failed, self.slowest_ms)
|
|
self.posts, self.failed, self.slowest_ms = 0, 0, 0.0
|
|
return got
|
|
|
|
|
|
ingest = IngestTally()
|
|
|
|
|
|
async def process_health(db) -> dict:
|
|
"""What Settings → System shows (#5467): each of Steward's own metrics now
|
|
and over the last day, and the problems among them, worst first.
|
|
|
|
Returns {"rows": [{metric, now, at, low, high, series}], "problems": [str],
|
|
"seen": bool}. Reads plugin_metric_latest and a day of each series (the
|
|
covering series index makes that cheap, #5490)."""
|
|
from steward.alerts.service import metric_preview
|
|
from steward.core.formatting import human_duration
|
|
rows = []
|
|
for metric in PROCESS_METRICS:
|
|
p = await metric_preview(db, "steward", "process", metric)
|
|
rows.append({"metric": metric, **(p or {"now": None, "at": None, "series": [],
|
|
"low": None, "high": None})})
|
|
by = {r["metric"]: r for r in rows}
|
|
problems = []
|
|
if (by["ingest_failed"]["now"] or 0) > 0:
|
|
problems.append(f"{by['ingest_failed']['now']:.0f} agent posts failed in the last minute")
|
|
elif (by["ingest_failed"]["high"] or 0) > 0:
|
|
problems.append(f"agent posts failed in the last day, up to {by['ingest_failed']['high']:.0f} "
|
|
"in a minute")
|
|
if (by["unwatched_secs"]["high"] or 0) > 0:
|
|
problems.append(f"Steward wasn't watching for up to {human_duration(by['unwatched_secs']['high'])} "
|
|
"in the last day (a deploy or an outage)")
|
|
from steward.core.self_monitor import FD_WARN_PCT
|
|
if (by["open_fds_pct"]["now"] or 0) >= FD_WARN_PCT:
|
|
problems.append(f"open files at {by['open_fds_pct']['now']:.0f}% of the limit")
|
|
return {"rows": rows, "problems": problems, "seen": any(r["at"] for r in rows)}
|