"""Steward's own health between self-monitor ticks (#5467). The host-agent ingest reports each post here as it finishes; the self-monitor tick (core/self_monitor.py) takes the minute's totals and records them as metrics under source "steward", resource "process", beside the open-file counts, where alert rules can watch them like any other metric. One process serves Steward, so a module-level counter is the whole of it. """ from __future__ import annotations from dataclasses import dataclass # Steward's own metrics, in the order the Settings → System page lists them. PROCESS_METRICS = ["ingest_posts", "ingest_failed", "ingest_slowest_ms", "db_roundtrip_ms", "unwatched_secs", "cleanup_secs", "open_fds", "open_fds_pct"] @dataclass class IngestTally: posts: int = 0 failed: int = 0 slowest_ms: float = 0.0 def done(self, ms: float, ok: bool = True) -> None: """One ingest post finished in `ms`; `ok` False when it didn't land (the agent gave up, or the handler raised).""" self.posts += 1 if not ok: self.failed += 1 self.slowest_ms = max(self.slowest_ms, ms) def take(self) -> "IngestTally": """The totals since the last take, starting the next minute afresh.""" got = IngestTally(self.posts, self.failed, self.slowest_ms) self.posts, self.failed, self.slowest_ms = 0, 0, 0.0 return got ingest = IngestTally() async def process_health(db) -> dict: """What Settings → System shows (#5467): each of Steward's own metrics now and over the last day, and the problems among them, worst first. Returns {"rows": [{metric, now, at, low, high, series}], "problems": [str], "seen": bool}. Reads plugin_metric_latest and a day of each series (the covering series index makes that cheap, #5490).""" from steward.alerts.service import metric_preview from steward.core.formatting import human_duration rows = [] for metric in PROCESS_METRICS: p = await metric_preview(db, "steward", "process", metric) rows.append({"metric": metric, **(p or {"now": None, "at": None, "series": [], "low": None, "high": None})}) by = {r["metric"]: r for r in rows} problems = [] if (by["ingest_failed"]["now"] or 0) > 0: problems.append(f"{by['ingest_failed']['now']:.0f} agent posts failed in the last minute") elif (by["ingest_failed"]["high"] or 0) > 0: problems.append(f"agent posts failed in the last day, up to {by['ingest_failed']['high']:.0f} " "in a minute") if (by["unwatched_secs"]["high"] or 0) > 0: problems.append(f"Steward wasn't watching for up to {human_duration(by['unwatched_secs']['high'])} " "in the last day (a deploy or an outage)") from steward.core.self_monitor import FD_WARN_PCT if (by["open_fds_pct"]["now"] or 0) >= FD_WARN_PCT: problems.append(f"open files at {by['open_fds_pct']['now']:.0f}% of the limit") return {"rows": rows, "problems": problems, "seen": any(r["at"] for r in rows)} # How many tables Settings → System lists by name; the rest are summed. TABLES_SHOWN = 15 async def table_sizes(db) -> dict: """Steward's tables by size on disk, largest first (#5469), from Postgres's own catalog: pg_total_relation_size (indexes and TOAST included) and the planner's row estimate (reltuples; None for a table never analysed). Returns {"total": bytes of the whole database, "tables": [{name, bytes, rows}] for the TABLES_SHOWN largest, "rest": {count, bytes} of the others}.""" from sqlalchemy import text rows = (await db.execute(text( "SELECT c.relname, pg_total_relation_size(c.oid), c.reltuples::bigint " "FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace " "WHERE c.relkind IN ('r', 'p') AND n.nspname = current_schema() " "ORDER BY 2 DESC"))).all() total = (await db.execute(text("SELECT pg_database_size(current_database())"))).scalar() tables = [{"name": name, "bytes": size, "rows": est if est is not None and est >= 0 else None} for name, size, est in rows] rest = tables[TABLES_SHOWN:] return {"total": total, "tables": tables[:TABLES_SHOWN], "rest": {"count": len(rest), "bytes": sum(t["bytes"] for t in rest)}} async def last_cleanup(db) -> dict | None: """The last hourly cleanup that finished (#5469), as core/cleanup.py kept it: {"at": datetime, "secs", "removed": [(what, n)] with n > 0, "passes": how many passes ran}. None before the first one finishes.""" from datetime import datetime from steward.core.settings import get_setting run = await get_setting(db, "cleanup.last_run") or {} if not run.get("at"): return None removed = [(what, n) for what, n in run.get("removed") or []] return {"at": datetime.fromisoformat(run["at"]), "secs": run.get("secs"), "removed": [(what, n) for what, n in removed if n], "passes": len(removed)}