bvandeusenandClaude Opus 5.5
0b2fbf13fc
#5469 "Settings → System: table sizes and what the last cleanup removed". - Storage: each table's size on disk (pg_total_relation_size, indexes and TOAST included) and Postgres's row estimate (reltuples), largest first, the 15 largest by name and the rest summed, under the database's total size. Read from the catalog only, never by counting rows. - Last cleanup: when it ran, how long it took, and what each pass removed (each aged table, run log files, the metric and Docker rollups, the log ring), or "Nothing was past its window". run_cleanup now returns that summary and keeps it as the cleanup.last_run setting, as reports.last_sent_at does; cleanup_secs (#5467) is recorded from the same figure. - A hint links to where each retention window is set. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
109 lines
5.0 KiB
Python
109 lines
5.0 KiB
Python
"""Steward's own health between self-monitor ticks (#5467).
|
|
|
|
The host-agent ingest reports each post here as it finishes; the self-monitor
|
|
tick (core/self_monitor.py) takes the minute's totals and records them as
|
|
metrics under source "steward", resource "process", beside the open-file
|
|
counts, where alert rules can watch them like any other metric. One process
|
|
serves Steward, so a module-level counter is the whole of it.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
from dataclasses import dataclass
|
|
|
|
# Steward's own metrics, in the order the Settings → System page lists them.
|
|
PROCESS_METRICS = ["ingest_posts", "ingest_failed", "ingest_slowest_ms", "db_roundtrip_ms",
|
|
"unwatched_secs", "cleanup_secs", "open_fds", "open_fds_pct"]
|
|
|
|
|
|
@dataclass
|
|
class IngestTally:
|
|
posts: int = 0
|
|
failed: int = 0
|
|
slowest_ms: float = 0.0
|
|
|
|
def done(self, ms: float, ok: bool = True) -> None:
|
|
"""One ingest post finished in `ms`; `ok` False when it didn't land
|
|
(the agent gave up, or the handler raised)."""
|
|
self.posts += 1
|
|
if not ok:
|
|
self.failed += 1
|
|
self.slowest_ms = max(self.slowest_ms, ms)
|
|
|
|
def take(self) -> "IngestTally":
|
|
"""The totals since the last take, starting the next minute afresh."""
|
|
got = IngestTally(self.posts, self.failed, self.slowest_ms)
|
|
self.posts, self.failed, self.slowest_ms = 0, 0, 0.0
|
|
return got
|
|
|
|
|
|
ingest = IngestTally()
|
|
|
|
|
|
async def process_health(db) -> dict:
|
|
"""What Settings → System shows (#5467): each of Steward's own metrics now
|
|
and over the last day, and the problems among them, worst first.
|
|
|
|
Returns {"rows": [{metric, now, at, low, high, series}], "problems": [str],
|
|
"seen": bool}. Reads plugin_metric_latest and a day of each series (the
|
|
covering series index makes that cheap, #5490)."""
|
|
from steward.alerts.service import metric_preview
|
|
from steward.core.formatting import human_duration
|
|
rows = []
|
|
for metric in PROCESS_METRICS:
|
|
p = await metric_preview(db, "steward", "process", metric)
|
|
rows.append({"metric": metric, **(p or {"now": None, "at": None, "series": [],
|
|
"low": None, "high": None})})
|
|
by = {r["metric"]: r for r in rows}
|
|
problems = []
|
|
if (by["ingest_failed"]["now"] or 0) > 0:
|
|
problems.append(f"{by['ingest_failed']['now']:.0f} agent posts failed in the last minute")
|
|
elif (by["ingest_failed"]["high"] or 0) > 0:
|
|
problems.append(f"agent posts failed in the last day, up to {by['ingest_failed']['high']:.0f} "
|
|
"in a minute")
|
|
if (by["unwatched_secs"]["high"] or 0) > 0:
|
|
problems.append(f"Steward wasn't watching for up to {human_duration(by['unwatched_secs']['high'])} "
|
|
"in the last day (a deploy or an outage)")
|
|
from steward.core.self_monitor import FD_WARN_PCT
|
|
if (by["open_fds_pct"]["now"] or 0) >= FD_WARN_PCT:
|
|
problems.append(f"open files at {by['open_fds_pct']['now']:.0f}% of the limit")
|
|
return {"rows": rows, "problems": problems, "seen": any(r["at"] for r in rows)}
|
|
|
|
|
|
# How many tables Settings → System lists by name; the rest are summed.
|
|
TABLES_SHOWN = 15
|
|
|
|
|
|
async def table_sizes(db) -> dict:
|
|
"""Steward's tables by size on disk, largest first (#5469), from Postgres's
|
|
own catalog: pg_total_relation_size (indexes and TOAST included) and the
|
|
planner's row estimate (reltuples; None for a table never analysed).
|
|
|
|
Returns {"total": bytes of the whole database, "tables": [{name, bytes,
|
|
rows}] for the TABLES_SHOWN largest, "rest": {count, bytes} of the others}."""
|
|
from sqlalchemy import text
|
|
rows = (await db.execute(text(
|
|
"SELECT c.relname, pg_total_relation_size(c.oid), c.reltuples::bigint "
|
|
"FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace "
|
|
"WHERE c.relkind IN ('r', 'p') AND n.nspname = current_schema() "
|
|
"ORDER BY 2 DESC"))).all()
|
|
total = (await db.execute(text("SELECT pg_database_size(current_database())"))).scalar()
|
|
tables = [{"name": name, "bytes": size, "rows": est if est is not None and est >= 0 else None}
|
|
for name, size, est in rows]
|
|
rest = tables[TABLES_SHOWN:]
|
|
return {"total": total, "tables": tables[:TABLES_SHOWN],
|
|
"rest": {"count": len(rest), "bytes": sum(t["bytes"] for t in rest)}}
|
|
|
|
|
|
async def last_cleanup(db) -> dict | None:
|
|
"""The last hourly cleanup that finished (#5469), as core/cleanup.py kept
|
|
it: {"at": datetime, "secs", "removed": [(what, n)] with n > 0, "passes":
|
|
how many passes ran}. None before the first one finishes."""
|
|
from datetime import datetime
|
|
from steward.core.settings import get_setting
|
|
run = await get_setting(db, "cleanup.last_run") or {}
|
|
if not run.get("at"):
|
|
return None
|
|
removed = [(what, n) for what, n in run.get("removed") or []]
|
|
return {"at": datetime.fromisoformat(run["at"]), "secs": run.get("secs"),
|
|
"removed": [(what, n) for what, n in removed if n], "passes": len(removed)}
|