From 0b2fbf13fca02faa238e8a7f511ba585487fa8d1 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Fri, 9 Oct 2026 14:29:55 -0400 Subject: [PATCH] =?UTF-8?q?feat(settings):=20Settings=20=E2=86=92=20System?= =?UTF-8?q?=20shows=20table=20sizes=20and=20what=20the=20last=20cleanup=20?= =?UTF-8?q?removed=20(#5469)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit #5469 "Settings → System: table sizes and what the last cleanup removed". - Storage: each table's size on disk (pg_total_relation_size, indexes and TOAST included) and Postgres's row estimate (reltuples), largest first, the 15 largest by name and the rest summed, under the database's total size. Read from the catalog only, never by counting rows. - Last cleanup: when it ran, how long it took, and what each pass removed (each aged table, run log files, the metric and Docker rollups, the log ring), or "Nothing was past its window". run_cleanup now returns that summary and keeps it as the cleanup.last_run setting, as reports.last_sent_at does; cleanup_secs (#5467) is recorded from the same figure. - A hint links to where each retention window is set. Co-Authored-By: Claude Opus 5.5 --- steward/app.py | 6 +-- steward/core/cleanup.py | 64 +++++++++++++++++++------- steward/core/health.py | 39 ++++++++++++++++ steward/core/settings.py | 3 ++ steward/settings/routes.py | 10 ++-- steward/templates/settings/system.html | 48 +++++++++++++++++++ tests/core/test_steward_health.py | 56 ++++++++++++++++++++-- tests/integration/test_cleanup.py | 31 +++++++++++-- 8 files changed, 227 insertions(+), 30 deletions(-) diff --git a/steward/app.py b/steward/app.py index e36784c..436f159 100644 --- a/steward/app.py +++ b/steward/app.py @@ -414,14 +414,12 @@ def _register_core_tasks(app: Quart) -> None: await run_due_monitors(app) async def run_cleanup(): - import time from .core.cleanup import run_cleanup as _cleanup from .core.self_monitor import record_cleanup_secs - started = time.monotonic() - await _cleanup(app) + done = await _cleanup(app) # How long it took, as Steward's own metric (#5467); a failed run # raises past this and records nothing. - await record_cleanup_secs(app, time.monotonic() - started) + await record_cleanup_secs(app, done["secs"]) async def run_alert_reconcile(): from .alerts.service import reconcile_covering_rules diff --git a/steward/core/cleanup.py b/steward/core/cleanup.py index bafde0b..3640bbd 100644 --- a/steward/core/cleanup.py +++ b/steward/core/cleanup.py @@ -2,6 +2,7 @@ from __future__ import annotations import asyncio import logging import os +import time from datetime import datetime, timedelta, timezone from typing import TYPE_CHECKING @@ -32,37 +33,66 @@ def prune_by_age(column) -> None: _AGED.append(column) -async def run_cleanup(app: "Quart") -> None: +# What each rollup count means, as Settings → System lists it (#5469). +_METRIC_COUNTS = [("raw_rows_rolled", "plugin_metrics: raw samples rolled into hourly"), + ("rollup_pruned", "plugin_metrics_hourly: past the rollup window"), + ("latest_pruned", "plugin_metric_latest: stopped reporting")] +_DOCKER_COUNTS = [("raw_rows_rolled", "docker_metrics: raw samples rolled into hourly"), + ("rollup_pruned", "docker_metrics_hourly: past the rollup window"), + ("events_pruned", "docker_events: past the events window"), + ("logs_age_pruned", "docker_logs: past the age window"), + ("logs_size_pruned", "docker_logs: over a container's size cap")] + + +async def run_cleanup(app: "Quart") -> dict: """Delete rows older than their retention window from every growing table, and run logs older than the data window, then run the metric and Docker - rollups. Windows are read from Settings each run (rule 25: no restart).""" - from steward.core.settings import get_setting + rollups. Windows are read from Settings each run (rule 25: no restart). + + Returns what it did, {"at", "secs", "removed": [[what, count]]}, and keeps + the same as the cleanup.last_run setting for Settings → System (#5469). A + run that raises records nothing, so the page keeps the last one that + finished.""" + from steward.core.settings import get_setting, set_setting from steward.ansible.executor import ARTIFACT_DIR now = datetime.now(timezone.utc) + started = time.monotonic() + removed: list[list] = [] async with app.db_sessionmaker() as session: async with session.begin(): data_days = int(await get_setting(session, "data.retention_days") or 90) audit_days = int(await get_setting(session, "audit.retention_days") or 365) cutoff = now - timedelta(days=data_days) - aged = [(c, cutoff) for c in _AGED] - aged.append((AuditEvent.timestamp, now - timedelta(days=audit_days))) - for column, before in aged: + aged = [(c, cutoff, data_days) for c in _AGED] + aged.append((AuditEvent.timestamp, now - timedelta(days=audit_days), audit_days)) + for column, before, days in aged: result = await session.execute( delete(column.class_).where(column < before) ) + table = column.class_.__tablename__ + removed.append([f"{table}: older than {days} days", result.rowcount or 0]) if result.rowcount: - logger.info("Pruned %d rows from %s", result.rowcount, - column.class_.__tablename__) + logger.info("Pruned %d rows from %s", result.rowcount, table) - removed = await asyncio.to_thread(prune_run_logs, ARTIFACT_DIR, cutoff) - if removed: - logger.info("Pruned %d Ansible run log files from %s", removed, ARTIFACT_DIR) + files = await asyncio.to_thread(prune_run_logs, ARTIFACT_DIR, cutoff) + removed.append([f"Ansible run log files: older than {data_days} days", files]) + if files: + logger.info("Pruned %d Ansible run log files from %s", files, ARTIFACT_DIR) # plugin_metrics is NOT blanket-deleted here — it's rolled up to hourly # then pruned, so multi-week host history stays cheap. - await _run_metrics_retention(session, now) - await _run_docker_retention(session, now) + counts = await _run_metrics_retention(session, now) + removed += [[what, counts.get(k) or 0] for k, what in _METRIC_COUNTS] + counts = await _run_docker_retention(session, now) + if counts is not None: + removed += [[what, counts.get(k) or 0] for k, what in _DOCKER_COUNTS] + + summary = {"at": now.isoformat(), "secs": round(time.monotonic() - started, 2), + "removed": removed} + async with session.begin(): + await set_setting(session, "cleanup.last_run", summary) + return summary def prune_run_logs(directory: str, before: datetime) -> int: @@ -85,7 +115,7 @@ def prune_run_logs(directory: str, before: datetime) -> int: return removed -async def _run_metrics_retention(session, now: datetime) -> None: +async def _run_metrics_retention(session, now: datetime) -> dict: """Roll up + prune plugin_metrics (raw → hourly → gone). Windows read fresh from settings each run (rule 25 — UI change takes effect next cleanup, no restart). get_setting's SELECT autobegins, so read inside the begin block.""" @@ -100,9 +130,10 @@ async def _run_metrics_retention(session, now: datetime) -> None: ) if counts and any(counts.values()): logger.info("Metrics retention: %s", counts) + return counts or {} -async def _run_docker_retention(session, now: datetime) -> None: +async def _run_docker_retention(session, now: datetime) -> dict | None: """Drive the docker plugin's rollup + prune and its log ring via its capabilities, if loaded. @@ -114,7 +145,7 @@ async def _run_docker_retention(session, now: datetime) -> None: """ from steward.core.capabilities import has_capability, invoke_capability if not has_capability("docker.run_retention"): - return + return None from steward.core.settings import get_setting from steward.models.users import UserRole @@ -139,3 +170,4 @@ async def _run_docker_retention(session, now: datetime) -> None: ) if counts and any(counts.values()): logger.info("Docker retention: %s", counts) + return counts diff --git a/steward/core/health.py b/steward/core/health.py index d5acc0b..659f221 100644 --- a/steward/core/health.py +++ b/steward/core/health.py @@ -67,3 +67,42 @@ async def process_health(db) -> dict: if (by["open_fds_pct"]["now"] or 0) >= FD_WARN_PCT: problems.append(f"open files at {by['open_fds_pct']['now']:.0f}% of the limit") return {"rows": rows, "problems": problems, "seen": any(r["at"] for r in rows)} + + +# How many tables Settings → System lists by name; the rest are summed. +TABLES_SHOWN = 15 + + +async def table_sizes(db) -> dict: + """Steward's tables by size on disk, largest first (#5469), from Postgres's + own catalog: pg_total_relation_size (indexes and TOAST included) and the + planner's row estimate (reltuples; None for a table never analysed). + + Returns {"total": bytes of the whole database, "tables": [{name, bytes, + rows}] for the TABLES_SHOWN largest, "rest": {count, bytes} of the others}.""" + from sqlalchemy import text + rows = (await db.execute(text( + "SELECT c.relname, pg_total_relation_size(c.oid), c.reltuples::bigint " + "FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace " + "WHERE c.relkind IN ('r', 'p') AND n.nspname = current_schema() " + "ORDER BY 2 DESC"))).all() + total = (await db.execute(text("SELECT pg_database_size(current_database())"))).scalar() + tables = [{"name": name, "bytes": size, "rows": est if est is not None and est >= 0 else None} + for name, size, est in rows] + rest = tables[TABLES_SHOWN:] + return {"total": total, "tables": tables[:TABLES_SHOWN], + "rest": {"count": len(rest), "bytes": sum(t["bytes"] for t in rest)}} + + +async def last_cleanup(db) -> dict | None: + """The last hourly cleanup that finished (#5469), as core/cleanup.py kept + it: {"at": datetime, "secs", "removed": [(what, n)] with n > 0, "passes": + how many passes ran}. None before the first one finishes.""" + from datetime import datetime + from steward.core.settings import get_setting + run = await get_setting(db, "cleanup.last_run") or {} + if not run.get("at"): + return None + removed = [(what, n) for what, n in run.get("removed") or []] + return {"at": datetime.fromisoformat(run["at"]), "secs": run.get("secs"), + "removed": [(what, n) for what, n in removed if n], "passes": len(removed)} diff --git a/steward/core/settings.py b/steward/core/settings.py index 8715b8c..97e3879 100644 --- a/steward/core/settings.py +++ b/steward/core/settings.py @@ -175,6 +175,9 @@ DEFAULTS: dict[str, Any] = { # so "Sunday 08:00" means 08:00 where they are (#5378). "reports.schedule_tz": "", "reports.last_sent_at": "", + # What the last hourly cleanup did (#5469), written by core/cleanup.py for + # Settings → System: {"at": ISO time, "secs": float, "removed": [[what, n]]}. + "cleanup.last_run": {}, } # Settings encrypted at rest (transparent encrypt-on-write / decrypt-on-read). diff --git a/steward/settings/routes.py b/steward/settings/routes.py index e2f5d09..19c2b33 100644 --- a/steward/settings/routes.py +++ b/steward/settings/routes.py @@ -627,16 +627,20 @@ async def _reports_page(settings: dict, flash): @require_role(UserRole.admin) async def system(): """Steward's own health (#5467): its process metrics now and over the day, - each one an alert rule can watch (source "Steward itself").""" - from steward.core.health import process_health + each one an alert rule can watch (source "Steward itself"); then its + tables' sizes and what the last hourly cleanup removed (#5469).""" + from steward.core.health import last_cleanup, process_health, table_sizes from steward.core.status import sparkline_svg async with current_app.db_sessionmaker() as db: health = await process_health(db) + # Table sizes and the last cleanup (#5469): Postgres's catalog and + # what the cleanup kept, never a scan of the tables themselves. + storage, cleanup = await table_sizes(db), await last_cleanup(db) for row in health["rows"]: # From zero to the row's own peak, with a floor (UI & Design ruling 8). row["spark"] = sparkline_svg(row["series"], width=120, height=20, stroke="currentColor", floor=1.0) if len(row["series"]) > 1 else "" - return await render_template("settings/system.html", **health) + return await render_template("settings/system.html", **health, storage=storage, cleanup=cleanup) @settings_bp.get("/reports/") diff --git a/steward/templates/settings/system.html b/steward/templates/settings/system.html index c5f0543..d7db32c 100644 --- a/steward/templates/settings/system.html +++ b/steward/templates/settings/system.html @@ -38,4 +38,52 @@

Recorded each minute (the cleanup each hour). Any of these can be watched by an alert rule: pick the source "Steward itself".

+ +{# Storage (#5469): Postgres's own figures for each table, then what the + hourly cleanup last removed. Read from the catalog, never by counting. #} +

Storage

+

The database takes {{ storage.total|bytes }} on disk.

+
+ + + + {% for t in storage.tables %} + + + + + + {% endfor %} + {% if storage.rest.count %} + + + {% endif %} + +
TableOn diskRows (estimate)
{{ t.name }}{{ t.bytes|bytes }}{% if t.rows is not none %}{{ "{:,}".format(t.rows) }}{% else %}—{% endif %}
+{{ storage.rest.count }} smaller tables{{ storage.rest.bytes|bytes }}
+
+ +

Last cleanup

+{% if not cleanup %} +

None has finished since this version started: the cleanup runs each hour.

+{% else %} +

Ran {{ cleanup.at|ago }} and took {{ cleanup.secs|duration }}. + {%- if not cleanup.removed %} Nothing was past its window ({{ cleanup.passes }} passes).{% endif %}

+{% if cleanup.removed %} +
+ + + + {% for what, n in cleanup.removed %} + + + {% endfor %} + +
What wentRows
{{ what }}{{ "{:,}".format(n) }}
+
+{% endif %} +{% endif %} +

+ How long each kind of data is kept is set under + General (data and audit log) and + Thresholds & retention (metrics, Docker and logs).

{% endblock %} diff --git a/tests/core/test_steward_health.py b/tests/core/test_steward_health.py index 324d4d5..5ed9673 100644 --- a/tests/core/test_steward_health.py +++ b/tests/core/test_steward_health.py @@ -8,6 +8,16 @@ from quart import render_template, session from steward.core import health AT = datetime(2026, 10, 9, 12, 0, tzinfo=timezone.utc) +STORAGE = {"total": 3 * 2**30, "rest": {"count": 0, "bytes": 0}, + "tables": [{"name": "plugin_metrics", "bytes": 2 * 2**30, "rows": 41_000_000}, + {"name": "docker_logs", "bytes": 512 * 2**20, "rows": None}]} + + +async def _page(app, ctx, storage=STORAGE, cleanup=None): + async with app.test_request_context("/settings/system/"): + session["user_role"] = "admin" + page = await render_template("settings/system.html", **ctx, storage=storage, cleanup=cleanup) + return " ".join(page.split()) def _previews(**over): @@ -52,10 +62,50 @@ async def test_the_page_reads_as_a_verdict_then_each_measure(app, previews): got = await health.process_health(None) for r in got["rows"]: r["spark"] = "" - async with app.test_request_context("/settings/system/"): - session["user_role"] = "admin" - page = " ".join((await render_template("settings/system.html", **got)).split()) + page = await _page(app, got) assert 'Steward wasn't watching for up to 7m' in page assert 'title="<' not in page # a