feat(settings): Settings → System shows table sizes and what the last cleanup removed (#5469)

#5469 "Settings → System: table sizes and what the last cleanup removed".

- Storage: each table's size on disk (pg_total_relation_size, indexes and TOAST
  included) and Postgres's row estimate (reltuples), largest first, the 15
  largest by name and the rest summed, under the database's total size. Read
  from the catalog only, never by counting rows.
- Last cleanup: when it ran, how long it took, and what each pass removed (each
  aged table, run log files, the metric and Docker rollups, the log ring), or
  "Nothing was past its window". run_cleanup now returns that summary and keeps
  it as the cleanup.last_run setting, as reports.last_sent_at does; cleanup_secs
  (#5467) is recorded from the same figure.
- A hint links to where each retention window is set.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

steward/app.py

@@ -414,14 +414,12 @@ def _register_core_tasks(app: Quart) -> None:
         await run_due_monitors(app)
 
     async def run_cleanup():
-        import time
         from .core.cleanup import run_cleanup as _cleanup
         from .core.self_monitor import record_cleanup_secs
-        started = time.monotonic()
-        await _cleanup(app)
+        done = await _cleanup(app)
         # How long it took, as Steward's own metric (#5467); a failed run
         # raises past this and records nothing.
-        await record_cleanup_secs(app, time.monotonic() - started)
+        await record_cleanup_secs(app, done["secs"])
 
     async def run_alert_reconcile():
         from .alerts.service import reconcile_covering_rules

steward/core/cleanup.py

@@ -2,6 +2,7 @@ from __future__ import annotations
 import asyncio
 import logging
 import os
+import time
 from datetime import datetime, timedelta, timezone
 from typing import TYPE_CHECKING
 
@@ -32,37 +33,66 @@ def prune_by_age(column) -> None:
         _AGED.append(column)
 
 
-async def run_cleanup(app: "Quart") -> None:
+# What each rollup count means, as Settings → System lists it (#5469).
+_METRIC_COUNTS = [("raw_rows_rolled", "plugin_metrics: raw samples rolled into hourly"),
+                  ("rollup_pruned", "plugin_metrics_hourly: past the rollup window"),
+                  ("latest_pruned", "plugin_metric_latest: stopped reporting")]
+_DOCKER_COUNTS = [("raw_rows_rolled", "docker_metrics: raw samples rolled into hourly"),
+                  ("rollup_pruned", "docker_metrics_hourly: past the rollup window"),
+                  ("events_pruned", "docker_events: past the events window"),
+                  ("logs_age_pruned", "docker_logs: past the age window"),
+                  ("logs_size_pruned", "docker_logs: over a container's size cap")]
+
+
+async def run_cleanup(app: "Quart") -> dict:
     """Delete rows older than their retention window from every growing table,
     and run logs older than the data window, then run the metric and Docker
-    rollups. Windows are read from Settings each run (rule 25: no restart)."""
-    from steward.core.settings import get_setting
+    rollups. Windows are read from Settings each run (rule 25: no restart).
+
+    Returns what it did, {"at", "secs", "removed": [[what, count]]}, and keeps
+    the same as the cleanup.last_run setting for Settings → System (#5469). A
+    run that raises records nothing, so the page keeps the last one that
+    finished."""
+    from steward.core.settings import get_setting, set_setting
     from steward.ansible.executor import ARTIFACT_DIR
 
     now = datetime.now(timezone.utc)
+    started = time.monotonic()
+    removed: list[list] = []
     async with app.db_sessionmaker() as session:
         async with session.begin():
             data_days = int(await get_setting(session, "data.retention_days") or 90)
             audit_days = int(await get_setting(session, "audit.retention_days") or 365)
             cutoff = now - timedelta(days=data_days)
-            aged = [(c, cutoff) for c in _AGED]
-            aged.append((AuditEvent.timestamp, now - timedelta(days=audit_days)))
-            for column, before in aged:
+            aged = [(c, cutoff, data_days) for c in _AGED]
+            aged.append((AuditEvent.timestamp, now - timedelta(days=audit_days), audit_days))
+            for column, before, days in aged:
                 result = await session.execute(
                     delete(column.class_).where(column < before)
                 )
+                table = column.class_.__tablename__
+                removed.append([f"{table}: older than {days} days", result.rowcount or 0])
                 if result.rowcount:
-                    logger.info("Pruned %d rows from %s", result.rowcount,
-                                column.class_.__tablename__)
+                    logger.info("Pruned %d rows from %s", result.rowcount, table)
 
-        removed = await asyncio.to_thread(prune_run_logs, ARTIFACT_DIR, cutoff)
-        if removed:
-            logger.info("Pruned %d Ansible run log files from %s", removed, ARTIFACT_DIR)
+        files = await asyncio.to_thread(prune_run_logs, ARTIFACT_DIR, cutoff)
+        removed.append([f"Ansible run log files: older than {data_days} days", files])
+        if files:
+            logger.info("Pruned %d Ansible run log files from %s", files, ARTIFACT_DIR)
 
         # plugin_metrics is NOT blanket-deleted here — it's rolled up to hourly
         # then pruned, so multi-week host history stays cheap.
-        await _run_metrics_retention(session, now)
-        await _run_docker_retention(session, now)
+        counts = await _run_metrics_retention(session, now)
+        removed += [[what, counts.get(k) or 0] for k, what in _METRIC_COUNTS]
+        counts = await _run_docker_retention(session, now)
+        if counts is not None:
+            removed += [[what, counts.get(k) or 0] for k, what in _DOCKER_COUNTS]
+
+        summary = {"at": now.isoformat(), "secs": round(time.monotonic() - started, 2),
+                   "removed": removed}
+        async with session.begin():
+            await set_setting(session, "cleanup.last_run", summary)
+    return summary
 
 
 def prune_run_logs(directory: str, before: datetime) -> int:
@@ -85,7 +115,7 @@ def prune_run_logs(directory: str, before: datetime) -> int:
     return removed
 
 
-async def _run_metrics_retention(session, now: datetime) -> None:
+async def _run_metrics_retention(session, now: datetime) -> dict:
     """Roll up + prune plugin_metrics (raw → hourly → gone). Windows read fresh
     from settings each run (rule 25 — UI change takes effect next cleanup, no
     restart). get_setting's SELECT autobegins, so read inside the begin block."""
@@ -100,9 +130,10 @@ async def _run_metrics_retention(session, now: datetime) -> None:
         )
     if counts and any(counts.values()):
         logger.info("Metrics retention: %s", counts)
+    return counts or {}
 
 
-async def _run_docker_retention(session, now: datetime) -> None:
+async def _run_docker_retention(session, now: datetime) -> dict | None:
     """Drive the docker plugin's rollup + prune and its log ring via its
     capabilities, if loaded.
 
@@ -114,7 +145,7 @@ async def _run_docker_retention(session, now: datetime) -> None:
     """
     from steward.core.capabilities import has_capability, invoke_capability
     if not has_capability("docker.run_retention"):
-        return
+        return None
     from steward.core.settings import get_setting
     from steward.models.users import UserRole
 
@@ -139,3 +170,4 @@ async def _run_docker_retention(session, now: datetime) -> None:
         )
     if counts and any(counts.values()):
         logger.info("Docker retention: %s", counts)
+    return counts

steward/core/health.py

@@ -67,3 +67,42 @@ async def process_health(db) -> dict:
     if (by["open_fds_pct"]["now"] or 0) >= FD_WARN_PCT:
         problems.append(f"open files at {by['open_fds_pct']['now']:.0f}% of the limit")
     return {"rows": rows, "problems": problems, "seen": any(r["at"] for r in rows)}
+
+
+# How many tables Settings → System lists by name; the rest are summed.
+TABLES_SHOWN = 15
+
+
+async def table_sizes(db) -> dict:
+    """Steward's tables by size on disk, largest first (#5469), from Postgres's
+    own catalog: pg_total_relation_size (indexes and TOAST included) and the
+    planner's row estimate (reltuples; None for a table never analysed).
+
+    Returns {"total": bytes of the whole database, "tables": [{name, bytes,
+    rows}] for the TABLES_SHOWN largest, "rest": {count, bytes} of the others}."""
+    from sqlalchemy import text
+    rows = (await db.execute(text(
+        "SELECT c.relname, pg_total_relation_size(c.oid), c.reltuples::bigint "
+        "FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace "
+        "WHERE c.relkind IN ('r', 'p') AND n.nspname = current_schema() "
+        "ORDER BY 2 DESC"))).all()
+    total = (await db.execute(text("SELECT pg_database_size(current_database())"))).scalar()
+    tables = [{"name": name, "bytes": size, "rows": est if est is not None and est >= 0 else None}
+              for name, size, est in rows]
+    rest = tables[TABLES_SHOWN:]
+    return {"total": total, "tables": tables[:TABLES_SHOWN],
+            "rest": {"count": len(rest), "bytes": sum(t["bytes"] for t in rest)}}
+
+
+async def last_cleanup(db) -> dict | None:
+    """The last hourly cleanup that finished (#5469), as core/cleanup.py kept
+    it: {"at": datetime, "secs", "removed": [(what, n)] with n > 0, "passes":
+    how many passes ran}. None before the first one finishes."""
+    from datetime import datetime
+    from steward.core.settings import get_setting
+    run = await get_setting(db, "cleanup.last_run") or {}
+    if not run.get("at"):
+        return None
+    removed = [(what, n) for what, n in run.get("removed") or []]
+    return {"at": datetime.fromisoformat(run["at"]), "secs": run.get("secs"),
+            "removed": [(what, n) for what, n in removed if n], "passes": len(removed)}

steward/core/settings.py

@@ -175,6 +175,9 @@ DEFAULTS: dict[str, Any] = {
     # so "Sunday 08:00" means 08:00 where they are (#5378).
     "reports.schedule_tz": "",
     "reports.last_sent_at": "",
+    # What the last hourly cleanup did (#5469), written by core/cleanup.py for
+    # Settings → System: {"at": ISO time, "secs": float, "removed": [[what, n]]}.
+    "cleanup.last_run": {},
 }
 
 # Settings encrypted at rest (transparent encrypt-on-write / decrypt-on-read).

steward/settings/routes.py

@@ -627,16 +627,20 @@ async def _reports_page(settings: dict, flash):
 @require_role(UserRole.admin)
 async def system():
     """Steward's own health (#5467): its process metrics now and over the day,
-    each one an alert rule can watch (source "Steward itself")."""
-    from steward.core.health import process_health
+    each one an alert rule can watch (source "Steward itself"); then its
+    tables' sizes and what the last hourly cleanup removed (#5469)."""
+    from steward.core.health import last_cleanup, process_health, table_sizes
     from steward.core.status import sparkline_svg
     async with current_app.db_sessionmaker() as db:
         health = await process_health(db)
+        # Table sizes and the last cleanup (#5469): Postgres's catalog and
+        # what the cleanup kept, never a scan of the tables themselves.
+        storage, cleanup = await table_sizes(db), await last_cleanup(db)
     for row in health["rows"]:
         # From zero to the row's own peak, with a floor (UI & Design ruling 8).
         row["spark"] = sparkline_svg(row["series"], width=120, height=20,
                                      stroke="currentColor", floor=1.0) if len(row["series"]) > 1 else ""
-    return await render_template("settings/system.html", **health)
+    return await render_template("settings/system.html", **health, storage=storage, cleanup=cleanup)
 
 
 @settings_bp.get("/reports/")

steward/templates/settings/system.html

@@ -38,4 +38,52 @@
 <p class="field-hint" style="margin-top:0.75rem;">
   Recorded each minute (the cleanup each hour). Any of these can be watched by an
   <a href="/alerts/rules/new">alert rule</a>: pick the source "Steward itself".</p>
+
+{# Storage (#5469): Postgres's own figures for each table, then what the
+   hourly cleanup last removed. Read from the catalog, never by counting. #}
+<h2 class="section-title" style="margin:2rem 0 0.5rem;">Storage</h2>
+<p class="field-hint" style="margin:0 0 1rem;">The database takes {{ storage.total|bytes }} on disk.</p>
+<div class="card-flush">
+  <table class="table">
+    <thead><tr><th>Table</th><th class="num">On disk</th><th class="num">Rows (estimate)</th></tr></thead>
+    <tbody>
+      {% for t in storage.tables %}
+      <tr>
+        <td class="cell-name"><span title="{{ t.name }}">{{ t.name }}</span></td>
+        <td class="num nowrap">{{ t.bytes|bytes }}</td>
+        <td class="num nowrap">{% if t.rows is not none %}{{ "{:,}".format(t.rows) }}{% else %}—{% endif %}</td>
+      </tr>
+      {% endfor %}
+      {% if storage.rest.count %}
+      <tr><td class="tone-dim">+{{ storage.rest.count }} smaller tables</td>
+        <td class="num nowrap tone-dim">{{ storage.rest.bytes|bytes }}</td><td></td></tr>
+      {% endif %}
+    </tbody>
+  </table>
+</div>
+
+<h2 class="section-title" style="margin:2rem 0 0.5rem;">Last cleanup</h2>
+{% if not cleanup %}
+<p class="tone-dim">None has finished since this version started: the cleanup runs each hour.</p>
+{% else %}
+<p style="margin:0 0 1rem;">Ran {{ cleanup.at|ago }} and took {{ cleanup.secs|duration }}.
+  {%- if not cleanup.removed %} Nothing was past its window ({{ cleanup.passes }} passes).{% endif %}</p>
+{% if cleanup.removed %}
+<div class="card-flush">
+  <table class="table">
+    <thead><tr><th>What went</th><th class="num">Rows</th></tr></thead>
+    <tbody>
+      {% for what, n in cleanup.removed %}
+      <tr><td class="cell-name"><span title="{{ what }}">{{ what }}</span></td>
+        <td class="num nowrap">{{ "{:,}".format(n) }}</td></tr>
+      {% endfor %}
+    </tbody>
+  </table>
+</div>
+{% endif %}
+{% endif %}
+<p class="field-hint" style="margin-top:0.75rem;">
+  How long each kind of data is kept is set under
+  <a href="/settings/general/">General</a> (data and audit log) and
+  <a href="/settings/thresholds/">Thresholds &amp; retention</a> (metrics, Docker and logs).</p>
 {% endblock %}

tests/core/test_steward_health.py

@@ -8,6 +8,16 @@ from quart import render_template, session
 from steward.core import health
 
 AT = datetime(2026, 10, 9, 12, 0, tzinfo=timezone.utc)
+STORAGE = {"total": 3 * 2**30, "rest": {"count": 0, "bytes": 0},
+           "tables": [{"name": "plugin_metrics", "bytes": 2 * 2**30, "rows": 41_000_000},
+                      {"name": "docker_logs", "bytes": 512 * 2**20, "rows": None}]}
+
+
+async def _page(app, ctx, storage=STORAGE, cleanup=None):
+    async with app.test_request_context("/settings/system/"):
+        session["user_role"] = "admin"
+        page = await render_template("settings/system.html", **ctx, storage=storage, cleanup=cleanup)
+    return " ".join(page.split())
 
 
 def _previews(**over):
@@ -52,10 +62,50 @@ async def test_the_page_reads_as_a_verdict_then_each_measure(app, previews):
     got = await health.process_health(None)
     for r in got["rows"]:
         r["spark"] = ""
-    async with app.test_request_context("/settings/system/"):
-        session["user_role"] = "admin"
-        page = " ".join((await render_template("settings/system.html", **got)).split())
+    page = await _page(app, got)
     assert '<span class="tone-warn">Steward wasn&#39;t watching for up to 7m' in page
     assert 'title="<' not in page          # a <time> element never lands inside an attribute
     assert "Database round trip" in page and "4 ms" in page and "2 ms – 9 ms" in page
     assert page.index("page-verdict") < page.index("Agent posts per minute")
+
+
+# ── Storage and the last cleanup (#5469) ─────────────────────────────────────
+
+async def test_tables_largest_first_then_what_the_cleanup_removed(app, previews):
+    previews()
+    got = await health.process_health(None)
+    for r in got["rows"]:
+        r["spark"] = ""
+    cleanup = {"at": AT, "secs": 12.5, "passes": 14,
+               "removed": [("plugin_metrics: raw samples rolled into hourly", 86_400)]}
+    page = await _page(app, got, cleanup=cleanup)
+    assert "The database takes 3.0 GiB on disk." in page
+    assert page.index(">plugin_metrics<") < page.index(">docker_logs<")
+    assert "41,000,000" in page and "512.0 MiB" in page
+    assert "took 12s" in page and "86,400" in page
+    assert page.index("Steward itself") < page.index("Storage") < page.index("Last cleanup")
+
+
+async def test_a_quiet_cleanup_and_one_not_yet_run_each_say_so(app, previews):
+    previews()
+    got = await health.process_health(None)
+    for r in got["rows"]:
+        r["spark"] = ""
+    quiet = await _page(app, got, cleanup={"at": AT, "secs": 1.0, "passes": 14, "removed": []})
+    assert "Nothing was past its window (14 passes)." in quiet
+    assert "None has finished since this version started" in await _page(app, got)
+
+
+async def test_the_last_cleanup_keeps_only_what_went(monkeypatch):
+    stored = {"at": "2026-10-09T12:00:00+00:00", "secs": 3.2,
+              "removed": [["audit_events: older than 365 days", 0], ["docker_logs: past the age window", 7]]}
+
+    async def fake(db, key):
+        assert key == "cleanup.last_run"
+        return stored
+    monkeypatch.setattr("steward.core.settings.get_setting", fake)
+    got = await health.last_cleanup(None)
+    assert got["at"] == AT and got["passes"] == 2
+    assert got["removed"] == [("docker_logs: past the age window", 7)]
+    stored = {}
+    assert await health.last_cleanup(None) is None

tests/integration/test_cleanup.py

@@ -22,7 +22,7 @@ def app():
 def test_the_audit_window_comes_from_settings(app):
     from sqlalchemy import text
     from steward.core.cleanup import run_cleanup
-    from steward.core.settings import set_setting
+    from steward.core.settings import get_setting, set_setting
     from steward.models.audit import AuditEvent
 
     now = datetime.now(timezone.utc)
@@ -35,16 +35,23 @@ def test_the_audit_window_comes_from_settings(app):
                 s.add(AuditEvent(action=tag, entity_id="old", timestamp=now - timedelta(days=40)))
                 s.add(AuditEvent(action=tag, entity_id="new", timestamp=now - timedelta(days=20)))
         try:
-            await run_cleanup(app)
+            done = await run_cleanup(app)
         finally:
             async with app.db_sessionmaker() as s:
                 async with s.begin():
                     await set_setting(s, "audit.retention_days", 365)
         async with app.db_sessionmaker() as s:
-            return [r[0] for r in (await s.execute(text(
+            kept = [r[0] for r in (await s.execute(text(
                 "SELECT entity_id FROM audit_events WHERE action = :a"), {"a": tag})).all()]
+            return kept, done, await get_setting(s, "cleanup.last_run")
 
-    assert asyncio.run(_go()) == ["new"]
+    kept, done, stored = asyncio.run(_go())
+    assert kept == ["new"]
+    # What it did is kept for Settings → System (#5469), each pass by name.
+    counts = dict(done["removed"])
+    assert counts["audit_events: older than 30 days"] >= 1
+    assert "plugin_metrics: raw samples rolled into hourly" in counts and done["secs"] >= 0
+    assert stored == done
 
 
 def test_the_chart_seam_follows_the_raw_metrics_setting(app):
@@ -77,3 +84,19 @@ def test_the_chart_seam_follows_the_raw_metrics_setting(app):
                     await set_setting(s, "metrics.retention.raw_days", 7)
 
     assert [v for _, v in asyncio.run(_go())] == [10.0]
+
+
+def test_table_sizes_come_from_the_catalog_largest_first(app):
+    """Settings → System's storage figures (#5469), read against real Postgres."""
+    from steward.core.health import TABLES_SHOWN, table_sizes
+
+    async def _go():
+        async with app.db_sessionmaker() as s:
+            return await table_sizes(s)
+
+    got = asyncio.run(_go())
+    sizes = [t["bytes"] for t in got["tables"]]
+    assert sizes == sorted(sizes, reverse=True) and len(sizes) <= TABLES_SHOWN
+    names = {t["name"] for t in got["tables"]}
+    assert got["rest"]["count"] or "audit_events" in names       # the schema's own tables
+    assert got["total"] >= sum(sizes) + got["rest"]["bytes"] > 0