diff --git a/steward/app.py b/steward/app.py
index e36784c..436f159 100644
--- a/steward/app.py
+++ b/steward/app.py
@@ -414,14 +414,12 @@ def _register_core_tasks(app: Quart) -> None:
await run_due_monitors(app)
async def run_cleanup():
- import time
from .core.cleanup import run_cleanup as _cleanup
from .core.self_monitor import record_cleanup_secs
- started = time.monotonic()
- await _cleanup(app)
+ done = await _cleanup(app)
# How long it took, as Steward's own metric (#5467); a failed run
# raises past this and records nothing.
- await record_cleanup_secs(app, time.monotonic() - started)
+ await record_cleanup_secs(app, done["secs"])
async def run_alert_reconcile():
from .alerts.service import reconcile_covering_rules
diff --git a/steward/core/cleanup.py b/steward/core/cleanup.py
index bafde0b..3640bbd 100644
--- a/steward/core/cleanup.py
+++ b/steward/core/cleanup.py
@@ -2,6 +2,7 @@ from __future__ import annotations
import asyncio
import logging
import os
+import time
from datetime import datetime, timedelta, timezone
from typing import TYPE_CHECKING
@@ -32,37 +33,66 @@ def prune_by_age(column) -> None:
_AGED.append(column)
-async def run_cleanup(app: "Quart") -> None:
+# What each rollup count means, as Settings → System lists it (#5469).
+_METRIC_COUNTS = [("raw_rows_rolled", "plugin_metrics: raw samples rolled into hourly"),
+ ("rollup_pruned", "plugin_metrics_hourly: past the rollup window"),
+ ("latest_pruned", "plugin_metric_latest: stopped reporting")]
+_DOCKER_COUNTS = [("raw_rows_rolled", "docker_metrics: raw samples rolled into hourly"),
+ ("rollup_pruned", "docker_metrics_hourly: past the rollup window"),
+ ("events_pruned", "docker_events: past the events window"),
+ ("logs_age_pruned", "docker_logs: past the age window"),
+ ("logs_size_pruned", "docker_logs: over a container's size cap")]
+
+
+async def run_cleanup(app: "Quart") -> dict:
"""Delete rows older than their retention window from every growing table,
and run logs older than the data window, then run the metric and Docker
- rollups. Windows are read from Settings each run (rule 25: no restart)."""
- from steward.core.settings import get_setting
+ rollups. Windows are read from Settings each run (rule 25: no restart).
+
+ Returns what it did, {"at", "secs", "removed": [[what, count]]}, and keeps
+ the same as the cleanup.last_run setting for Settings → System (#5469). A
+ run that raises records nothing, so the page keeps the last one that
+ finished."""
+ from steward.core.settings import get_setting, set_setting
from steward.ansible.executor import ARTIFACT_DIR
now = datetime.now(timezone.utc)
+ started = time.monotonic()
+ removed: list[list] = []
async with app.db_sessionmaker() as session:
async with session.begin():
data_days = int(await get_setting(session, "data.retention_days") or 90)
audit_days = int(await get_setting(session, "audit.retention_days") or 365)
cutoff = now - timedelta(days=data_days)
- aged = [(c, cutoff) for c in _AGED]
- aged.append((AuditEvent.timestamp, now - timedelta(days=audit_days)))
- for column, before in aged:
+ aged = [(c, cutoff, data_days) for c in _AGED]
+ aged.append((AuditEvent.timestamp, now - timedelta(days=audit_days), audit_days))
+ for column, before, days in aged:
result = await session.execute(
delete(column.class_).where(column < before)
)
+ table = column.class_.__tablename__
+ removed.append([f"{table}: older than {days} days", result.rowcount or 0])
if result.rowcount:
- logger.info("Pruned %d rows from %s", result.rowcount,
- column.class_.__tablename__)
+ logger.info("Pruned %d rows from %s", result.rowcount, table)
- removed = await asyncio.to_thread(prune_run_logs, ARTIFACT_DIR, cutoff)
- if removed:
- logger.info("Pruned %d Ansible run log files from %s", removed, ARTIFACT_DIR)
+ files = await asyncio.to_thread(prune_run_logs, ARTIFACT_DIR, cutoff)
+ removed.append([f"Ansible run log files: older than {data_days} days", files])
+ if files:
+ logger.info("Pruned %d Ansible run log files from %s", files, ARTIFACT_DIR)
# plugin_metrics is NOT blanket-deleted here — it's rolled up to hourly
# then pruned, so multi-week host history stays cheap.
- await _run_metrics_retention(session, now)
- await _run_docker_retention(session, now)
+ counts = await _run_metrics_retention(session, now)
+ removed += [[what, counts.get(k) or 0] for k, what in _METRIC_COUNTS]
+ counts = await _run_docker_retention(session, now)
+ if counts is not None:
+ removed += [[what, counts.get(k) or 0] for k, what in _DOCKER_COUNTS]
+
+ summary = {"at": now.isoformat(), "secs": round(time.monotonic() - started, 2),
+ "removed": removed}
+ async with session.begin():
+ await set_setting(session, "cleanup.last_run", summary)
+ return summary
def prune_run_logs(directory: str, before: datetime) -> int:
@@ -85,7 +115,7 @@ def prune_run_logs(directory: str, before: datetime) -> int:
return removed
-async def _run_metrics_retention(session, now: datetime) -> None:
+async def _run_metrics_retention(session, now: datetime) -> dict:
"""Roll up + prune plugin_metrics (raw → hourly → gone). Windows read fresh
from settings each run (rule 25 — UI change takes effect next cleanup, no
restart). get_setting's SELECT autobegins, so read inside the begin block."""
@@ -100,9 +130,10 @@ async def _run_metrics_retention(session, now: datetime) -> None:
)
if counts and any(counts.values()):
logger.info("Metrics retention: %s", counts)
+ return counts or {}
-async def _run_docker_retention(session, now: datetime) -> None:
+async def _run_docker_retention(session, now: datetime) -> dict | None:
"""Drive the docker plugin's rollup + prune and its log ring via its
capabilities, if loaded.
@@ -114,7 +145,7 @@ async def _run_docker_retention(session, now: datetime) -> None:
"""
from steward.core.capabilities import has_capability, invoke_capability
if not has_capability("docker.run_retention"):
- return
+ return None
from steward.core.settings import get_setting
from steward.models.users import UserRole
@@ -139,3 +170,4 @@ async def _run_docker_retention(session, now: datetime) -> None:
)
if counts and any(counts.values()):
logger.info("Docker retention: %s", counts)
+ return counts
diff --git a/steward/core/health.py b/steward/core/health.py
index d5acc0b..659f221 100644
--- a/steward/core/health.py
+++ b/steward/core/health.py
@@ -67,3 +67,42 @@ async def process_health(db) -> dict:
if (by["open_fds_pct"]["now"] or 0) >= FD_WARN_PCT:
problems.append(f"open files at {by['open_fds_pct']['now']:.0f}% of the limit")
return {"rows": rows, "problems": problems, "seen": any(r["at"] for r in rows)}
+
+
+# How many tables Settings → System lists by name; the rest are summed.
+TABLES_SHOWN = 15
+
+
+async def table_sizes(db) -> dict:
+ """Steward's tables by size on disk, largest first (#5469), from Postgres's
+ own catalog: pg_total_relation_size (indexes and TOAST included) and the
+ planner's row estimate (reltuples; None for a table never analysed).
+
+ Returns {"total": bytes of the whole database, "tables": [{name, bytes,
+ rows}] for the TABLES_SHOWN largest, "rest": {count, bytes} of the others}."""
+ from sqlalchemy import text
+ rows = (await db.execute(text(
+ "SELECT c.relname, pg_total_relation_size(c.oid), c.reltuples::bigint "
+ "FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace "
+ "WHERE c.relkind IN ('r', 'p') AND n.nspname = current_schema() "
+ "ORDER BY 2 DESC"))).all()
+ total = (await db.execute(text("SELECT pg_database_size(current_database())"))).scalar()
+ tables = [{"name": name, "bytes": size, "rows": est if est is not None and est >= 0 else None}
+ for name, size, est in rows]
+ rest = tables[TABLES_SHOWN:]
+ return {"total": total, "tables": tables[:TABLES_SHOWN],
+ "rest": {"count": len(rest), "bytes": sum(t["bytes"] for t in rest)}}
+
+
+async def last_cleanup(db) -> dict | None:
+ """The last hourly cleanup that finished (#5469), as core/cleanup.py kept
+ it: {"at": datetime, "secs", "removed": [(what, n)] with n > 0, "passes":
+ how many passes ran}. None before the first one finishes."""
+ from datetime import datetime
+ from steward.core.settings import get_setting
+ run = await get_setting(db, "cleanup.last_run") or {}
+ if not run.get("at"):
+ return None
+ removed = [(what, n) for what, n in run.get("removed") or []]
+ return {"at": datetime.fromisoformat(run["at"]), "secs": run.get("secs"),
+ "removed": [(what, n) for what, n in removed if n], "passes": len(removed)}
diff --git a/steward/core/settings.py b/steward/core/settings.py
index 8715b8c..97e3879 100644
--- a/steward/core/settings.py
+++ b/steward/core/settings.py
@@ -175,6 +175,9 @@ DEFAULTS: dict[str, Any] = {
# so "Sunday 08:00" means 08:00 where they are (#5378).
"reports.schedule_tz": "",
"reports.last_sent_at": "",
+ # What the last hourly cleanup did (#5469), written by core/cleanup.py for
+ # Settings → System: {"at": ISO time, "secs": float, "removed": [[what, n]]}.
+ "cleanup.last_run": {},
}
# Settings encrypted at rest (transparent encrypt-on-write / decrypt-on-read).
diff --git a/steward/settings/routes.py b/steward/settings/routes.py
index e2f5d09..19c2b33 100644
--- a/steward/settings/routes.py
+++ b/steward/settings/routes.py
@@ -627,16 +627,20 @@ async def _reports_page(settings: dict, flash):
@require_role(UserRole.admin)
async def system():
"""Steward's own health (#5467): its process metrics now and over the day,
- each one an alert rule can watch (source "Steward itself")."""
- from steward.core.health import process_health
+ each one an alert rule can watch (source "Steward itself"); then its
+ tables' sizes and what the last hourly cleanup removed (#5469)."""
+ from steward.core.health import last_cleanup, process_health, table_sizes
from steward.core.status import sparkline_svg
async with current_app.db_sessionmaker() as db:
health = await process_health(db)
+ # Table sizes and the last cleanup (#5469): Postgres's catalog and
+ # what the cleanup kept, never a scan of the tables themselves.
+ storage, cleanup = await table_sizes(db), await last_cleanup(db)
for row in health["rows"]:
# From zero to the row's own peak, with a floor (UI & Design ruling 8).
row["spark"] = sparkline_svg(row["series"], width=120, height=20,
stroke="currentColor", floor=1.0) if len(row["series"]) > 1 else ""
- return await render_template("settings/system.html", **health)
+ return await render_template("settings/system.html", **health, storage=storage, cleanup=cleanup)
@settings_bp.get("/reports/")
diff --git a/steward/templates/settings/system.html b/steward/templates/settings/system.html
index c5f0543..d7db32c 100644
--- a/steward/templates/settings/system.html
+++ b/steward/templates/settings/system.html
@@ -38,4 +38,52 @@
Recorded each minute (the cleanup each hour). Any of these can be watched by an
alert rule: pick the source "Steward itself".
+
+{# Storage (#5469): Postgres's own figures for each table, then what the
+ hourly cleanup last removed. Read from the catalog, never by counting. #}
+
Storage
+
The database takes {{ storage.total|bytes }} on disk.
+
+
+
Table
On disk
Rows (estimate)
+
+ {% for t in storage.tables %}
+
+
{{ t.name }}
+
{{ t.bytes|bytes }}
+
{% if t.rows is not none %}{{ "{:,}".format(t.rows) }}{% else %}—{% endif %}
+
+ {% endfor %}
+ {% if storage.rest.count %}
+
+{{ storage.rest.count }} smaller tables
+
{{ storage.rest.bytes|bytes }}
+ {% endif %}
+
+
+
+
+
Last cleanup
+{% if not cleanup %}
+
None has finished since this version started: the cleanup runs each hour.
+{% else %}
+
Ran {{ cleanup.at|ago }} and took {{ cleanup.secs|duration }}.
+ {%- if not cleanup.removed %} Nothing was past its window ({{ cleanup.passes }} passes).{% endif %}
+{% if cleanup.removed %}
+
+
+
What went
Rows
+
+ {% for what, n in cleanup.removed %}
+
{{ what }}
+
{{ "{:,}".format(n) }}
+ {% endfor %}
+
+
+
+{% endif %}
+{% endif %}
+
+ How long each kind of data is kept is set under
+ General (data and audit log) and
+ Thresholds & retention (metrics, Docker and logs).
{% endblock %}
diff --git a/tests/core/test_steward_health.py b/tests/core/test_steward_health.py
index 324d4d5..5ed9673 100644
--- a/tests/core/test_steward_health.py
+++ b/tests/core/test_steward_health.py
@@ -8,6 +8,16 @@ from quart import render_template, session
from steward.core import health
AT = datetime(2026, 10, 9, 12, 0, tzinfo=timezone.utc)
+STORAGE = {"total": 3 * 2**30, "rest": {"count": 0, "bytes": 0},
+ "tables": [{"name": "plugin_metrics", "bytes": 2 * 2**30, "rows": 41_000_000},
+ {"name": "docker_logs", "bytes": 512 * 2**20, "rows": None}]}
+
+
+async def _page(app, ctx, storage=STORAGE, cleanup=None):
+ async with app.test_request_context("/settings/system/"):
+ session["user_role"] = "admin"
+ page = await render_template("settings/system.html", **ctx, storage=storage, cleanup=cleanup)
+ return " ".join(page.split())
def _previews(**over):
@@ -52,10 +62,50 @@ async def test_the_page_reads_as_a_verdict_then_each_measure(app, previews):
got = await health.process_health(None)
for r in got["rows"]:
r["spark"] = ""
- async with app.test_request_context("/settings/system/"):
- session["user_role"] = "admin"
- page = " ".join((await render_template("settings/system.html", **got)).split())
+ page = await _page(app, got)
assert 'Steward wasn't watching for up to 7m' in page
assert 'title="<' not in page # a