bvandeusenandClaude Opus 5.5
0b2fbf13fc
#5469 "Settings → System: table sizes and what the last cleanup removed". - Storage: each table's size on disk (pg_total_relation_size, indexes and TOAST included) and Postgres's row estimate (reltuples), largest first, the 15 largest by name and the rest summed, under the database's total size. Read from the catalog only, never by counting rows. - Last cleanup: when it ran, how long it took, and what each pass removed (each aged table, run log files, the metric and Docker rollups, the log ring), or "Nothing was past its window". run_cleanup now returns that summary and keeps it as the cleanup.last_run setting, as reports.last_sent_at does; cleanup_secs (#5467) is recorded from the same figure. - A hint links to where each retention window is set. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
112 lines
4.7 KiB
Python
112 lines
4.7 KiB
Python
"""Settings → System (#5467): Steward's own health, a verdict then each measure
|
||
now and over the day. The verdict names what is wrong, and only that."""
|
||
from datetime import datetime, timezone
|
||
|
||
import pytest
|
||
from quart import render_template, session
|
||
|
||
from steward.core import health
|
||
|
||
AT = datetime(2026, 10, 9, 12, 0, tzinfo=timezone.utc)
|
||
STORAGE = {"total": 3 * 2**30, "rest": {"count": 0, "bytes": 0},
|
||
"tables": [{"name": "plugin_metrics", "bytes": 2 * 2**30, "rows": 41_000_000},
|
||
{"name": "docker_logs", "bytes": 512 * 2**20, "rows": None}]}
|
||
|
||
|
||
async def _page(app, ctx, storage=STORAGE, cleanup=None):
|
||
async with app.test_request_context("/settings/system/"):
|
||
session["user_role"] = "admin"
|
||
page = await render_template("settings/system.html", **ctx, storage=storage, cleanup=cleanup)
|
||
return " ".join(page.split())
|
||
|
||
|
||
def _previews(**over):
|
||
base = {m: {"now": 0.0, "at": AT, "series": [0.0, 0.0], "low": 0.0, "high": 0.0}
|
||
for m in health.PROCESS_METRICS}
|
||
base["db_roundtrip_ms"] = {"now": 4.0, "at": AT, "series": [3.0, 5.0], "low": 2.0, "high": 9.0}
|
||
for m, v in over.items():
|
||
base[m] = {**base[m], **v}
|
||
return base
|
||
|
||
|
||
@pytest.fixture
|
||
def previews(monkeypatch):
|
||
def use(**over):
|
||
got = _previews(**over)
|
||
|
||
async def fake(db, source, resource, metric, now=None):
|
||
assert (source, resource) == ("steward", "process")
|
||
return got[metric]
|
||
monkeypatch.setattr("steward.alerts.service.metric_preview", fake)
|
||
return use
|
||
|
||
|
||
async def test_a_healthy_steward_has_no_problems(previews):
|
||
previews()
|
||
got = await health.process_health(None)
|
||
assert got["seen"] and got["problems"] == []
|
||
assert [r["metric"] for r in got["rows"]] == health.PROCESS_METRICS
|
||
|
||
|
||
async def test_failures_and_gaps_are_named(previews):
|
||
previews(ingest_failed={"now": 2.0, "high": 2.0}, unwatched_secs={"high": 420.0},
|
||
open_fds_pct={"now": 85.0})
|
||
problems = (await health.process_health(None))["problems"]
|
||
assert problems[0] == "2 agent posts failed in the last minute"
|
||
assert "wasn't watching for up to 7m" in problems[1]
|
||
assert problems[2] == "open files at 85% of the limit"
|
||
|
||
|
||
async def test_the_page_reads_as_a_verdict_then_each_measure(app, previews):
|
||
previews(unwatched_secs={"high": 420.0})
|
||
got = await health.process_health(None)
|
||
for r in got["rows"]:
|
||
r["spark"] = ""
|
||
page = await _page(app, got)
|
||
assert '<span class="tone-warn">Steward wasn't watching for up to 7m' in page
|
||
assert 'title="<' not in page # a <time> element never lands inside an attribute
|
||
assert "Database round trip" in page and "4 ms" in page and "2 ms – 9 ms" in page
|
||
assert page.index("page-verdict") < page.index("Agent posts per minute")
|
||
|
||
|
||
# ── Storage and the last cleanup (#5469) ─────────────────────────────────────
|
||
|
||
async def test_tables_largest_first_then_what_the_cleanup_removed(app, previews):
|
||
previews()
|
||
got = await health.process_health(None)
|
||
for r in got["rows"]:
|
||
r["spark"] = ""
|
||
cleanup = {"at": AT, "secs": 12.5, "passes": 14,
|
||
"removed": [("plugin_metrics: raw samples rolled into hourly", 86_400)]}
|
||
page = await _page(app, got, cleanup=cleanup)
|
||
assert "The database takes 3.0 GiB on disk." in page
|
||
assert page.index(">plugin_metrics<") < page.index(">docker_logs<")
|
||
assert "41,000,000" in page and "512.0 MiB" in page
|
||
assert "took 12s" in page and "86,400" in page
|
||
assert page.index("Steward itself") < page.index("Storage") < page.index("Last cleanup")
|
||
|
||
|
||
async def test_a_quiet_cleanup_and_one_not_yet_run_each_say_so(app, previews):
|
||
previews()
|
||
got = await health.process_health(None)
|
||
for r in got["rows"]:
|
||
r["spark"] = ""
|
||
quiet = await _page(app, got, cleanup={"at": AT, "secs": 1.0, "passes": 14, "removed": []})
|
||
assert "Nothing was past its window (14 passes)." in quiet
|
||
assert "None has finished since this version started" in await _page(app, got)
|
||
|
||
|
||
async def test_the_last_cleanup_keeps_only_what_went(monkeypatch):
|
||
stored = {"at": "2026-10-09T12:00:00+00:00", "secs": 3.2,
|
||
"removed": [["audit_events: older than 365 days", 0], ["docker_logs: past the age window", 7]]}
|
||
|
||
async def fake(db, key):
|
||
assert key == "cleanup.last_run"
|
||
return stored
|
||
monkeypatch.setattr("steward.core.settings.get_setting", fake)
|
||
got = await health.last_cleanup(None)
|
||
assert got["at"] == AT and got["passes"] == 2
|
||
assert got["removed"] == [("docker_logs: past the age window", 7)]
|
||
stored = {}
|
||
assert await health.last_cleanup(None) is None
|