feat(retention): every growing table is pruned; the audit log has its own window; charts read the raw window from Settings (#5155)

- prune_by_age(column) registers a time-series column with the hourly cleanup;
  Traefik (metrics, access summaries, global stats) and UniFi (WAN, client,
  WLAN, DPI, events, alarms, speedtests) register theirs in setup().
- Alert events join monitor results and Ansible runs on data.retention_days,
  read from Settings each run; Ansible run log files older than it are deleted.
- audit.retention_days (365) keeps the audit log longer, set on Settings > General.
- metric_series.points reads metrics.retention.raw_days, so the chart's
  raw/rollup seam is where the rollup put it (RAW_DAYS removed).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

plugins/traefik/__init__.py

@@ -12,6 +12,11 @@ def setup(app: "Quart") -> None:
     global _app
     _app = app
     from .models import TraefikMetric, TraefikCert, TraefikGlobalStat, TraefikAccessSummary  # noqa: F401
+    from steward.core.cleanup import prune_by_age
+    # Samples older than data.retention_days are pruned by the cleanup (#5155).
+    prune_by_age(TraefikMetric.scraped_at)
+    prune_by_age(TraefikGlobalStat.scraped_at)
+    prune_by_age(TraefikAccessSummary.period_end)
 
 
 def get_scheduled_tasks() -> list:

plugins/unifi/__init__.py

@@ -16,6 +16,14 @@ def setup(app: "Quart") -> None:
         UnifiDevice, UnifiKnownClient, UnifiEvent, UnifiAlarm,
         UnifiSpeedtestResult, UnifiNetwork, UnifiPortForward, UnifiFwRule,
     )
+    from steward.core.cleanup import prune_by_age
+    # Samples and history older than data.retention_days are pruned by the
+    # cleanup (#5155). Devices, known clients and config tables are current state.
+    for column in (UnifiWanStat.scraped_at, UnifiClientSnapshot.scraped_at,
+                   UnifiWlanStat.scraped_at, UnifiDpiSnapshot.scraped_at,
+                   UnifiEvent.event_time, UnifiAlarm.alarm_time,
+                   UnifiSpeedtestResult.run_at):
+        prune_by_age(column)
 
 
 def get_scheduled_tasks() -> list:

steward/core/cleanup.py

@@ -1,12 +1,16 @@
 from __future__ import annotations
+import asyncio
 import logging
+import os
 from datetime import datetime, timedelta, timezone
 from typing import TYPE_CHECKING
 
 from sqlalchemy import delete
 
-from steward.models.monitors import MonitorResult
+from steward.models.alerts import AlertEvent
 from steward.models.ansible import AnsibleRun
+from steward.models.audit import AuditEvent
+from steward.models.monitors import MonitorResult
 
 if TYPE_CHECKING:
     from quart import Quart
@@ -14,24 +18,46 @@ if TYPE_CHECKING:
 logger = logging.getLogger(__name__)
 
 
-async def run_cleanup(app: "Quart") -> None:
-    """Delete rows older than DATA_RETENTION_DAYS from time-series tables, then
-    run Docker-specific rollup + retention (delegated to the docker plugin)."""
-    retention_days: int = app.config.get("DATA_RETENTION_DAYS", 90)
-    now = datetime.now(timezone.utc)
-    cutoff = now - timedelta(days=retention_days)
+# Time-series columns pruned at data.retention_days (#5155). A plugin adds its
+# own with prune_by_age() in setup(), so the core never imports plugin models.
+# Tables kept another way are not listed: plugin_metrics and the docker tables
+# are rolled up (below), and the audit log has its own, longer window.
+_AGED: list = [MonitorResult.checked_at, AnsibleRun.started_at, AlertEvent.transitioned_at]
 
+
+def prune_by_age(column) -> None:
+    """Have the hourly cleanup delete rows whose `column` (a mapped DateTime
+    attribute, e.g. TraefikMetric.scraped_at) is older than data.retention_days."""
+    if not any(c is column for c in _AGED):
+        _AGED.append(column)
+
+
+async def run_cleanup(app: "Quart") -> None:
+    """Delete rows older than their retention window from every growing table,
+    and run logs older than the data window, then run the metric and Docker
+    rollups. Windows are read from Settings each run (rule 25: no restart)."""
+    from steward.core.settings import get_setting
+    from steward.ansible.executor import ARTIFACT_DIR
+
+    now = datetime.now(timezone.utc)
     async with app.db_sessionmaker() as session:
         async with session.begin():
-            for model, ts_col in [
-                (MonitorResult, MonitorResult.checked_at),
-                (AnsibleRun, AnsibleRun.started_at),
-            ]:
+            data_days = int(await get_setting(session, "data.retention_days") or 90)
+            audit_days = int(await get_setting(session, "audit.retention_days") or 365)
+            cutoff = now - timedelta(days=data_days)
+            aged = [(c, cutoff) for c in _AGED]
+            aged.append((AuditEvent.timestamp, now - timedelta(days=audit_days)))
+            for column, before in aged:
                 result = await session.execute(
-                    delete(model).where(ts_col < cutoff)
+                    delete(column.class_).where(column < before)
                 )
                 if result.rowcount:
-                    logger.info(f"Pruned {result.rowcount} rows from {model.__tablename__}")
+                    logger.info("Pruned %d rows from %s", result.rowcount,
+                                column.class_.__tablename__)
+
+        removed = await asyncio.to_thread(prune_run_logs, ARTIFACT_DIR, cutoff)
+        if removed:
+            logger.info("Pruned %d Ansible run log files from %s", removed, ARTIFACT_DIR)
 
         # plugin_metrics is NOT blanket-deleted here — it's rolled up to hourly
         # then pruned, so multi-week host history stays cheap.
@@ -39,6 +65,26 @@ async def run_cleanup(app: "Quart") -> None:
         await _run_docker_retention(session, now)
 
 
+def prune_run_logs(directory: str, before: datetime) -> int:
+    """Delete the run log files in `directory` last written before `before`,
+    the same window as the runs they belong to. Returns how many went."""
+    removed = 0
+    try:
+        entries = list(os.scandir(directory))
+    except OSError:          # no run has written one yet, or not readable
+        return 0
+    limit = before.timestamp()
+    for entry in entries:
+        try:
+            if (entry.name.endswith(".log") and entry.is_file()
+                    and entry.stat().st_mtime < limit):
+                os.unlink(entry.path)
+                removed += 1
+        except OSError:
+            logger.warning("Could not prune run log %s", entry.path)
+    return removed
+
+
 async def _run_metrics_retention(session, now: datetime) -> None:
     """Roll up + prune plugin_metrics (raw → hourly → gone). Windows read fresh
     from settings each run (rule 25 — UI change takes effect next cleanup, no

steward/core/metric_series.py

@@ -13,10 +13,6 @@ from sqlalchemy import func, or_, select, text
 from steward.core.time_range import bucket_seconds
 from steward.models.metrics import PluginMetric, PluginMetricHourly
 
-# Raw rows older than this are rolled into plugin_metrics_hourly and deleted
-# (see core/metrics_retention.py), so a range reaching past it reads the rollup.
-RAW_DAYS = 7
-
 # Loose index scans: step through the distinct leading values of the
 # (source_module, resource_name, …) index one probe at a time instead of
 # scanning every sample. Postgres has no native skip scan before 18.
@@ -79,9 +75,14 @@ async def points(db, source_module: str, resource: str, metric: str, since: date
     The part of the range older than the raw window comes from the hourly
     rollup; the rest is bucket-averaged in SQL (buckets no wider than an hour
     when both are present, so the series is evenly grained across the seam).
+    The raw window is metrics.retention.raw_days, the setting the rollup in
+    core/metrics_retention.py reads, so the seam is where the rollup put it.
     """
+    from steward.core.settings import get_setting
+
+    raw_days = int(await get_setting(db, "metrics.retention.raw_days") or 7)
     now = datetime.now(timezone.utc)
-    raw_cutoff = (now - timedelta(days=RAW_DAYS)).replace(minute=0, second=0, microsecond=0)
+    raw_cutoff = (now - timedelta(days=raw_days)).replace(minute=0, second=0, microsecond=0)
     out: list[tuple[datetime, float]] = []
 
     if since < raw_cutoff:

steward/core/settings.py

@@ -47,7 +47,11 @@ DEFAULTS: dict[str, Any] = {
     # Failed logins from one address before it is refused for the window.
     "login.max_failures": 5,
     "login.lockout_minutes": 15,
+    # Every growing time-series table (monitor results, Ansible runs and their
+    # log files, alert history, Traefik and UniFi samples) keeps this many days.
     "data.retention_days": 90,
+    # The audit log is a security record: kept longer than operational data.
+    "audit.retention_days": 365,
     "monitors.poll_interval_seconds": 60,
     "smtp.host": "",
     "smtp.port": 587,

steward/settings/routes.py

@@ -101,7 +101,7 @@ async def save_general():
     form = await request.form
     whole: dict[str, int] = {}
     for key in ("session.lifetime_hours", "general.trusted_proxy_hops",
-                "login.max_failures", "login.lockout_minutes"):
+                "login.max_failures", "login.lockout_minutes", "audit.retention_days"):
         try:
             whole[key] = int(form.get(key, DEFAULTS[key]))
         except (TypeError, ValueError):

steward/templates/settings/general.html

@@ -58,7 +58,14 @@
     <label>Data retention <span style="color:var(--text-muted);font-size:0.8rem;">(days)</span></label>
     <input type="number" name="data.retention_days" min="1"
            value="{{ settings['data.retention_days'] }}" style="max-width:120px;">
-    <div style="font-size:0.78rem;color:var(--text-muted);margin-top:0.25rem;">Old ping/DNS results older than this are pruned by the cleanup job.</div>
+    <div style="font-size:0.78rem;color:var(--text-muted);margin-top:0.25rem;">Monitor results, Ansible runs and their logs, alert history, and Traefik and UniFi samples older than this are pruned by the hourly cleanup. Host metrics, Docker data and container logs have their own windows under Settings → Thresholds.</div>
+  </div>
+
+  <div class="form-group">
+    <label>Audit log retention <span style="color:var(--text-muted);font-size:0.8rem;">(days)</span></label>
+    <input type="number" name="audit.retention_days" min="1"
+           value="{{ settings['audit.retention_days'] }}" style="max-width:120px;">
+    <div style="font-size:0.78rem;color:var(--text-muted);margin-top:0.25rem;">Who changed what. Kept longer than operational data; entries older than this are pruned by the hourly cleanup.</div>
   </div>
 
   <div class="form-group">

tests/core/test_cleanup.py

@@ -0,0 +1,44 @@
+"""Unit: what the hourly cleanup prunes (#5155)."""
+import os
+from datetime import datetime, timedelta, timezone
+
+from steward.core import cleanup
+
+
+def test_run_logs_older_than_the_window_are_pruned(tmp_path):
+    now = datetime.now(timezone.utc)
+    old, new, other = tmp_path / "old.log", tmp_path / "new.log", tmp_path / "notes.txt"
+    for f in (old, new, other):
+        f.write_text("x")
+    aged = (now - timedelta(days=100)).timestamp()
+    os.utime(old, (aged, aged))
+    os.utime(other, (aged, aged))
+
+    assert cleanup.prune_run_logs(str(tmp_path), now - timedelta(days=90)) == 1
+    assert sorted(p.name for p in tmp_path.iterdir()) == ["new.log", "notes.txt"]
+
+
+def test_no_run_log_directory_prunes_nothing(tmp_path):
+    assert cleanup.prune_run_logs(str(tmp_path / "missing"), datetime.now(timezone.utc)) == 0
+
+
+def test_core_time_series_are_on_the_data_window():
+    from steward.models.alerts import AlertEvent
+    from steward.models.ansible import AnsibleRun
+    from steward.models.audit import AuditEvent
+    from steward.models.monitors import MonitorResult
+
+    tables = {c.class_.__tablename__ for c in cleanup._AGED}
+    assert {MonitorResult.__tablename__, AnsibleRun.__tablename__,
+            AlertEvent.__tablename__} <= tables
+    # The audit log has its own, longer window, not the data one.
+    assert AuditEvent.__tablename__ not in tables
+
+
+def test_a_column_registers_once(monkeypatch):
+    from steward.models.monitors import MonitorResult
+
+    monkeypatch.setattr(cleanup, "_AGED", [])
+    cleanup.prune_by_age(MonitorResult.checked_at)
+    cleanup.prune_by_age(MonitorResult.checked_at)   # a plugin set up twice
+    assert len(cleanup._AGED) == 1

tests/integration/test_cleanup.py

@@ -0,0 +1,79 @@
+"""Integration: retention windows read from Settings reach the pruning (#5155)."""
+from __future__ import annotations
+
+import asyncio
+import os
+import uuid
+from datetime import datetime, timedelta, timezone
+
+import pytest
+
+pytestmark = pytest.mark.integration
+
+
+@pytest.fixture
+def app():
+    if not os.environ.get("STEWARD_DATABASE_URL"):
+        pytest.skip("needs Postgres")
+    from steward.app import create_app
+    return create_app(testing=False)
+
+
+def test_the_audit_window_comes_from_settings(app):
+    from sqlalchemy import text
+    from steward.core.cleanup import run_cleanup
+    from steward.core.settings import set_setting
+    from steward.models.audit import AuditEvent
+
+    now = datetime.now(timezone.utc)
+    tag = f"test.cleanup.{uuid.uuid4().hex[:8]}"
+
+    async def _go():
+        async with app.db_sessionmaker() as s:
+            async with s.begin():
+                await set_setting(s, "audit.retention_days", 30)
+                s.add(AuditEvent(action=tag, entity_id="old", timestamp=now - timedelta(days=40)))
+                s.add(AuditEvent(action=tag, entity_id="new", timestamp=now - timedelta(days=20)))
+        try:
+            await run_cleanup(app)
+        finally:
+            async with app.db_sessionmaker() as s:
+                async with s.begin():
+                    await set_setting(s, "audit.retention_days", 365)
+        async with app.db_sessionmaker() as s:
+            return [r[0] for r in (await s.execute(text(
+                "SELECT entity_id FROM audit_events WHERE action = :a"), {"a": tag})).all()]
+
+    assert asyncio.run(_go()) == ["new"]
+
+
+def test_the_chart_seam_follows_the_raw_metrics_setting(app):
+    # With raw_days 1, a range two days back reads the hourly rollup there and
+    # not the raw sample beside it; at the default 7 it would read the raw one.
+    from steward.core import metric_series
+    from steward.core.settings import set_setting
+    from steward.models.metrics import PluginMetric, PluginMetricHourly
+
+    now = datetime.now(timezone.utc)
+    two_days = (now - timedelta(days=2)).replace(minute=0, second=0, microsecond=0)
+    res = f"seam-{uuid.uuid4().hex[:8]}"
+
+    async def _go():
+        async with app.db_sessionmaker() as s:
+            async with s.begin():
+                await set_setting(s, "metrics.retention.raw_days", 1)
+                s.add(PluginMetricHourly(source_module="test", resource_name=res,
+                                         metric_name="cpu_pct", bucket=two_days,
+                                         value_avg=10.0, value_max=10.0, sample_count=1))
+                s.add(PluginMetric(source_module="test", resource_name=res,
+                                   metric_name="cpu_pct", value=99.0, recorded_at=two_days))
+        try:
+            async with app.db_sessionmaker() as s:
+                return await metric_series.points(s, "test", res, "cpu_pct",
+                                                  now - timedelta(days=3))
+        finally:
+            async with app.db_sessionmaker() as s:
+                async with s.begin():
+                    await set_setting(s, "metrics.retention.raw_days", 7)
+
+    assert [v for _, v in asyncio.run(_go())] == [10.0]