feat(retention): every growing table is pruned; the audit log has its own window; charts read the raw window from Settings (#5155)

- prune_by_age(column) registers a time-series column with the hourly cleanup;
  Traefik (metrics, access summaries, global stats) and UniFi (WAN, client,
  WLAN, DPI, events, alarms, speedtests) register theirs in setup().
- Alert events join monitor results and Ansible runs on data.retention_days,
  read from Settings each run; Ansible run log files older than it are deleted.
- audit.retention_days (365) keeps the audit log longer, set on Settings > General.
- metric_series.points reads metrics.retention.raw_days, so the chart's
  raw/rollup seam is where the rollup put it (RAW_DAYS removed).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

steward/core/cleanup.py

@@ -1,12 +1,16 @@
 from __future__ import annotations
+import asyncio
 import logging
+import os
 from datetime import datetime, timedelta, timezone
 from typing import TYPE_CHECKING
 
 from sqlalchemy import delete
 
-from steward.models.monitors import MonitorResult
+from steward.models.alerts import AlertEvent
 from steward.models.ansible import AnsibleRun
+from steward.models.audit import AuditEvent
+from steward.models.monitors import MonitorResult
 
 if TYPE_CHECKING:
     from quart import Quart
@@ -14,24 +18,46 @@ if TYPE_CHECKING:
 logger = logging.getLogger(__name__)
 
 
-async def run_cleanup(app: "Quart") -> None:
-    """Delete rows older than DATA_RETENTION_DAYS from time-series tables, then
-    run Docker-specific rollup + retention (delegated to the docker plugin)."""
-    retention_days: int = app.config.get("DATA_RETENTION_DAYS", 90)
-    now = datetime.now(timezone.utc)
-    cutoff = now - timedelta(days=retention_days)
+# Time-series columns pruned at data.retention_days (#5155). A plugin adds its
+# own with prune_by_age() in setup(), so the core never imports plugin models.
+# Tables kept another way are not listed: plugin_metrics and the docker tables
+# are rolled up (below), and the audit log has its own, longer window.
+_AGED: list = [MonitorResult.checked_at, AnsibleRun.started_at, AlertEvent.transitioned_at]
 
+
+def prune_by_age(column) -> None:
+    """Have the hourly cleanup delete rows whose `column` (a mapped DateTime
+    attribute, e.g. TraefikMetric.scraped_at) is older than data.retention_days."""
+    if not any(c is column for c in _AGED):
+        _AGED.append(column)
+
+
+async def run_cleanup(app: "Quart") -> None:
+    """Delete rows older than their retention window from every growing table,
+    and run logs older than the data window, then run the metric and Docker
+    rollups. Windows are read from Settings each run (rule 25: no restart)."""
+    from steward.core.settings import get_setting
+    from steward.ansible.executor import ARTIFACT_DIR
+
+    now = datetime.now(timezone.utc)
     async with app.db_sessionmaker() as session:
         async with session.begin():
-            for model, ts_col in [
-                (MonitorResult, MonitorResult.checked_at),
-                (AnsibleRun, AnsibleRun.started_at),
-            ]:
+            data_days = int(await get_setting(session, "data.retention_days") or 90)
+            audit_days = int(await get_setting(session, "audit.retention_days") or 365)
+            cutoff = now - timedelta(days=data_days)
+            aged = [(c, cutoff) for c in _AGED]
+            aged.append((AuditEvent.timestamp, now - timedelta(days=audit_days)))
+            for column, before in aged:
                 result = await session.execute(
-                    delete(model).where(ts_col < cutoff)
+                    delete(column.class_).where(column < before)
                 )
                 if result.rowcount:
-                    logger.info(f"Pruned {result.rowcount} rows from {model.__tablename__}")
+                    logger.info("Pruned %d rows from %s", result.rowcount,
+                                column.class_.__tablename__)
+
+        removed = await asyncio.to_thread(prune_run_logs, ARTIFACT_DIR, cutoff)
+        if removed:
+            logger.info("Pruned %d Ansible run log files from %s", removed, ARTIFACT_DIR)
 
         # plugin_metrics is NOT blanket-deleted here — it's rolled up to hourly
         # then pruned, so multi-week host history stays cheap.
@@ -39,6 +65,26 @@ async def run_cleanup(app: "Quart") -> None:
         await _run_docker_retention(session, now)
 
 
+def prune_run_logs(directory: str, before: datetime) -> int:
+    """Delete the run log files in `directory` last written before `before`,
+    the same window as the runs they belong to. Returns how many went."""
+    removed = 0
+    try:
+        entries = list(os.scandir(directory))
+    except OSError:          # no run has written one yet, or not readable
+        return 0
+    limit = before.timestamp()
+    for entry in entries:
+        try:
+            if (entry.name.endswith(".log") and entry.is_file()
+                    and entry.stat().st_mtime < limit):
+                os.unlink(entry.path)
+                removed += 1
+        except OSError:
+            logger.warning("Could not prune run log %s", entry.path)
+    return removed
+
+
 async def _run_metrics_retention(session, now: datetime) -> None:
     """Roll up + prune plugin_metrics (raw → hourly → gone). Windows read fresh
     from settings each run (rule 25 — UI change takes effect next cleanup, no

steward/core/metric_series.py

@@ -13,10 +13,6 @@ from sqlalchemy import func, or_, select, text
 from steward.core.time_range import bucket_seconds
 from steward.models.metrics import PluginMetric, PluginMetricHourly
 
-# Raw rows older than this are rolled into plugin_metrics_hourly and deleted
-# (see core/metrics_retention.py), so a range reaching past it reads the rollup.
-RAW_DAYS = 7
-
 # Loose index scans: step through the distinct leading values of the
 # (source_module, resource_name, …) index one probe at a time instead of
 # scanning every sample. Postgres has no native skip scan before 18.
@@ -79,9 +75,14 @@ async def points(db, source_module: str, resource: str, metric: str, since: date
     The part of the range older than the raw window comes from the hourly
     rollup; the rest is bucket-averaged in SQL (buckets no wider than an hour
     when both are present, so the series is evenly grained across the seam).
+    The raw window is metrics.retention.raw_days, the setting the rollup in
+    core/metrics_retention.py reads, so the seam is where the rollup put it.
     """
+    from steward.core.settings import get_setting
+
+    raw_days = int(await get_setting(db, "metrics.retention.raw_days") or 7)
     now = datetime.now(timezone.utc)
-    raw_cutoff = (now - timedelta(days=RAW_DAYS)).replace(minute=0, second=0, microsecond=0)
+    raw_cutoff = (now - timedelta(days=raw_days)).replace(minute=0, second=0, microsecond=0)
     out: list[tuple[datetime, float]] = []
 
     if since < raw_cutoff:

steward/core/settings.py

@@ -47,7 +47,11 @@ DEFAULTS: dict[str, Any] = {
     # Failed logins from one address before it is refused for the window.
     "login.max_failures": 5,
     "login.lockout_minutes": 15,
+    # Every growing time-series table (monitor results, Ansible runs and their
+    # log files, alert history, Traefik and UniFi samples) keeps this many days.
     "data.retention_days": 90,
+    # The audit log is a security record: kept longer than operational data.
+    "audit.retention_days": 365,
     "monitors.poll_interval_seconds": 60,
     "smtp.host": "",
     "smtp.port": 587,