feat(docker): retention + hourly rollup for metrics/events with Settings windows

Bounds Docker time-series growth (the main scaling concern). New
docker_metrics_hourly table + docker_006 migration; a plugin retention module
(docker.run_retention capability) rolls raw docker_metrics older than the raw
window into hourly averages (idempotent upsert), deletes the rolled raw rows,
then prunes stale rollups + lifecycle events. Core cleanup.py drives it each
hourly run via the capability (no plugin-model import), reading the three
retention windows fresh from settings so changes apply without restart (rule 25).

Settings → "Thresholds & Retention" gains a Docker retention card (raw /
rolled-up / events windows, working defaults 7/90/30 days). Unit tests cover the
hour-aligned cutoff/bucketing helpers; integration test exercises the real
rollup-average + prune across both windows.

Milestone 77 task #941.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_016Jg27rgypiW2efULXJDtMC

steward/core/cleanup.py

@@ -16,9 +16,11 @@ logger = logging.getLogger(__name__)
 
 
 async def run_cleanup(app: "Quart") -> None:
-    """Delete rows older than DATA_RETENTION_DAYS from time-series tables."""
+    """Delete rows older than DATA_RETENTION_DAYS from time-series tables, then
+    run Docker-specific rollup + retention (delegated to the docker plugin)."""
     retention_days: int = app.config.get("DATA_RETENTION_DAYS", 90)
-    cutoff = datetime.now(timezone.utc) - timedelta(days=retention_days)
+    now = datetime.now(timezone.utc)
+    cutoff = now - timedelta(days=retention_days)
 
     async with app.db_sessionmaker() as session:
         async with session.begin():
@@ -32,3 +34,34 @@ async def run_cleanup(app: "Quart") -> None:
                 )
                 if result.rowcount:
                     logger.info(f"Pruned {result.rowcount} rows from {model.__tablename__}")
+
+        await _run_docker_retention(session, now)
+
+
+async def _run_docker_retention(session, now: datetime) -> None:
+    """Drive the docker plugin's rollup + prune via its capability, if loaded.
+
+    Windows are read fresh from settings each run (rule 25 — a change in the
+    Settings UI takes effect on the next hourly cleanup, no restart). Kept in its
+    own transaction so a docker-side failure can't roll back the generic prune
+    above. No-op when the docker plugin is disabled (capability absent).
+    """
+    from steward.core.capabilities import has_capability, invoke_capability
+    if not has_capability("docker.run_retention"):
+        return
+    from steward.core.settings import get_setting
+    from steward.models.users import UserRole
+
+    # Reads + rollup/prune share one transaction — get_setting's SELECT would
+    # otherwise autobegin one, making a later session.begin() raise.
+    async with session.begin():
+        raw_days = int(await get_setting(session, "docker.retention.metrics_raw_days") or 7)
+        rollup_days = int(await get_setting(session, "docker.retention.metrics_rollup_days") or 90)
+        events_days = int(await get_setting(session, "docker.retention.events_days") or 30)
+        counts = await invoke_capability(
+            "docker.run_retention", UserRole.viewer, session,
+            events_days=events_days, metrics_raw_days=raw_days,
+            metrics_rollup_days=rollup_days, now=now,
+        )
+    if counts and any(counts.values()):
+        logger.info("Docker retention: %s", counts)

steward/core/settings.py

@@ -78,6 +78,12 @@ DEFAULTS: dict[str, Any] = {
     "thresholds.load_warn": 80, "thresholds.load_crit": 100,
     "thresholds.temp_warn": 70, "thresholds.temp_crit": 85,
     "thresholds.uptime_warn": 99.0, "thresholds.uptime_crit": 95.0,
+    # Docker time-series retention (rule 25 — tunable, no restart). Raw 30s
+    # samples are heavy, so keep a short raw window then roll up to hourly
+    # averages kept much longer; lifecycle events are light, keep a month.
+    "docker.retention.metrics_raw_days": 7,
+    "docker.retention.metrics_rollup_days": 90,
+    "docker.retention.events_days": 30,
     "plugins.index_url": "https://git.fabledsword.com/bvandeusen/Steward-plugins/raw/branch/main/index.yaml",
     # Default-enabled plugins. These are the generic, non-vendor-specific
     # bundled plugins (protocols/standards, not a single product) — useful on

steward/settings/routes.py

@@ -127,6 +127,14 @@ _THRESHOLD_FIELDS = [
     ("latency_warn", "ping.threshold.good_ms", False), ("latency_crit", "ping.threshold.warn_ms", False),
 ]
 
+# Docker time-series retention windows (days). Read fresh by the cleanup task,
+# so a save takes effect on the next hourly run — no app.config wiring needed.
+_RETENTION_FIELDS = [
+    ("docker_metrics_raw_days", "docker.retention.metrics_raw_days"),
+    ("docker_metrics_rollup_days", "docker.retention.metrics_rollup_days"),
+    ("docker_events_days", "docker.retention.events_days"),
+]
+
 
 @settings_bp.get("/thresholds/")
 @require_role(UserRole.admin)
@@ -151,6 +159,15 @@ async def save_thresholds():
                 except (TypeError, ValueError):
                     continue
                 await set_setting(db, key, val)
+            for field, key in _RETENTION_FIELDS:
+                raw = form.get(field, "")
+                if raw == "":
+                    continue
+                try:
+                    val = max(1, int(raw))  # at least a day — 0 would prune everything
+                except (TypeError, ValueError):
+                    continue
+                await set_setting(db, key, val)
     await _reload_app_config()
     await log_audit(current_app, session.get("user_id"), session.get("username", ""),
                     "settings.saved", detail={"section": "thresholds"})

steward/templates/settings/_tabs.html

@@ -3,7 +3,7 @@
 <div style="display:flex;gap:0;border-bottom:1px solid var(--border-mid);margin-bottom:1.5rem;">
   {% set tabs = [
     ("general",       "General",        "/settings/general/"),
-    ("thresholds",    "Thresholds",     "/settings/thresholds/"),
+    ("thresholds",    "Thresholds & Retention", "/settings/thresholds/"),
     ("notifications", "Notifications",  "/settings/notifications/"),
     ("reports",       "Reports",        "/settings/reports/"),
     ("auth",          "Auth",           "/settings/auth/"),

steward/templates/settings/thresholds.html

@@ -1,6 +1,6 @@
 {# steward/templates/settings/thresholds.html — tunable degraded/critical cutoffs #}
 {% extends "base.html" %}
-{% block title %}Settings — Thresholds — Steward{% endblock %}
+{% block title %}Settings — Thresholds & Retention — Steward{% endblock %}
 {% block content %}
 {% set active_tab = "thresholds" %}
 {% include "settings/_tabs.html" %}
@@ -42,6 +42,34 @@
   {{ pair("Uptime / SLA", "uptime_warn", "thresholds.uptime_warn", "uptime_crit", "thresholds.uptime_crit", "%", "Lower is worse — amber below warn, red below crit.", step="0.1") }}
 </div>
 
+{% macro days(label, field, key, hint) %}
+<div class="form-group" style="margin-bottom:1.1rem;">
+  <label>{{ label }} <span style="color:var(--text-muted);font-size:0.8rem;">(days)</span></label>
+  <div style="margin-top:0.25rem;">
+    <input type="number" name="{{ field }}" min="1" step="1"
+           value="{{ settings[key] }}" style="max-width:110px;">
+  </div>
+  {% if hint %}<div style="font-size:0.78rem;color:var(--text-muted);margin-top:0.3rem;">{{ hint }}</div>{% endif %}
+</div>
+{% endmacro %}
+
+<div class="card" style="max-width:640px;margin-top:1.5rem;">
+  <h2 class="section-title" style="margin-bottom:0.5rem;">Docker data retention</h2>
+  <p style="font-size:0.82rem;color:var(--text-muted);margin-bottom:1.25rem;">
+    Bounds how much Docker history is stored. Raw per-sample container metrics are
+    kept for the raw window, then rolled up into hourly averages kept for the
+    rollup window; lifecycle events are kept for the events window. Applied by the
+    hourly cleanup task — a change takes effect on its next run.
+  </p>
+
+  {{ days("Raw metrics", "docker_metrics_raw_days", "docker.retention.metrics_raw_days",
+          "Keep per-sample container CPU/memory points this long, then roll up to hourly.") }}
+  {{ days("Rolled-up metrics", "docker_metrics_rollup_days", "docker.retention.metrics_rollup_days",
+          "Keep the hourly-averaged series this long for multi-day history.") }}
+  {{ days("Lifecycle events", "docker_events_days", "docker.retention.events_days",
+          "Keep container start/stop/die/health history this long.") }}
+</div>
+
 <div style="margin-top:1rem;display:flex;align-items:center;gap:1rem;">
   <button type="submit" class="btn">Save</button>
   <span style="font-size:0.82rem;color:var(--text-muted);">Takes effect immediately — no restart.</span>