feat(timeline): one record of what happened to the infrastructure, newest first (#6348)

A stored log, timeline_entries, with one row per thing that happened. Each
source writes its row in the same transaction as its own record, so the
history outlives the 30/90/365-day retention of the tables it points to:

- alert events (core/alerts._add_event): fired, resolved, acknowledged,
  went silent, readings back;
- finished Ansible runs (executor, and the startup sweep of interrupted
  runs): changed something, changed nothing, failed, dry run;
- audit rows (core/audit.log_audit): Steward's settings, users and access,
  sign-ins, backups. Actions another writer records are left to it.

Entries carry a kind (fixed) and an area (Service, Host OS, Hardware,
Storage, Network, Security, Monitoring). Settings → Timeline edits the
areas and each type's area and visibility; both are applied when the
timeline is read, so a change re-sorts past entries and a hidden type
comes back when shown. Kept forever unless a retention is set.

The first start fills the timeline from the last 90 days of records once,
skipping any already covered. A host rename carries its entries.

UI: Timeline in the sidebar (filter by host, area, kind; scrolls on), a
History card on each host page. MCP: get_timeline.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

steward/core/alerts.py

@@ -571,11 +571,12 @@ async def _evaluate_rule(
             state.acknowledged_at = None
     said = "; ".join(n for n in (resumed, note) if n) or None
     if step.event:
-        event_id = _add_event(session, rule.id, "no_data" if resumed else from_state.value,
-                              step.event, value, now, note=said)
+        event_id = await _add_event(session, rule, "no_data" if resumed else from_state.value,
+                                    step.event, value, now, note=said)
         notifications.append((step.event, event_id, None))
     elif resumed:
-        event_id = _add_event(session, rule.id, "no_data", state.state.value, value, now, note=said)
+        event_id = await _add_event(session, rule, "no_data", state.state.value, value, now, note=said,
+                                    resumed=True)
         notifications.append(("data_resumed", event_id, resumed))
     await session.flush()
     return notifications
@@ -595,8 +596,8 @@ async def reset_rule_state(session: AsyncSession, rule: AlertRule, note: str,
         return
     now = datetime.now(timezone.utc)
     if state.state in (AlertStateEnum.firing, AlertStateEnum.acknowledged):
-        _add_event(session, rule.id, state.state.value, "resolved", None, now, note=note,
-                   actor_id=actor_id)
+        await _add_event(session, rule, state.state.value, "resolved", None, now, note=note,
+                         actor_id=actor_id)
     # The watch starts over now: silence is measured from here, against the
     # cadence of whatever the rule watches from now on.
     state.last_transition_at = now
@@ -610,21 +611,27 @@ async def reset_rule_state(session: AsyncSession, rule: AlertRule, note: str,
     state.acknowledged_at = None
 
 
-def _add_event(
+async def _add_event(
     session: AsyncSession,
-    rule_id: str,
+    rule: AlertRule,
     from_state: str,
     to_state: str,
     value: float | None,
     now: datetime,
     note: str | None = None,
     actor_id: str | None = None,
+    actor: str | None = None,
+    resumed: bool = False,
 ) -> str:
-    """Add an AlertEvent row and return its ID."""
+    """Add an AlertEvent row, and its timeline entry, and return its ID.
+    `actor` names who did it on the timeline when `actor_id`'s username
+    alone would not say it ("<user> via MCP"). `resumed` marks a series
+    back from silence that left the alert's state as it was."""
+    from steward.timeline.record import from_alert
     event_id = str(uuid.uuid4())
     event = AlertEvent(
         id=event_id,
-        rule_id=rule_id,
+        rule_id=rule.id,
         from_state=from_state,
         to_state=to_state,
         metric_value=value,
@@ -633,6 +640,11 @@ def _add_event(
         actor_id=actor_id,
     )
     session.add(event)
+    try:
+        await from_alert(session, rule, event, actor, resumed=resumed)
+    except Exception:
+        # The alert's own history matters more than its timeline entry.
+        logger.exception("Alert event %s: its timeline entry could not be written", event_id)
     return event_id
 
 
@@ -825,7 +837,7 @@ async def check_silence(app: "Quart") -> None:
                 muted = _muted_by(windows, rule)
                 # Silent with its agent: recorded, and "agent offline" is the email (#5721).
                 hushed = AGENT_OFFLINE if host_behind(rule) in offline else None
-                event_id = _add_event(session, rule.id, state.state.value, "no_data", None, now,
+                event_id = await _add_event(session, rule, state.state.value, "no_data", None, now,
                                       note="; ".join(n for n in (note, hushed, muted) if n))
                 if rule.notify_no_data and not muted and not hushed:
                     notify.append((rule, event_id, note))

steward/core/audit.py

@@ -22,12 +22,17 @@ async def log_audit(
     entity_id: str | None = None,
     detail: dict | None = None,
 ) -> None:
-    """Write one audit event. Never raises — failures are logged and swallowed."""
+    """Write one audit event, and its timeline entry when the action has one
+    (steward.timeline.catalog.audit_type). Never raises — failures are logged
+    and swallowed."""
+    from steward.core.ids import time_ordered_id
     from steward.models.audit import AuditEvent
+    from steward.timeline.record import from_audit
     try:
         async with app.db_sessionmaker() as db:
             async with db.begin():
-                db.add(AuditEvent(
+                event = AuditEvent(
+                    id=time_ordered_id(),
                     user_id=user_id,
                     username=username or "unknown",
                     action=action,
@@ -35,7 +40,14 @@ async def log_audit(
                     entity_id=entity_id,
                     detail_json=json.dumps(detail) if detail else None,
                     timestamp=datetime.now(timezone.utc),
-                ))
+                )
+                db.add(event)
+                try:
+                    # In a savepoint: an entry that fails leaves the audit row standing.
+                    async with db.begin_nested():
+                        await from_audit(db, event, detail)
+                except Exception:
+                    logger.exception("Audit event %r: its timeline entry could not be written", action)
     except Exception:
         logger.exception("Failed to write audit event action=%r", action)
 

steward/core/cleanup.py

@@ -9,6 +9,7 @@ from typing import TYPE_CHECKING
 from sqlalchemy import delete
 
 from steward.core.anomalies import prune as prune_anomalies
+from steward.timeline.service import prune as prune_timeline
 from steward.models.alerts import AlertEvent
 from steward.models.ansible import AnsibleRun
 from steward.models.audit import AuditEvent
@@ -81,6 +82,10 @@ async def run_cleanup(app: "Quart") -> dict:
             pruned = await prune_anomalies(session, cutoff)
             removed.append([f"metric_anomalies: reviewed, ended over {data_days} days ago",
                             pruned])
+            # The timeline keeps its own window, everything by default.
+            pruned, days = await prune_timeline(session, now)
+            if days:
+                removed.append([f"timeline_entries: older than {days} days", pruned])
 
         files = await asyncio.to_thread(prune_run_logs, ARTIFACT_DIR, cutoff)
         removed.append([f"Ansible run log files: older than {data_days} days", files])

steward/core/settings.py

@@ -52,6 +52,17 @@ DEFAULTS: dict[str, Any] = {
     "data.retention_days": 90,
     # The audit log is a security record: kept longer than operational data.
     "audit.retention_days": 365,
+    # The timeline is the record of what happened to the infrastructure: kept
+    # until this many days are set (0 keeps it all). Its areas, and each entry
+    # type's area and visibility, are Settings → Timeline (steward/timeline).
+    "timeline.retention_days": 0,
+    "timeline.areas": [["service", "Service"], ["host_os", "Host OS"], ["hardware", "Hardware"],
+                       ["storage", "Storage"], ["network", "Network"], ["security", "Security"],
+                       ["monitoring", "Monitoring"]],
+    "timeline.types": {},
+    "timeline.area_moved": {},
+    # When the timeline was first filled from the records that predate it.
+    "timeline.backfilled_at": "",
     "monitors.poll_interval_seconds": 60,
     # Root log level: DEBUG, INFO, WARNING or ERROR (steward/core/logging_setup.py).
     # WARNING also hides the per-request access lines.