feat(timeline): one record of what happened to the infrastructure, newest first (#6348)

A stored log, timeline_entries, with one row per thing that happened. Each
source writes its row in the same transaction as its own record, so the
history outlives the 30/90/365-day retention of the tables it points to:

- alert events (core/alerts._add_event): fired, resolved, acknowledged,
  went silent, readings back;
- finished Ansible runs (executor, and the startup sweep of interrupted
  runs): changed something, changed nothing, failed, dry run;
- audit rows (core/audit.log_audit): Steward's settings, users and access,
  sign-ins, backups. Actions another writer records are left to it.

Entries carry a kind (fixed) and an area (Service, Host OS, Hardware,
Storage, Network, Security, Monitoring). Settings → Timeline edits the
areas and each type's area and visibility; both are applied when the
timeline is read, so a change re-sorts past entries and a hidden type
comes back when shown. Kept forever unless a retention is set.

The first start fills the timeline from the last 90 days of records once,
skipping any already covered. A host rename carries its entries.

UI: Timeline in the sidebar (filter by host, area, kind; scrolls on), a
History card on each host page. MCP: get_timeline.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

steward/timeline/record.py

@@ -0,0 +1,233 @@
+"""Writing the timeline. Every source writes through record(), in the same
+transaction as its own record, as the thing happens: the entry stands on its
+own, so it is still there after the source's own row ages out.
+
+from_alert, from_run and from_audit build the entry for one alert event,
+finished Ansible run and audit row. The live writers and the backfill use the
+same three, so an entry reads the same however it got there.
+"""
+from __future__ import annotations
+
+import logging
+from datetime import datetime
+
+from sqlalchemy import select
+
+from . import catalog
+from steward.models.timeline import TimelineEntry
+
+logger = logging.getLogger(__name__)
+
+
+def record(session, *, type: str, at: datetime, summary: str, area: str | None = None,
+           hosts=(), subject: str | None = None, detail: dict | None = None,
+           actor: str = "system", until: datetime | None = None,
+           ref: tuple[str, str] | None = None) -> TimelineEntry:
+    """Add one entry to `session`. `area` defaults to its type's; `ref` is
+    (kind, id) of the full record it points to, which is at most one entry."""
+    known = catalog.TYPES.get(type)
+    entry = TimelineEntry(
+        at=at, until=until, type=type,
+        area=area or (known.area if known else catalog.OTHER[0]),
+        hosts=sorted({h for h in hosts if h}), subject=subject[:255] if subject else None, summary=summary,
+        detail=detail or None, actor=(actor or "system")[:64],
+        ref_type=ref[0] if ref else None, ref_id=str(ref[1]) if ref else None)
+    session.add(entry)
+    return entry
+
+
+async def username(session, user_id: str | None) -> str | None:
+    if not user_id:
+        return None
+    from steward.models.users import User
+    user = await session.get(User, user_id)
+    return user.username if user else None
+
+
+# ── Alerts ──────────────────────────────────────────────────────────────────
+
+_MONITOR_SOURCES = ("icmp", "tcp", "dns", "http")
+
+
+async def hosts_of_rule(session, source: str, resource: str) -> list[str]:
+    """The host an alert rule's resource is on, by name, when it has one: an
+    agent series ("<host>" or "<host>:<part>"), a container ("<host>/<name>")
+    or a monitor tied to a host."""
+    from steward.core.alerts import host_of_resource
+    if source == "host_agent":
+        return [host_of_resource(source, resource) or resource.split(":", 1)[0]]
+    if source == "docker":
+        return [resource.split("/", 1)[0]]
+    if source in _MONITOR_SOURCES:
+        from steward.models.hosts import Host
+        from steward.models.monitors import Monitor
+        name = (await session.execute(
+            select(Host.name).join(Monitor, Monitor.host_id == Host.id)
+            .where(Monitor.type == source, Monitor.name == resource).limit(1))).scalar()
+        return [name] if name else []
+    return []
+
+
+def _alert_type(from_state: str, to_state: str) -> str | None:
+    if to_state == "firing":
+        return "alert.fired"
+    if to_state == "resolved":
+        return "alert.resolved"
+    if to_state == "acknowledged":
+        return "alert.acknowledged"
+    if to_state == "no_data":
+        return "alert.silent"
+    if from_state == "no_data":
+        return "alert.resumed"
+    return None
+
+
+def alert_summary(type_: str, rule, value: float | None) -> str:
+    from steward.alerts.service import _BINARY
+    from steward.core.formatting import metric_label, metric_value
+    metric, res = rule.metric_name, rule.resource_name
+    said = metric_label(metric)
+    # The reading, and on a firing the condition it broke.
+    if metric in _BINARY:
+        reading = breach = f"{res} is {metric_value(metric, value)}"
+    else:
+        op = getattr(rule.operator, "value", rule.operator)
+        reading = f"{said} on {res} is {metric_value(metric, value)}"
+        breach = f"{reading} ({op} {metric_value(metric, rule.threshold)})"
+    return {
+        "alert.fired": f"{rule.name} fired: {breach}" if value is not None else f"{rule.name} fired",
+        "alert.resolved": f"{rule.name} resolved" + (f": {reading}" if value is not None else ""),
+        "alert.acknowledged": f"{rule.name} acknowledged",
+        "alert.silent": f"{rule.name}: no {said} reading from {res}",
+        "alert.resumed": f"{rule.name}: {said} readings from {res} again",
+    }[type_]
+
+
+async def from_alert(session, rule, event, actor: str | None = None,
+                     resumed: bool = False) -> TimelineEntry | None:
+    """The entry for one alert event (an AlertEvent row), or None for a
+    transition the timeline doesn't show (pending and back). `resumed` is a
+    series back from silence with the alert's state unchanged, which can be
+    an already-firing alert still firing."""
+    type_ = "alert.resumed" if resumed else _alert_type(event.from_state, event.to_state)
+    if type_ is None:
+        return None
+    return record(
+        session, type=type_, at=event.transitioned_at,
+        summary=alert_summary(type_, rule, event.metric_value),
+        area=catalog.area_of_metric(rule.source_module, rule.metric_name),
+        hosts=await hosts_of_rule(session, rule.source_module, rule.resource_name),
+        subject=rule.resource_name,
+        detail={"rule_id": rule.id, "rule": rule.name, "source": rule.source_module,
+                "metric": rule.metric_name, "value": event.metric_value, "note": event.note,
+                "url": f"/alerts/history?rule={rule.id}"},
+        actor=actor or await username(session, getattr(event, "actor_id", None)) or "system",
+        ref=("alert_event", event.id))
+
+
+# ── Ansible runs ────────────────────────────────────────────────────────────
+
+def _run_type(run) -> str:
+    status = getattr(run.status, "value", run.status)
+    if status != "success":
+        return "ansible.failed"
+    if (run.params or {}).get("check"):
+        return "ansible.check"
+    hosts = ((run.results or {}).get("hosts") or {}).values()
+    return "ansible.changed" if any((h or {}).get("changed") for h in hosts) else "ansible.unchanged"
+
+
+async def hosts_of_run(session, names) -> list[str]:
+    """The Steward hosts behind a run's inventory names: a target linked to a
+    host is that host; a name that is a host's own is that host."""
+    from steward.models.ansible_inventory import AnsibleTarget
+    from steward.models.hosts import Host
+    names = set(names)
+    if not names:
+        return []
+    linked = dict((await session.execute(
+        select(AnsibleTarget.name, Host.name).join(Host, AnsibleTarget.host_id == Host.id)
+        .where(AnsibleTarget.name.in_(names)))).all())
+    direct = set((await session.execute(
+        select(Host.name).where(Host.name.in_(names - set(linked))))).scalars())
+    return sorted(set(linked.values()) | direct)
+
+
+async def from_run(session, run) -> TimelineEntry:
+    """The entry for one finished Ansible run."""
+    from steward.ansible.inventory_gen import run_title, scope_labels
+    from steward.ansible.origins import run_origins
+    type_ = _run_type(run)
+    per_host = (run.results or {}).get("hosts") or {}
+    scope = (await scope_labels(session, [run.inventory_scope])).get(run.inventory_scope)
+    title = run_title(run.playbook_path, scope)
+    status = getattr(run.status, "value", run.status)
+    changed = sum(1 for h in per_host.values() if (h or {}).get("changed"))
+    failed = sum(1 for h in per_host.values() if (h or {}).get("failed") or (h or {}).get("unreachable"))
+    if type_ == "ansible.failed":
+        said = {"cancelled": "cancelled", "interrupted": "stopped by a restart"}.get(status, "failed")
+        tail = f"{said} on {failed} host{'s' if failed != 1 else ''}" if failed else said
+    elif type_ == "ansible.check":
+        tail = f"dry run, would change {changed} host{'s' if changed != 1 else ''}" if changed \
+            else "dry run, nothing to change"
+    elif type_ == "ansible.changed":
+        tail = f"changed {changed} host{'s' if changed != 1 else ''}"
+    else:
+        tail = "nothing to change"
+    origin = (await run_origins(session, [run])).get(run.id) or {}
+    return record(
+        session, type=type_, at=run.started_at, until=run.finished_at,
+        summary=f"{title}: {tail}", hosts=await hosts_of_run(session, per_host),
+        subject=run.playbook_path,
+        detail={"status": status, "scope": scope, "origin": origin.get("text"),
+                "failures": ((run.results or {}).get("failures") or [])[:3],
+                "url": f"/ansible/runs/{run.id}"},
+        actor=origin.get("text") or "Steward", ref=("ansible_run", run.id))
+
+
+async def run_finished(session, run_id: str) -> None:
+    """Write the entry for a run that has just recorded its final status, in
+    a savepoint of the caller's transaction. Never raises: the run's own
+    record matters more than its entry."""
+    from steward.models.ansible import AnsibleRun
+    try:
+        async with session.begin_nested():
+            run = await session.get(AnsibleRun, run_id)
+            if run is not None:
+                await from_run(session, run)
+    except Exception:
+        logger.exception("Ansible run %s: its timeline entry could not be written", run_id)
+
+
+# ── The audit log ───────────────────────────────────────────────────────────
+
+async def from_audit(session, event, detail: dict | None) -> TimelineEntry | None:
+    """The entry for one audit row (an AuditEvent), or None for an action the
+    timeline leaves to another writer (catalog.audit_type)."""
+    from steward.audit.service import action_text, entity_labels
+    type_ = catalog.audit_type(event.action)
+    if type_ is None:
+        return None
+    detail = detail or {}
+    entity = None
+    if event.entity_type and event.entity_id:
+        labels = await entity_labels(session, [{"event": event, "detail": detail}])
+        entity = labels.get((event.entity_type, str(event.entity_id)))
+    what = action_text(event.action)
+    if entity:
+        what = f"{what} {entity['label']}"
+    hosts = []
+    if event.entity_type == "host" and entity:
+        hosts = [entity["label"]]
+    elif event.entity_type == "monitor":
+        from steward.models.hosts import Host
+        from steward.models.monitors import Monitor
+        name = (await session.execute(
+            select(Host.name).join(Monitor, Monitor.host_id == Host.id)
+            .where(Monitor.id == str(event.entity_id)))).scalar()
+        hosts = [name] if name else []
+    return record(
+        session, type=type_, at=event.timestamp, summary=what[:1].upper() + what[1:],
+        hosts=hosts, subject=entity["label"] if entity else None,
+        detail={"action": event.action, "url": entity.get("url") if entity else None},
+        actor=event.username, ref=("audit_event", event.id))