feat(timeline): maintenance, reboots, deploys, crash-loops and reviewed anomalies on the timeline (#6349)

The host agent writes each maintenance job once it ends, and each stretch a
host was away (a reboot, a crash, lost contact, the agent restarting), kept up
to date until it is back. Docker judges each snapshot: a unit on a new image,
or a container recreated, is a deploy; deaths past the Thresholds crash-loop
count are one crash-loop entry that later deaths extend; single restarts are
recorded but hidden. An anomaly has one entry whose type follows its review:
only those judged real are shown by default.

A source whose record changes keeps one entry for it (record_or_update).
Plugins register the records their entries point to, so an aged-out job or
downtime reads "Details no longer kept". The first fill runs once per source
(timeline.filled), the host agent's jobs and downtime included.

get_anomaly's "what else was going on" reads the timeline for the host or
resource instead of querying alerts, runs, maintenance, downtime and container
deaths itself; Docker's own source keeps only starts, stops and health
changes. A new kind, Observed, is for what Steward saw happen with no rule or
person behind it.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

steward/mcp/tools/_events.py

@@ -1,11 +1,12 @@
 """What else was going on: events on a host, or one resource, in a window
 (#5837), so an anomaly comes with its circumstances.
 
-Core gathers alert transitions, Ansible runs, other metric anomalies and
-Steward's own gaps in recording. A plugin adds its own through
-`mcp_events(db, host_id, host_name, since, until)` in its __init__.py (the
-host agent its maintenance jobs and downtime, Docker its container events),
-and for a resource with no host through
+Core reads the timeline (alert episodes, Ansible runs, and what plugins
+wrote there: maintenance, downtime, deploys, restarts), other metric
+anomalies, and Steward's own gaps in recording. A plugin adds what the
+timeline doesn't hold through `mcp_events(db, host_id, host_name, since,
+until)` in its __init__.py (Docker its stops and health changes, syslog its
+warnings), and for a resource with no host through
 `mcp_resource_events(db, source, resource, since, until)` (Traefik: the
 containers of the swarm service behind a router, #5862); each returns a list
 of `event(...)`.
@@ -16,15 +17,11 @@ import logging
 from dataclasses import dataclass
 from datetime import datetime, timedelta
 
-from sqlalchemy import and_, or_, select
+from sqlalchemy import and_, func, or_, select
 
-from steward.alerts.service import watches_host
 from steward.core.alerts import host_of_resource
 from steward.core.formatting import human_duration
 from steward.mcp._context import mcp_app
-from steward.models.alerts import AlertEvent, AlertRule
-from steward.models.ansible import AnsibleRun
-from steward.models.ansible_inventory import AnsibleTarget
 from steward.models.hosts import Host
 from steward.models.metrics import PluginMetric
 from steward.models.monitors import Monitor
@@ -80,47 +77,23 @@ def _overlaps(start, end):
     return clause
 
 
-async def alert_events(db, place: Place, since: datetime, until: datetime) -> list[dict]:
-    """Alert transitions on the resource itself, or anything its host
-    carries: its agent series, its containers, its monitors."""
-    monitor_keys = set()
-    if place.host_id:
-        monitor_keys = {(t, n) for t, n in (await db.execute(
-            select(Monitor.type, Monitor.name).where(Monitor.host_id == place.host_id))).all()}
+async def timeline_events(db, place: Place, since: datetime, until: datetime) -> list[dict]:
+    """What the timeline holds about the host, or the resource itself, while
+    the window was open: alert episodes, playbook runs, maintenance, reboots
+    and lost contact, deploys, restarts and crash-loops, settings changes.
+    Hidden types are included: a dry run or a single restart can be what
+    explains a reading. Anomalies come from other_anomalies, by occurrence."""
+    from steward.models.timeline import TimelineEntry
+    about = [TimelineEntry.subject == place.resource]
+    if place.host_name:
+        about.append(TimelineEntry.hosts.contains([place.host_name]))
     rows = (await db.execute(
-        select(AlertEvent, AlertRule).join(AlertRule, AlertRule.id == AlertEvent.rule_id)
-        .where(AlertEvent.transitioned_at >= since, AlertEvent.transitioned_at <= until)
-        .order_by(AlertEvent.transitioned_at).limit(500))).all()
-    out = []
-    for ev, rule in rows:
-        mine = (rule.source_module, rule.resource_name) == (place.source, place.resource)
-        if not mine and not (place.host_name and watches_host(rule, place.host_name, monitor_keys)):
-            continue
-        note = f" ({ev.note})" if ev.note else ""
-        out.append(event(ev.transitioned_at, "alert", f"{rule.name}: {ev.from_state} → {ev.to_state}{note}",
-                         subject=rule.resource_name, ids={"rule_id": rule.id}))
-    return out
-
-
-async def ansible_runs(db, place: Place, since: datetime, until: datetime) -> list[dict]:
-    """Ansible runs that touched the host while the window was open (by the
-    hosts in their recap: a run still going hasn't said yet)."""
-    if not place.host_id:
-        return []
-    names = {place.host_name} | set((await db.execute(
-        select(AnsibleTarget.name).where(AnsibleTarget.host_id == place.host_id))).scalars())
-    runs = (await db.execute(
-        select(AnsibleRun).where(_overlaps(AnsibleRun.started_at, AnsibleRun.finished_at)(since, until))
-        .order_by(AnsibleRun.started_at).limit(200))).scalars()
-    out = []
-    for r in runs:
-        if not names & set(((r.results or {}).get("hosts") or {}).keys()):
-            continue
-        check = " in check mode" if (r.params or {}).get("check") else ""
-        out.append(event(r.started_at, "ansible_run",
-                         f"Ansible {r.playbook_path}{check}: {r.status.value}",
-                         until=r.finished_at, ids={"run_id": r.id}))
-    return out
+        select(TimelineEntry)
+        .where(or_(*about), _overlaps(TimelineEntry.at, func.coalesce(TimelineEntry.until, TimelineEntry.at))
+               (since, until), or_(TimelineEntry.ref_type.is_(None), TimelineEntry.ref_type != "metric_anomaly"))
+        .order_by(TimelineEntry.at).limit(500))).scalars()
+    return [event(e.at, e.type, e.summary, until=e.until, subject=e.subject,
+                  ids={e.ref_type: e.ref_id} if e.ref_type else None) for e in rows]
 
 
 def _same_host(source: str, resource: str, place: Place) -> bool:
@@ -248,8 +221,8 @@ async def around(db, place: Place, anomaly_id: str, since: datetime, until: date
     elsewhere; and how many events were left out, each source keeping its
     PER_SOURCE nearest `focus`."""
     anomalies, elsewhere = await other_anomalies(db, place, anomaly_id, since, until)
-    groups = [await alert_events(db, place, since, until), await ansible_runs(db, place, since, until),
-              anomalies, *await plugin_events(place, since, until),
+    groups = [await timeline_events(db, place, since, until), anomalies,
+              *await plugin_events(place, since, until),
               *await resource_events(place, since, until), await steward_gaps(db, since, until)]
     found, left_out = [], 0
     for group in groups: