feat(timeline): maintenance, reboots, deploys, crash-loops and reviewed anomalies on the timeline (#6349)

The host agent writes each maintenance job once it ends, and each stretch a
host was away (a reboot, a crash, lost contact, the agent restarting), kept up
to date until it is back. Docker judges each snapshot: a unit on a new image,
or a container recreated, is a deploy; deaths past the Thresholds crash-loop
count are one crash-loop entry that later deaths extend; single restarts are
recorded but hidden. An anomaly has one entry whose type follows its review:
only those judged real are shown by default.

A source whose record changes keeps one entry for it (record_or_update).
Plugins register the records their entries point to, so an aged-out job or
downtime reads "Details no longer kept". The first fill runs once per source
(timeline.filled), the host agent's jobs and downtime included.

get_anomaly's "what else was going on" reads the timeline for the host or
resource instead of querying alerts, runs, maintenance, downtime and container
deaths itself; Docker's own source keeps only starts, stops and health
changes. A new kind, Observed, is for what Steward saw happen with no rule or
person behind it.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

plugins/host_agent/downtime.py

@@ -51,36 +51,45 @@ async def record_events(session, host_id: str, events, last_seen_at: datetime |
     """Apply one post's events to the host's downtime, oldest first; returns
     how many rows were added or changed. `last_seen_at` is when the agent was
     last heard before this post: a boot after it that nobody announced is a
-    restart without a shutdown."""
+    restart without a shutdown. Each row added or changed goes on the
+    timeline, kept up to date as it ends."""
     if not isinstance(events, list):
         return 0
     timed = sorted(((t, e) for e in events if isinstance(e, dict)
                     for t in [_when(e)] if t is not None), key=lambda te: te[0])
-    changed = 0
+    changed = []
     for at, event in timed:
         kind = event.get("kind")
+        row = None
         if kind == "stopping":
-            changed += await _stopping(session, host_id, at, event.get("host_shutdown"))
+            row = await _stopping(session, host_id, at, event.get("host_shutdown"))
         elif kind == "started":
-            changed += await _started(session, host_id, at, _at(event.get("boot_at")), last_seen_at)
+            row = await _started(session, host_id, at, _at(event.get("boot_at")), last_seen_at)
         elif kind == "lost_contact":
-            changed += await _lost_contact(session, host_id, at, _at(event.get("to")))
-    return changed
+            row = await _lost_contact(session, host_id, at, _at(event.get("to")))
+        if row is not None and row not in changed:
+            changed.append(row)
+    if changed:
+        from . import timeline
+        await session.flush()
+        for row in changed:
+            await timeline.write(session, f"host downtime {row.id}", timeline.downtime_entry(session, row))
+    return len(changed)
 
 
-async def _stopping(session, host_id: str, at: datetime, host_shutdown) -> int:
+async def _stopping(session, host_id: str, at: datetime, host_shutdown) -> HostDowntime | None:
     seen = (await session.execute(select(HostDowntime.id).where(
         HostDowntime.host_id == host_id, HostDowntime.kind.in_(ANNOUNCED),
         HostDowntime.started_at == at))).first()
     if seen:
-        return 0
-    session.add(HostDowntime(host_id=host_id, kind="reboot" if host_shutdown else "agent_stop",
-                             started_at=at))
-    return 1
+        return None
+    row = HostDowntime(host_id=host_id, kind="reboot" if host_shutdown else "agent_stop", started_at=at)
+    session.add(row)
+    return row
 
 
 async def _started(session, host_id: str, at: datetime, boot_at: datetime | None,
-                   last_seen_at: datetime | None) -> int:
+                   last_seen_at: datetime | None) -> HostDowntime | None:
     stop = (await session.execute(select(HostDowntime).where(
         HostDowntime.host_id == host_id, HostDowntime.kind.in_(ANNOUNCED),
         HostDowntime.ended_at.is_(None), HostDowntime.started_at <= at)
@@ -91,31 +100,34 @@ async def _started(session, host_id: str, at: datetime, boot_at: datetime | None
         # boot after it, or it stayed up while only the agent restarted.
         if boot_at is not None:
             stop.kind = "reboot" if boot_at > stop.started_at else "agent_stop"
-        return 1
+        return stop
     if boot_at is None or last_seen_at is None or boot_at <= last_seen_at:
-        return 0
+        return None
     seen = (await session.execute(select(HostDowntime.id).where(
         HostDowntime.host_id == host_id, HostDowntime.kind == "unclean",
         HostDowntime.ended_at == at))).first()
     if seen:
-        return 0
-    session.add(HostDowntime(host_id=host_id, kind="unclean", started_at=last_seen_at, ended_at=at))
-    return 1
+        return None
+    row = HostDowntime(host_id=host_id, kind="unclean", started_at=last_seen_at, ended_at=at)
+    session.add(row)
+    return row
 
 
-async def _lost_contact(session, host_id: str, since: datetime, until: datetime | None) -> int:
+async def _lost_contact(session, host_id: str, since: datetime,
+                        until: datetime | None) -> HostDowntime | None:
     if until is None or until <= since:
-        return 0
+        return None
     row = (await session.execute(select(HostDowntime).where(
         HostDowntime.host_id == host_id, HostDowntime.kind == "lost_contact",
         HostDowntime.started_at == since))).scalar_one_or_none()
     if row is None:
-        session.add(HostDowntime(host_id=host_id, kind="lost_contact", started_at=since, ended_at=until))
-        return 1
+        row = HostDowntime(host_id=host_id, kind="lost_contact", started_at=since, ended_at=until)
+        session.add(row)
+        return row
     if row.ended_at is None or until > row.ended_at:
         row.ended_at = until
-        return 1
-    return 0
+        return row
+    return None
 
 
 async def announced_stops(session, now: datetime, grace_seconds: int) -> dict[str, datetime]: