feat(timeline): maintenance, reboots, deploys, crash-loops and reviewed anomalies on the timeline (#6349)

The host agent writes each maintenance job once it ends, and each stretch a
host was away (a reboot, a crash, lost contact, the agent restarting), kept up
to date until it is back. Docker judges each snapshot: a unit on a new image,
or a container recreated, is a deploy; deaths past the Thresholds crash-loop
count are one crash-loop entry that later deaths extend; single restarts are
recorded but hidden. An anomaly has one entry whose type follows its review:
only those judged real are shown by default.

A source whose record changes keeps one entry for it (record_or_update).
Plugins register the records their entries point to, so an aged-out job or
downtime reads "Details no longer kept". The first fill runs once per source
(timeline.filled), the host agent's jobs and downtime included.

get_anomaly's "what else was going on" reads the timeline for the host or
resource instead of querying alerts, runs, maintenance, downtime and container
deaths itself; Docker's own source keeps only starts, stops and health
changes. A new kind, Observed, is for what Steward saw happen with no rule or
person behind it.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

plugins/host_agent/__init__.py

@@ -34,6 +34,9 @@ def setup(app: "Quart") -> None:
     from steward.core.cleanup import prune_by_age
     from .models import HostDowntime
     prune_by_age(HostDowntime.started_at)
+    # Maintenance runs and the stretches a host was away go on the timeline.
+    from .timeline import register as register_timeline
+    register_timeline()
 
 
 def get_audit_entities() -> dict:
@@ -62,9 +65,9 @@ async def mcp_overview(db) -> list[dict]:
     return await findings(db)
 
 
-async def mcp_events(db, host_id: str, host_name: str, since, until) -> list[dict]:
-    from .mcp_tools import mcp_events as events
-    return await events(db, host_id, host_name, since, until)
+async def timeline_backfill(session, since) -> list:
+    from .timeline import backfill
+    return await backfill(session, since)
 
 
 def get_alert_metrics() -> dict[str, list[str]]:

plugins/host_agent/cadence.py

@@ -22,7 +22,7 @@ from __future__ import annotations
 import json
 from datetime import datetime, time, timedelta, timezone
 
-from . import maintenance
+from . import maintenance, timeline
 
 WEEKDAYS = ("Monday", "Tuesday", "Wednesday", "Thursday", "Friday", "Saturday", "Sunday")
 # How long a host may take to be back in service after its job finished.
@@ -241,6 +241,8 @@ async def advance(session, run, plan, now: datetime) -> str:
         if move == "fail":
             if job is not None and job.state == "done":
                 maintenance._finish(job, "failed", now, why)     # the host never came back into service
+                await timeline.write(session, f"maintenance job {job.id}",
+                                     timeline.job_entry(session, job, host.name if host else None))
             name = host.name if host else host_id
             _end(run, "failed", now, f"{name}: {why}")
         elif move == "stop":
@@ -325,5 +327,7 @@ async def failed_hosts(session, now: datetime) -> dict[str, float]:
                 HostAgentRegistration.host_id == host.id))).scalar_one_or_none()
             if await _back(session, host, job, reg, now) is None:
                 job.came_back_at = now
+                await timeline.write(session, f"maintenance job {job.id}",
+                                     timeline.job_entry(session, job, host.name))
         out[name] = 1.0 if maintenance.still_failed(job) else 0.0
     return out

plugins/host_agent/downtime.py

@@ -51,36 +51,45 @@ async def record_events(session, host_id: str, events, last_seen_at: datetime |
     """Apply one post's events to the host's downtime, oldest first; returns
     how many rows were added or changed. `last_seen_at` is when the agent was
     last heard before this post: a boot after it that nobody announced is a
-    restart without a shutdown."""
+    restart without a shutdown. Each row added or changed goes on the
+    timeline, kept up to date as it ends."""
     if not isinstance(events, list):
         return 0
     timed = sorted(((t, e) for e in events if isinstance(e, dict)
                     for t in [_when(e)] if t is not None), key=lambda te: te[0])
-    changed = 0
+    changed = []
     for at, event in timed:
         kind = event.get("kind")
+        row = None
         if kind == "stopping":
-            changed += await _stopping(session, host_id, at, event.get("host_shutdown"))
+            row = await _stopping(session, host_id, at, event.get("host_shutdown"))
         elif kind == "started":
-            changed += await _started(session, host_id, at, _at(event.get("boot_at")), last_seen_at)
+            row = await _started(session, host_id, at, _at(event.get("boot_at")), last_seen_at)
         elif kind == "lost_contact":
-            changed += await _lost_contact(session, host_id, at, _at(event.get("to")))
-    return changed
+            row = await _lost_contact(session, host_id, at, _at(event.get("to")))
+        if row is not None and row not in changed:
+            changed.append(row)
+    if changed:
+        from . import timeline
+        await session.flush()
+        for row in changed:
+            await timeline.write(session, f"host downtime {row.id}", timeline.downtime_entry(session, row))
+    return len(changed)
 
 
-async def _stopping(session, host_id: str, at: datetime, host_shutdown) -> int:
+async def _stopping(session, host_id: str, at: datetime, host_shutdown) -> HostDowntime | None:
     seen = (await session.execute(select(HostDowntime.id).where(
         HostDowntime.host_id == host_id, HostDowntime.kind.in_(ANNOUNCED),
         HostDowntime.started_at == at))).first()
     if seen:
-        return 0
-    session.add(HostDowntime(host_id=host_id, kind="reboot" if host_shutdown else "agent_stop",
-                             started_at=at))
-    return 1
+        return None
+    row = HostDowntime(host_id=host_id, kind="reboot" if host_shutdown else "agent_stop", started_at=at)
+    session.add(row)
+    return row
 
 
 async def _started(session, host_id: str, at: datetime, boot_at: datetime | None,
-                   last_seen_at: datetime | None) -> int:
+                   last_seen_at: datetime | None) -> HostDowntime | None:
     stop = (await session.execute(select(HostDowntime).where(
         HostDowntime.host_id == host_id, HostDowntime.kind.in_(ANNOUNCED),
         HostDowntime.ended_at.is_(None), HostDowntime.started_at <= at)
@@ -91,31 +100,34 @@ async def _started(session, host_id: str, at: datetime, boot_at: datetime | None
         # boot after it, or it stayed up while only the agent restarted.
         if boot_at is not None:
             stop.kind = "reboot" if boot_at > stop.started_at else "agent_stop"
-        return 1
+        return stop
     if boot_at is None or last_seen_at is None or boot_at <= last_seen_at:
-        return 0
+        return None
     seen = (await session.execute(select(HostDowntime.id).where(
         HostDowntime.host_id == host_id, HostDowntime.kind == "unclean",
         HostDowntime.ended_at == at))).first()
     if seen:
-        return 0
-    session.add(HostDowntime(host_id=host_id, kind="unclean", started_at=last_seen_at, ended_at=at))
-    return 1
+        return None
+    row = HostDowntime(host_id=host_id, kind="unclean", started_at=last_seen_at, ended_at=at)
+    session.add(row)
+    return row
 
 
-async def _lost_contact(session, host_id: str, since: datetime, until: datetime | None) -> int:
+async def _lost_contact(session, host_id: str, since: datetime,
+                        until: datetime | None) -> HostDowntime | None:
     if until is None or until <= since:
-        return 0
+        return None
     row = (await session.execute(select(HostDowntime).where(
         HostDowntime.host_id == host_id, HostDowntime.kind == "lost_contact",
         HostDowntime.started_at == since))).scalar_one_or_none()
     if row is None:
-        session.add(HostDowntime(host_id=host_id, kind="lost_contact", started_at=since, ended_at=until))
-        return 1
+        row = HostDowntime(host_id=host_id, kind="lost_contact", started_at=since, ended_at=until)
+        session.add(row)
+        return row
     if row.ended_at is None or until > row.ended_at:
         row.ended_at = until
-        return 1
-    return 0
+        return row
+    return None
 
 
 async def announced_stops(session, now: datetime, grace_seconds: int) -> dict[str, datetime]:

plugins/host_agent/maintenance.py

@@ -354,6 +354,17 @@ async def _host_name(session, host_id: str) -> str:
     return host.name if host is not None else host_id
 
 
+async def _moved(session, job, now: datetime) -> None:
+    """After a job moved: its window follows it, and once it ends it goes on
+    the timeline."""
+    host_name = await _host_name(session, job.host_id)
+    await _window(session, job, host_name, now)
+    if job.state in FINISHED:
+        from . import timeline
+        await timeline.write(session, f"maintenance job {job.id}",
+                             timeline.job_entry(session, job, host_name))
+
+
 async def apply_events(session, host_id: str, events, now: datetime) -> int:
     """Apply one post's `maintenance` results and `started` boots to the jobs
     they belong to; returns how many jobs changed. A result is matched by its
@@ -382,7 +393,7 @@ async def apply_events(session, host_id: str, events, now: datetime) -> int:
                 if apply_boot(job, boot_at, now):
                     changed.add(job)
     for job in changed:
-        await _window(session, job, await _host_name(session, job.host_id), now)
+        await _moved(session, job, now)
     return len(changed)
 
 
@@ -440,7 +451,7 @@ async def tick(session, now: datetime) -> int:
             gave_up += 1
             moved = True
         if moved:
-            await _window(session, job, await _host_name(session, job.host_id), now)
+            await _moved(session, job, now)
     return gave_up
 
 

plugins/host_agent/mcp_tools.py

@@ -15,9 +15,8 @@ from steward.mcp.tools.overview import finding
 from steward.models.hosts import Host
 from steward.models.users import UserRole
 
-from . import cadence, downtime, maintenance, senders
-from .models import (HostAgentRegistration, HostDowntime, HostMaintenanceJob, HostMaintenancePlan,
-                     HostMaintenanceRun)
+from . import cadence, maintenance, senders
+from .models import HostAgentRegistration, HostMaintenanceJob, HostMaintenancePlan, HostMaintenanceRun
 
 # A registration this old that has never checked in is a failed install, not one in progress.
 PENDING_GRACE = timedelta(minutes=30)
@@ -354,31 +353,6 @@ async def mcp_overview(db) -> list[dict]:
     return out
 
 
-async def mcp_events(db, host_id: str, host_name: str, since, until) -> list[dict]:
-    """What the host agent knows happened on a host in a window (get_anomaly):
-    maintenance jobs, and downtime (reboots, agent stops, lost contact)."""
-    from steward.mcp.tools._events import event
-    out = []
-    jobs = (await db.execute(select(HostMaintenanceJob).where(
-        HostMaintenanceJob.host_id == host_id, HostMaintenanceJob.created_at <= until,
-        (HostMaintenanceJob.finished_at.is_(None)) | (HostMaintenanceJob.finished_at >= since))
-        .order_by(HostMaintenanceJob.created_at).limit(200))).scalars()
-    for job in jobs:
-        out.append(event(job.created_at, "maintenance", f"Maintenance: {maintenance.summary(job)}",
-                         until=job.finished_at, ids={"job_id": job.id}))
-    now = now_utc()
-    rows = (await db.execute(select(HostDowntime).where(
-        HostDowntime.host_id == host_id, HostDowntime.started_at <= until,
-        (HostDowntime.ended_at.is_(None)) | (HostDowntime.ended_at >= since))
-        .order_by(HostDowntime.started_at).limit(200))).scalars()
-    for row in rows:
-        seen = downtime.describe(row, now)
-        out.append(event(row.started_at, "downtime",
-                         f"{seen['label']} ({int(seen['seconds'])} s{', still going' if seen['ongoing'] else ''})",
-                         until=row.ended_at))
-    return out
-
-
 def register_tools(register) -> None:
     register(list_agents, role=UserRole.viewer, read_only=True)
     register(list_maintenance_plans, role=UserRole.viewer, read_only=True)

plugins/host_agent/timeline.py

@@ -0,0 +1,105 @@
+# plugins/host_agent/timeline.py
+"""What the host agent puts on the timeline: each maintenance job once it
+ends, and each stretch a host was away (a reboot, a crash, lost contact, the
+agent restarting). One entry per job and per downtime row, kept up to date
+as the row changes: a downtime that ends gets its length, a failed job
+whose host came back late reads as back.
+"""
+from __future__ import annotations
+
+import logging
+
+from sqlalchemy import select
+
+from steward.timeline import catalog
+from steward.timeline.record import record_or_update, username
+
+from . import downtime, maintenance
+
+logger = logging.getLogger(__name__)
+
+TYPES = (
+    ("maintenance.done", "Maintenance run", "automation", "host_os", True,
+     "A host's agent updated it, and rebooted it when the update needed one."),
+    ("maintenance.failed", "Maintenance failed", "automation", "host_os", True,
+     "A maintenance job failed or timed out."),
+    ("host.rebooted", "Host rebooted", "observed", "host_os", True,
+     "A host went down and came back, announced or not (a crash, a power cut)."),
+    ("host.lost_contact", "Agent lost contact", "observed", "network", True,
+     "A host's agent ran but couldn't reach Steward."),
+    ("host.agent_restarted", "Agent stopped or restarted", "observed", "host_os", False,
+     "The agent stopped or restarted while its host stayed up."),
+)
+
+
+def register() -> None:
+    for t in TYPES:
+        catalog.register_type(*t)
+    from .models import HostDowntime, HostMaintenanceJob
+    catalog.register_ref("maintenance_job", HostMaintenanceJob)
+    catalog.register_ref("host_downtime", HostDowntime)
+
+
+async def _host_name(session, host_id: str) -> str | None:
+    from steward.models.hosts import Host
+    host = await session.get(Host, host_id)
+    return host.name if host is not None else None
+
+
+async def job_entry(session, job, host_name: str | None = None):
+    """The entry for a finished maintenance job, written or brought up to date."""
+    if job.state not in maintenance.FINISHED:
+        return None
+    host_name = host_name or await _host_name(session, job.host_id) or job.host_id
+    return await record_or_update(
+        session, ("maintenance_job", job.id),
+        type="maintenance.failed" if maintenance.still_failed(job) else "maintenance.done",
+        at=job.created_at, until=job.finished_at,
+        summary=f"Maintenance on {host_name}: {maintenance.summary(job)}",
+        hosts=[host_name], subject=host_name,
+        detail={"state": job.state, "packages": job.packages, "rebooted": job.back_at is not None,
+                "note": job.detail, "url": f"/hosts/{job.host_id}"},
+        actor=await username(session, job.created_by) or "Steward")
+
+
+_DOWNTIME_TYPES = {"reboot": "host.rebooted", "unclean": "host.rebooted",
+                   "lost_contact": "host.lost_contact", "agent_stop": "host.agent_restarted"}
+
+
+async def downtime_entry(session, row, host_name: str | None = None):
+    """The entry for one stretch a host was away, written or brought up to date."""
+    from steward.core.formatting import human_duration
+    host_name = host_name or await _host_name(session, row.host_id) or row.host_id
+    seen = downtime.describe(row, row.ended_at or row.started_at)
+    said = seen["label"][:1].lower() + seen["label"][1:]
+    said += f" for {human_duration(seen['seconds'])}" if row.ended_at is not None else ", not back yet"
+    return await record_or_update(
+        session, ("host_downtime", row.id),
+        type=_DOWNTIME_TYPES.get(row.kind, "host.rebooted"),
+        at=row.started_at, until=row.ended_at, summary=f"{host_name}: {said}",
+        hosts=[host_name], subject=host_name,
+        detail={"kind": row.kind, "url": f"/hosts/{row.host_id}"}, actor="Steward")
+
+
+async def write(session, what: str, coro) -> None:
+    """Run one entry writer in a savepoint: the source's own row matters more
+    than its entry, so a failure is logged and nothing else is undone."""
+    try:
+        async with session.begin_nested():
+            await coro
+    except Exception:
+        logger.exception("Timeline: the entry for %s could not be written", what)
+
+
+async def backfill(session, since) -> list:
+    """Entries for the jobs and downtime that predate the timeline."""
+    from .models import HostDowntime, HostMaintenanceJob
+    out = []
+    for job in (await session.execute(select(HostMaintenanceJob).where(
+            HostMaintenanceJob.created_at >= since,
+            HostMaintenanceJob.state.in_(maintenance.FINISHED)))).scalars().all():
+        out.append(await job_entry(session, job))
+    for row in (await session.execute(select(HostDowntime).where(
+            HostDowntime.started_at >= since))).scalars().all():
+        out.append(await downtime_entry(session, row))
+    return out