feat(timeline): maintenance, reboots, deploys, crash-loops and reviewed anomalies on the timeline (#6349)

The host agent writes each maintenance job once it ends, and each stretch a
host was away (a reboot, a crash, lost contact, the agent restarting), kept up
to date until it is back. Docker judges each snapshot: a unit on a new image,
or a container recreated, is a deploy; deaths past the Thresholds crash-loop
count are one crash-loop entry that later deaths extend; single restarts are
recorded but hidden. An anomaly has one entry whose type follows its review:
only those judged real are shown by default.

A source whose record changes keeps one entry for it (record_or_update).
Plugins register the records their entries point to, so an aged-out job or
downtime reads "Details no longer kept". The first fill runs once per source
(timeline.filled), the host agent's jobs and downtime included.

get_anomaly's "what else was going on" reads the timeline for the host or
resource instead of querying alerts, runs, maintenance, downtime and container
deaths itself; Docker's own source keeps only starts, stops and health
changes. A new kind, Observed, is for what Steward saw happen with no rule or
person behind it.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

plugins/docker/__init__.py

@@ -52,6 +52,9 @@ def setup(app: "Quart") -> None:
     # A host's agent carries its containers' figures: they go silent with it (#5721).
     from steward.core.alerts import agent_fed
     agent_fed("docker", lambda resource: resource.split("/", 1)[0])
+    # Deploys and crash-loops go on the timeline.
+    from .timeline import register as register_timeline
+    register_timeline()
 
 
 def get_scheduled_tasks() -> list:

plugins/docker/ingest.py

@@ -56,6 +56,15 @@ def net_rates(before: dict, snapshots: list) -> list:
     return out
 
 
+def _crash_loop_deaths() -> int:
+    from quart import current_app
+    from .service import CRASH_LOOP_DEATHS, crash_loop_deaths
+    try:
+        return crash_loop_deaths(current_app.config)
+    except RuntimeError:                    # outside the app (a unit test)
+        return CRASH_LOOP_DEATHS
+
+
 def _derive_events(old_state: dict, new_containers: list) -> list:
     """Diff a fresh snapshot against stored per-container state → lifecycle events.
 
@@ -460,12 +469,19 @@ async def persist_host_docker(session, host, snapshots, swarm=None, disk=None,
         # the sample that noticed it was taken.
         finished = {c.get("name"): _parse_started_at(c.get("finished_at"))
                     for c in latest_containers}
+        derived = []
         for name, event, detail in _derive_events(old_state, latest_containers):
             at = finished.get(name) if event in ("die", "stop") else None
             session.add(DockerEvent(
                 host_id=host.id, container_name=name,
                 event=event, detail=detail, at=at or latest_at,
             ))
+            derived.append((name, event, detail, at or latest_at))
+        # Deploys and crash-loops go on the timeline, judged against the
+        # containers as they were before this snapshot.
+        from . import timeline
+        await timeline.from_snapshot(session, host, old_rows, latest_containers, derived, latest_at,
+                                     _crash_loop_deaths())
     await mark("events")
 
     # Current state, and the restart and health alerts, from the newest snapshot only.

plugins/docker/mcp_tools.py

@@ -750,13 +750,18 @@ async def prune_docker(host: str, what: str, check: bool = True) -> dict:
             "next": f'wait_for_run(run_id="{run.id}")'}
 
 
+# Deploys, restarts, deaths and OOM kills are on the timeline (timeline.py),
+# which get_anomaly reads; these are the container events it doesn't hold.
+_UNTIMELINED = ("start", "stop", "health_change")
+
+
 async def mcp_events(db, host_id: str, host_name: str, since, until) -> list[dict]:
-    """Container lifecycle events on a host in a window (get_anomaly): starts
-    (with the image, so a deploy shows), deaths with their exit, OOM kills,
-    restarts and health changes."""
+    """Container events on a host in a window (get_anomaly) that aren't on
+    the timeline: starts, stops and health changes."""
     from steward.mcp.tools._events import event
     rows = (await db.execute(select(DockerEvent).where(
-        DockerEvent.host_id == host_id, DockerEvent.at >= since, DockerEvent.at <= until)
+        DockerEvent.host_id == host_id, DockerEvent.event.in_(_UNTIMELINED),
+        DockerEvent.at >= since, DockerEvent.at <= until)
         .order_by(DockerEvent.at).limit(500))).scalars()
     return [event(e.at, "container", f"{e.container_name} {e.event}" + (f" ({e.detail})" if e.detail else ""),
                   subject=f"{host_name}/{e.container_name}") for e in rows]

plugins/docker/timeline.py

@@ -0,0 +1,160 @@
+# plugins/docker/timeline.py
+"""What Docker puts on the timeline, judged from each snapshot as it is
+ingested: a deploy (a container or swarm service running a new image, or a
+container recreated), and a crash-loop as one entry for the whole episode.
+A single restart or death is recorded but hidden. As on the Docker pages, a
+crash loop is Settings → Thresholds' number of deaths within CRASH_WINDOW:
+the one that reaches it opens a crash-loop entry, which each later one
+extends until the container has been quiet for CRASH_WINDOW.
+
+A container belongs to its swarm service when it has one, so a service's
+tasks, replaced on every deploy and every crash, read as one subject.
+"""
+from __future__ import annotations
+
+import logging
+from datetime import timedelta
+
+from sqlalchemy import func, select
+
+from steward.models.timeline import TimelineEntry
+from steward.timeline import catalog
+from steward.timeline.record import record
+
+logger = logging.getLogger(__name__)
+
+TYPES = (
+    ("docker.deployed", "Deployed", "change", "service", True,
+     "A container or swarm service started on a new image, or a container was recreated."),
+    ("docker.crash_loop", "Crash-looping", "observed", "service", True,
+     "A container kept restarting or dying: one entry for the whole episode."),
+    ("docker.restarted", "Container restarted or died", "observed", "service", False,
+     "A single restart, death or out-of-memory kill."),
+)
+
+CRASH_WINDOW = timedelta(hours=1)
+_TROUBLE = ("restart", "die", "oom")
+
+
+def register() -> None:
+    for t in TYPES:
+        catalog.register_type(*t)
+
+
+def unit(c: dict) -> str:
+    """What a container is part of: its swarm service, else itself."""
+    return c.get("service_name") or c.get("name")
+
+
+def short_image(image: str | None) -> str:
+    """An image as a person reads it: the name and tag, and a few characters
+    of the digest when there is one."""
+    if not image:
+        return "an unknown image"
+    name, _, digest = image.partition("@")
+    name = name.rsplit("/", 1)[-1]
+    if digest:
+        return f"{name} ({digest.split(':', 1)[-1][:12]})"
+    return name
+
+
+def deploys(old_rows: list, latest: list[dict]) -> list[dict]:
+    """Each unit whose containers now run an image they didn't, or a
+    container recreated under the same name: {unit, name, image, was,
+    recreated, swarm}. A swarm task landing on a host that ran none of its
+    service before is the scheduler moving it, not a deploy."""
+    images: dict[str, set] = {}
+    ids = {}
+    for r in old_rows:
+        images.setdefault(r.service_name or r.name, set()).add(r.image)
+        ids[r.name] = (r.container_id, r.image)
+    out, seen = [], set()
+    for c in latest:
+        if (c.get("status") or "").lower() != "running" or not c.get("name"):
+            continue
+        key, image = unit(c), c.get("image") or ""
+        if key in seen:
+            continue
+        before = images.get(key)
+        old_id, old_image = ids.get(c["name"], (None, None))
+        if before and image not in before:
+            was = old_image if old_image and old_image != image else sorted(before)[0]
+            out.append({"unit": key, "name": c["name"], "image": image, "was": was,
+                        "recreated": False, "swarm": bool(c.get("service_name"))})
+        elif old_id and c.get("container_id") and old_id != c["container_id"] and not c.get("service_name"):
+            out.append({"unit": key, "name": c["name"], "image": image, "was": old_image,
+                        "recreated": True, "swarm": False})
+        elif before is None and not c.get("service_name"):
+            out.append({"unit": key, "name": c["name"], "image": image, "was": None,
+                        "recreated": False, "swarm": False})
+        else:
+            continue
+        seen.add(key)
+    return out
+
+
+def _deploy_summary(d: dict, host_name: str) -> str:
+    what = f"{d['unit']} on {host_name}"
+    if d["recreated"]:
+        return f"{what}: recreated on {short_image(d['image'])}"
+    if d["was"] is None:
+        return f"{what}: started on {short_image(d['image'])}"
+    new, old = short_image(d["image"]), short_image(d["was"])
+    if d["image"].split("@")[0] == (d["was"] or "").split("@")[0]:
+        return f"{what}: new build of {new}"
+    return f"{what}: {old} → {new}"
+
+
+async def _trouble(session, host, key: str, name: str, at, said: str, deaths: int) -> None:
+    """One restart, death or OOM kill of `key`: extends its crash-loop, opens
+    one when it is the `deaths`-th within CRASH_WINDOW, else a hidden entry."""
+    subject = f"{host.name}/{key}"
+    loop = (await session.execute(select(TimelineEntry).where(
+        TimelineEntry.type == "docker.crash_loop", TimelineEntry.subject == subject,
+        TimelineEntry.until >= at - CRASH_WINDOW).order_by(TimelineEntry.at.desc()).limit(1))
+    ).scalar_one_or_none()
+    if loop is not None:
+        count = (loop.detail or {}).get("count", 0) + 1
+        loop.until = max(loop.until, at)
+        loop.detail = {**(loop.detail or {}), "count": count, "last": said}
+        loop.summary = f"{key} on {host.name} crash-looping: {count} restarts or deaths"
+        return
+    recent = (await session.execute(select(func.count(), func.min(TimelineEntry.at)).where(
+        TimelineEntry.type == "docker.restarted", TimelineEntry.subject == subject,
+        TimelineEntry.at >= at - CRASH_WINDOW))).one()
+    url = f"/plugins/docker/container/{host.id}/{name}"
+    if recent[0] + 1 >= deaths:
+        count = recent[0] + 1
+        record(session, type="docker.crash_loop", at=recent[1] or at, until=at,
+               summary=f"{key} on {host.name} crash-looping: {count} restarts or deaths",
+               hosts=[host.name], subject=subject, actor="Docker",
+               detail={"count": count, "last": said, "url": url})
+        return
+    record(session, type="docker.restarted", at=at, summary=f"{key} on {host.name}: {said}",
+           hosts=[host.name], subject=subject, actor="Docker", detail={"url": url})
+
+
+_SAID = {"restart": "restarted", "die": "died", "oom": "killed, out of memory"}
+
+
+async def from_snapshot(session, host, old_rows: list, latest: list[dict], events: list, at,
+                        deaths: int) -> None:
+    """The entries one snapshot makes: its deploys, and its restarts and
+    deaths (`events`: (name, event, detail, when)), in a savepoint so a
+    failure here never loses the snapshot. `deaths` makes a crash loop."""
+    try:
+        async with session.begin_nested():
+            for d in deploys(old_rows, latest):
+                url = "/plugins/docker/swarm" if d["swarm"] else f"/plugins/docker/container/{host.id}/{d['name']}"
+                record(session, type="docker.deployed", at=at, summary=_deploy_summary(d, host.name),
+                       hosts=[host.name], subject=f"{host.name}/{d['unit']}", actor="Docker",
+                       detail={"image": d["image"], "was": d["was"], "url": url})
+            units = {c.get("name"): unit(c) for c in latest if c.get("name")}
+            units.update({r.name: r.service_name or r.name for r in old_rows if r.name not in units})
+            for name, event, detail, when in events:
+                if event in _TROUBLE:
+                    said = _SAID[event] + (f" ({detail})" if detail else "")
+                    await _trouble(session, host, units.get(name, name), name, when or at, said, deaths)
+                    await session.flush()
+    except Exception:
+        logger.exception("Docker on %s: timeline entries could not be written", host.name)