feat(timeline): maintenance, reboots, deploys, crash-loops and reviewed anomalies on the timeline (#6349)

The host agent writes each maintenance job once it ends, and each stretch a
host was away (a reboot, a crash, lost contact, the agent restarting), kept up
to date until it is back. Docker judges each snapshot: a unit on a new image,
or a container recreated, is a deploy; deaths past the Thresholds crash-loop
count are one crash-loop entry that later deaths extend; single restarts are
recorded but hidden. An anomaly has one entry whose type follows its review:
only those judged real are shown by default.

A source whose record changes keeps one entry for it (record_or_update).
Plugins register the records their entries point to, so an aged-out job or
downtime reads "Details no longer kept". The first fill runs once per source
(timeline.filled), the host agent's jobs and downtime included.

get_anomaly's "what else was going on" reads the timeline for the host or
resource instead of querying alerts, runs, maintenance, downtime and container
deaths itself; Docker's own source keeps only starts, stops and health
changes. A new kind, Observed, is for what Steward saw happen with no rule or
person behind it.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

plugins/docker/__init__.py

@@ -52,6 +52,9 @@ def setup(app: "Quart") -> None:
     # A host's agent carries its containers' figures: they go silent with it (#5721).
     from steward.core.alerts import agent_fed
     agent_fed("docker", lambda resource: resource.split("/", 1)[0])
+    # Deploys and crash-loops go on the timeline.
+    from .timeline import register as register_timeline
+    register_timeline()
 
 
 def get_scheduled_tasks() -> list:

plugins/docker/ingest.py

@@ -56,6 +56,15 @@ def net_rates(before: dict, snapshots: list) -> list:
     return out
 
 
+def _crash_loop_deaths() -> int:
+    from quart import current_app
+    from .service import CRASH_LOOP_DEATHS, crash_loop_deaths
+    try:
+        return crash_loop_deaths(current_app.config)
+    except RuntimeError:                    # outside the app (a unit test)
+        return CRASH_LOOP_DEATHS
+
+
 def _derive_events(old_state: dict, new_containers: list) -> list:
     """Diff a fresh snapshot against stored per-container state → lifecycle events.
 
@@ -460,12 +469,19 @@ async def persist_host_docker(session, host, snapshots, swarm=None, disk=None,
         # the sample that noticed it was taken.
         finished = {c.get("name"): _parse_started_at(c.get("finished_at"))
                     for c in latest_containers}
+        derived = []
         for name, event, detail in _derive_events(old_state, latest_containers):
             at = finished.get(name) if event in ("die", "stop") else None
             session.add(DockerEvent(
                 host_id=host.id, container_name=name,
                 event=event, detail=detail, at=at or latest_at,
             ))
+            derived.append((name, event, detail, at or latest_at))
+        # Deploys and crash-loops go on the timeline, judged against the
+        # containers as they were before this snapshot.
+        from . import timeline
+        await timeline.from_snapshot(session, host, old_rows, latest_containers, derived, latest_at,
+                                     _crash_loop_deaths())
     await mark("events")
 
     # Current state, and the restart and health alerts, from the newest snapshot only.

plugins/docker/mcp_tools.py

@@ -750,13 +750,18 @@ async def prune_docker(host: str, what: str, check: bool = True) -> dict:
             "next": f'wait_for_run(run_id="{run.id}")'}
 
 
+# Deploys, restarts, deaths and OOM kills are on the timeline (timeline.py),
+# which get_anomaly reads; these are the container events it doesn't hold.
+_UNTIMELINED = ("start", "stop", "health_change")
+
+
 async def mcp_events(db, host_id: str, host_name: str, since, until) -> list[dict]:
-    """Container lifecycle events on a host in a window (get_anomaly): starts
-    (with the image, so a deploy shows), deaths with their exit, OOM kills,
-    restarts and health changes."""
+    """Container events on a host in a window (get_anomaly) that aren't on
+    the timeline: starts, stops and health changes."""
     from steward.mcp.tools._events import event
     rows = (await db.execute(select(DockerEvent).where(
-        DockerEvent.host_id == host_id, DockerEvent.at >= since, DockerEvent.at <= until)
+        DockerEvent.host_id == host_id, DockerEvent.event.in_(_UNTIMELINED),
+        DockerEvent.at >= since, DockerEvent.at <= until)
         .order_by(DockerEvent.at).limit(500))).scalars()
     return [event(e.at, "container", f"{e.container_name} {e.event}" + (f" ({e.detail})" if e.detail else ""),
                   subject=f"{host_name}/{e.container_name}") for e in rows]

plugins/docker/timeline.py

@@ -0,0 +1,160 @@
+# plugins/docker/timeline.py
+"""What Docker puts on the timeline, judged from each snapshot as it is
+ingested: a deploy (a container or swarm service running a new image, or a
+container recreated), and a crash-loop as one entry for the whole episode.
+A single restart or death is recorded but hidden. As on the Docker pages, a
+crash loop is Settings → Thresholds' number of deaths within CRASH_WINDOW:
+the one that reaches it opens a crash-loop entry, which each later one
+extends until the container has been quiet for CRASH_WINDOW.
+
+A container belongs to its swarm service when it has one, so a service's
+tasks, replaced on every deploy and every crash, read as one subject.
+"""
+from __future__ import annotations
+
+import logging
+from datetime import timedelta
+
+from sqlalchemy import func, select
+
+from steward.models.timeline import TimelineEntry
+from steward.timeline import catalog
+from steward.timeline.record import record
+
+logger = logging.getLogger(__name__)
+
+TYPES = (
+    ("docker.deployed", "Deployed", "change", "service", True,
+     "A container or swarm service started on a new image, or a container was recreated."),
+    ("docker.crash_loop", "Crash-looping", "observed", "service", True,
+     "A container kept restarting or dying: one entry for the whole episode."),
+    ("docker.restarted", "Container restarted or died", "observed", "service", False,
+     "A single restart, death or out-of-memory kill."),
+)
+
+CRASH_WINDOW = timedelta(hours=1)
+_TROUBLE = ("restart", "die", "oom")
+
+
+def register() -> None:
+    for t in TYPES:
+        catalog.register_type(*t)
+
+
+def unit(c: dict) -> str:
+    """What a container is part of: its swarm service, else itself."""
+    return c.get("service_name") or c.get("name")
+
+
+def short_image(image: str | None) -> str:
+    """An image as a person reads it: the name and tag, and a few characters
+    of the digest when there is one."""
+    if not image:
+        return "an unknown image"
+    name, _, digest = image.partition("@")
+    name = name.rsplit("/", 1)[-1]
+    if digest:
+        return f"{name} ({digest.split(':', 1)[-1][:12]})"
+    return name
+
+
+def deploys(old_rows: list, latest: list[dict]) -> list[dict]:
+    """Each unit whose containers now run an image they didn't, or a
+    container recreated under the same name: {unit, name, image, was,
+    recreated, swarm}. A swarm task landing on a host that ran none of its
+    service before is the scheduler moving it, not a deploy."""
+    images: dict[str, set] = {}
+    ids = {}
+    for r in old_rows:
+        images.setdefault(r.service_name or r.name, set()).add(r.image)
+        ids[r.name] = (r.container_id, r.image)
+    out, seen = [], set()
+    for c in latest:
+        if (c.get("status") or "").lower() != "running" or not c.get("name"):
+            continue
+        key, image = unit(c), c.get("image") or ""
+        if key in seen:
+            continue
+        before = images.get(key)
+        old_id, old_image = ids.get(c["name"], (None, None))
+        if before and image not in before:
+            was = old_image if old_image and old_image != image else sorted(before)[0]
+            out.append({"unit": key, "name": c["name"], "image": image, "was": was,
+                        "recreated": False, "swarm": bool(c.get("service_name"))})
+        elif old_id and c.get("container_id") and old_id != c["container_id"] and not c.get("service_name"):
+            out.append({"unit": key, "name": c["name"], "image": image, "was": old_image,
+                        "recreated": True, "swarm": False})
+        elif before is None and not c.get("service_name"):
+            out.append({"unit": key, "name": c["name"], "image": image, "was": None,
+                        "recreated": False, "swarm": False})
+        else:
+            continue
+        seen.add(key)
+    return out
+
+
+def _deploy_summary(d: dict, host_name: str) -> str:
+    what = f"{d['unit']} on {host_name}"
+    if d["recreated"]:
+        return f"{what}: recreated on {short_image(d['image'])}"
+    if d["was"] is None:
+        return f"{what}: started on {short_image(d['image'])}"
+    new, old = short_image(d["image"]), short_image(d["was"])
+    if d["image"].split("@")[0] == (d["was"] or "").split("@")[0]:
+        return f"{what}: new build of {new}"
+    return f"{what}: {old} → {new}"
+
+
+async def _trouble(session, host, key: str, name: str, at, said: str, deaths: int) -> None:
+    """One restart, death or OOM kill of `key`: extends its crash-loop, opens
+    one when it is the `deaths`-th within CRASH_WINDOW, else a hidden entry."""
+    subject = f"{host.name}/{key}"
+    loop = (await session.execute(select(TimelineEntry).where(
+        TimelineEntry.type == "docker.crash_loop", TimelineEntry.subject == subject,
+        TimelineEntry.until >= at - CRASH_WINDOW).order_by(TimelineEntry.at.desc()).limit(1))
+    ).scalar_one_or_none()
+    if loop is not None:
+        count = (loop.detail or {}).get("count", 0) + 1
+        loop.until = max(loop.until, at)
+        loop.detail = {**(loop.detail or {}), "count": count, "last": said}
+        loop.summary = f"{key} on {host.name} crash-looping: {count} restarts or deaths"
+        return
+    recent = (await session.execute(select(func.count(), func.min(TimelineEntry.at)).where(
+        TimelineEntry.type == "docker.restarted", TimelineEntry.subject == subject,
+        TimelineEntry.at >= at - CRASH_WINDOW))).one()
+    url = f"/plugins/docker/container/{host.id}/{name}"
+    if recent[0] + 1 >= deaths:
+        count = recent[0] + 1
+        record(session, type="docker.crash_loop", at=recent[1] or at, until=at,
+               summary=f"{key} on {host.name} crash-looping: {count} restarts or deaths",
+               hosts=[host.name], subject=subject, actor="Docker",
+               detail={"count": count, "last": said, "url": url})
+        return
+    record(session, type="docker.restarted", at=at, summary=f"{key} on {host.name}: {said}",
+           hosts=[host.name], subject=subject, actor="Docker", detail={"url": url})
+
+
+_SAID = {"restart": "restarted", "die": "died", "oom": "killed, out of memory"}
+
+
+async def from_snapshot(session, host, old_rows: list, latest: list[dict], events: list, at,
+                        deaths: int) -> None:
+    """The entries one snapshot makes: its deploys, and its restarts and
+    deaths (`events`: (name, event, detail, when)), in a savepoint so a
+    failure here never loses the snapshot. `deaths` makes a crash loop."""
+    try:
+        async with session.begin_nested():
+            for d in deploys(old_rows, latest):
+                url = "/plugins/docker/swarm" if d["swarm"] else f"/plugins/docker/container/{host.id}/{d['name']}"
+                record(session, type="docker.deployed", at=at, summary=_deploy_summary(d, host.name),
+                       hosts=[host.name], subject=f"{host.name}/{d['unit']}", actor="Docker",
+                       detail={"image": d["image"], "was": d["was"], "url": url})
+            units = {c.get("name"): unit(c) for c in latest if c.get("name")}
+            units.update({r.name: r.service_name or r.name for r in old_rows if r.name not in units})
+            for name, event, detail, when in events:
+                if event in _TROUBLE:
+                    said = _SAID[event] + (f" ({detail})" if detail else "")
+                    await _trouble(session, host, units.get(name, name), name, when or at, said, deaths)
+                    await session.flush()
+    except Exception:
+        logger.exception("Docker on %s: timeline entries could not be written", host.name)

plugins/host_agent/__init__.py

@@ -34,6 +34,9 @@ def setup(app: "Quart") -> None:
     from steward.core.cleanup import prune_by_age
     from .models import HostDowntime
     prune_by_age(HostDowntime.started_at)
+    # Maintenance runs and the stretches a host was away go on the timeline.
+    from .timeline import register as register_timeline
+    register_timeline()
 
 
 def get_audit_entities() -> dict:
@@ -62,9 +65,9 @@ async def mcp_overview(db) -> list[dict]:
     return await findings(db)
 
 
-async def mcp_events(db, host_id: str, host_name: str, since, until) -> list[dict]:
-    from .mcp_tools import mcp_events as events
-    return await events(db, host_id, host_name, since, until)
+async def timeline_backfill(session, since) -> list:
+    from .timeline import backfill
+    return await backfill(session, since)
 
 
 def get_alert_metrics() -> dict[str, list[str]]:

plugins/host_agent/cadence.py

@@ -22,7 +22,7 @@ from __future__ import annotations
 import json
 from datetime import datetime, time, timedelta, timezone
 
-from . import maintenance
+from . import maintenance, timeline
 
 WEEKDAYS = ("Monday", "Tuesday", "Wednesday", "Thursday", "Friday", "Saturday", "Sunday")
 # How long a host may take to be back in service after its job finished.
@@ -241,6 +241,8 @@ async def advance(session, run, plan, now: datetime) -> str:
         if move == "fail":
             if job is not None and job.state == "done":
                 maintenance._finish(job, "failed", now, why)     # the host never came back into service
+                await timeline.write(session, f"maintenance job {job.id}",
+                                     timeline.job_entry(session, job, host.name if host else None))
             name = host.name if host else host_id
             _end(run, "failed", now, f"{name}: {why}")
         elif move == "stop":
@@ -325,5 +327,7 @@ async def failed_hosts(session, now: datetime) -> dict[str, float]:
                 HostAgentRegistration.host_id == host.id))).scalar_one_or_none()
             if await _back(session, host, job, reg, now) is None:
                 job.came_back_at = now
+                await timeline.write(session, f"maintenance job {job.id}",
+                                     timeline.job_entry(session, job, host.name))
         out[name] = 1.0 if maintenance.still_failed(job) else 0.0
     return out

plugins/host_agent/downtime.py

@@ -51,36 +51,45 @@ async def record_events(session, host_id: str, events, last_seen_at: datetime |
     """Apply one post's events to the host's downtime, oldest first; returns
     how many rows were added or changed. `last_seen_at` is when the agent was
     last heard before this post: a boot after it that nobody announced is a
-    restart without a shutdown."""
+    restart without a shutdown. Each row added or changed goes on the
+    timeline, kept up to date as it ends."""
     if not isinstance(events, list):
         return 0
     timed = sorted(((t, e) for e in events if isinstance(e, dict)
                     for t in [_when(e)] if t is not None), key=lambda te: te[0])
-    changed = 0
+    changed = []
     for at, event in timed:
         kind = event.get("kind")
+        row = None
         if kind == "stopping":
-            changed += await _stopping(session, host_id, at, event.get("host_shutdown"))
+            row = await _stopping(session, host_id, at, event.get("host_shutdown"))
         elif kind == "started":
-            changed += await _started(session, host_id, at, _at(event.get("boot_at")), last_seen_at)
+            row = await _started(session, host_id, at, _at(event.get("boot_at")), last_seen_at)
         elif kind == "lost_contact":
-            changed += await _lost_contact(session, host_id, at, _at(event.get("to")))
-    return changed
+            row = await _lost_contact(session, host_id, at, _at(event.get("to")))
+        if row is not None and row not in changed:
+            changed.append(row)
+    if changed:
+        from . import timeline
+        await session.flush()
+        for row in changed:
+            await timeline.write(session, f"host downtime {row.id}", timeline.downtime_entry(session, row))
+    return len(changed)
 
 
-async def _stopping(session, host_id: str, at: datetime, host_shutdown) -> int:
+async def _stopping(session, host_id: str, at: datetime, host_shutdown) -> HostDowntime | None:
     seen = (await session.execute(select(HostDowntime.id).where(
         HostDowntime.host_id == host_id, HostDowntime.kind.in_(ANNOUNCED),
         HostDowntime.started_at == at))).first()
     if seen:
-        return 0
-    session.add(HostDowntime(host_id=host_id, kind="reboot" if host_shutdown else "agent_stop",
-                             started_at=at))
-    return 1
+        return None
+    row = HostDowntime(host_id=host_id, kind="reboot" if host_shutdown else "agent_stop", started_at=at)
+    session.add(row)
+    return row
 
 
 async def _started(session, host_id: str, at: datetime, boot_at: datetime | None,
-                   last_seen_at: datetime | None) -> int:
+                   last_seen_at: datetime | None) -> HostDowntime | None:
     stop = (await session.execute(select(HostDowntime).where(
         HostDowntime.host_id == host_id, HostDowntime.kind.in_(ANNOUNCED),
         HostDowntime.ended_at.is_(None), HostDowntime.started_at <= at)
@@ -91,31 +100,34 @@ async def _started(session, host_id: str, at: datetime, boot_at: datetime | None
         # boot after it, or it stayed up while only the agent restarted.
         if boot_at is not None:
             stop.kind = "reboot" if boot_at > stop.started_at else "agent_stop"
-        return 1
+        return stop
     if boot_at is None or last_seen_at is None or boot_at <= last_seen_at:
-        return 0
+        return None
     seen = (await session.execute(select(HostDowntime.id).where(
         HostDowntime.host_id == host_id, HostDowntime.kind == "unclean",
         HostDowntime.ended_at == at))).first()
     if seen:
-        return 0
-    session.add(HostDowntime(host_id=host_id, kind="unclean", started_at=last_seen_at, ended_at=at))
-    return 1
+        return None
+    row = HostDowntime(host_id=host_id, kind="unclean", started_at=last_seen_at, ended_at=at)
+    session.add(row)
+    return row
 
 
-async def _lost_contact(session, host_id: str, since: datetime, until: datetime | None) -> int:
+async def _lost_contact(session, host_id: str, since: datetime,
+                        until: datetime | None) -> HostDowntime | None:
     if until is None or until <= since:
-        return 0
+        return None
     row = (await session.execute(select(HostDowntime).where(
         HostDowntime.host_id == host_id, HostDowntime.kind == "lost_contact",
         HostDowntime.started_at == since))).scalar_one_or_none()
     if row is None:
-        session.add(HostDowntime(host_id=host_id, kind="lost_contact", started_at=since, ended_at=until))
-        return 1
+        row = HostDowntime(host_id=host_id, kind="lost_contact", started_at=since, ended_at=until)
+        session.add(row)
+        return row
     if row.ended_at is None or until > row.ended_at:
         row.ended_at = until
-        return 1
-    return 0
+        return row
+    return None
 
 
 async def announced_stops(session, now: datetime, grace_seconds: int) -> dict[str, datetime]:

plugins/host_agent/maintenance.py

@@ -354,6 +354,17 @@ async def _host_name(session, host_id: str) -> str:
     return host.name if host is not None else host_id
 
 
+async def _moved(session, job, now: datetime) -> None:
+    """After a job moved: its window follows it, and once it ends it goes on
+    the timeline."""
+    host_name = await _host_name(session, job.host_id)
+    await _window(session, job, host_name, now)
+    if job.state in FINISHED:
+        from . import timeline
+        await timeline.write(session, f"maintenance job {job.id}",
+                             timeline.job_entry(session, job, host_name))
+
+
 async def apply_events(session, host_id: str, events, now: datetime) -> int:
     """Apply one post's `maintenance` results and `started` boots to the jobs
     they belong to; returns how many jobs changed. A result is matched by its
@@ -382,7 +393,7 @@ async def apply_events(session, host_id: str, events, now: datetime) -> int:
                 if apply_boot(job, boot_at, now):
                     changed.add(job)
     for job in changed:
-        await _window(session, job, await _host_name(session, job.host_id), now)
+        await _moved(session, job, now)
     return len(changed)
 
 
@@ -440,7 +451,7 @@ async def tick(session, now: datetime) -> int:
             gave_up += 1
             moved = True
         if moved:
-            await _window(session, job, await _host_name(session, job.host_id), now)
+            await _moved(session, job, now)
     return gave_up
 
 

plugins/host_agent/mcp_tools.py

@@ -15,9 +15,8 @@ from steward.mcp.tools.overview import finding
 from steward.models.hosts import Host
 from steward.models.users import UserRole
 
-from . import cadence, downtime, maintenance, senders
-from .models import (HostAgentRegistration, HostDowntime, HostMaintenanceJob, HostMaintenancePlan,
-                     HostMaintenanceRun)
+from . import cadence, maintenance, senders
+from .models import HostAgentRegistration, HostMaintenanceJob, HostMaintenancePlan, HostMaintenanceRun
 
 # A registration this old that has never checked in is a failed install, not one in progress.
 PENDING_GRACE = timedelta(minutes=30)
@@ -354,31 +353,6 @@ async def mcp_overview(db) -> list[dict]:
     return out
 
 
-async def mcp_events(db, host_id: str, host_name: str, since, until) -> list[dict]:
-    """What the host agent knows happened on a host in a window (get_anomaly):
-    maintenance jobs, and downtime (reboots, agent stops, lost contact)."""
-    from steward.mcp.tools._events import event
-    out = []
-    jobs = (await db.execute(select(HostMaintenanceJob).where(
-        HostMaintenanceJob.host_id == host_id, HostMaintenanceJob.created_at <= until,
-        (HostMaintenanceJob.finished_at.is_(None)) | (HostMaintenanceJob.finished_at >= since))
-        .order_by(HostMaintenanceJob.created_at).limit(200))).scalars()
-    for job in jobs:
-        out.append(event(job.created_at, "maintenance", f"Maintenance: {maintenance.summary(job)}",
-                         until=job.finished_at, ids={"job_id": job.id}))
-    now = now_utc()
-    rows = (await db.execute(select(HostDowntime).where(
-        HostDowntime.host_id == host_id, HostDowntime.started_at <= until,
-        (HostDowntime.ended_at.is_(None)) | (HostDowntime.ended_at >= since))
-        .order_by(HostDowntime.started_at).limit(200))).scalars()
-    for row in rows:
-        seen = downtime.describe(row, now)
-        out.append(event(row.started_at, "downtime",
-                         f"{seen['label']} ({int(seen['seconds'])} s{', still going' if seen['ongoing'] else ''})",
-                         until=row.ended_at))
-    return out
-
-
 def register_tools(register) -> None:
     register(list_agents, role=UserRole.viewer, read_only=True)
     register(list_maintenance_plans, role=UserRole.viewer, read_only=True)

plugins/host_agent/timeline.py

@@ -0,0 +1,105 @@
+# plugins/host_agent/timeline.py
+"""What the host agent puts on the timeline: each maintenance job once it
+ends, and each stretch a host was away (a reboot, a crash, lost contact, the
+agent restarting). One entry per job and per downtime row, kept up to date
+as the row changes: a downtime that ends gets its length, a failed job
+whose host came back late reads as back.
+"""
+from __future__ import annotations
+
+import logging
+
+from sqlalchemy import select
+
+from steward.timeline import catalog
+from steward.timeline.record import record_or_update, username
+
+from . import downtime, maintenance
+
+logger = logging.getLogger(__name__)
+
+TYPES = (
+    ("maintenance.done", "Maintenance run", "automation", "host_os", True,
+     "A host's agent updated it, and rebooted it when the update needed one."),
+    ("maintenance.failed", "Maintenance failed", "automation", "host_os", True,
+     "A maintenance job failed or timed out."),
+    ("host.rebooted", "Host rebooted", "observed", "host_os", True,
+     "A host went down and came back, announced or not (a crash, a power cut)."),
+    ("host.lost_contact", "Agent lost contact", "observed", "network", True,
+     "A host's agent ran but couldn't reach Steward."),
+    ("host.agent_restarted", "Agent stopped or restarted", "observed", "host_os", False,
+     "The agent stopped or restarted while its host stayed up."),
+)
+
+
+def register() -> None:
+    for t in TYPES:
+        catalog.register_type(*t)
+    from .models import HostDowntime, HostMaintenanceJob
+    catalog.register_ref("maintenance_job", HostMaintenanceJob)
+    catalog.register_ref("host_downtime", HostDowntime)
+
+
+async def _host_name(session, host_id: str) -> str | None:
+    from steward.models.hosts import Host
+    host = await session.get(Host, host_id)
+    return host.name if host is not None else None
+
+
+async def job_entry(session, job, host_name: str | None = None):
+    """The entry for a finished maintenance job, written or brought up to date."""
+    if job.state not in maintenance.FINISHED:
+        return None
+    host_name = host_name or await _host_name(session, job.host_id) or job.host_id
+    return await record_or_update(
+        session, ("maintenance_job", job.id),
+        type="maintenance.failed" if maintenance.still_failed(job) else "maintenance.done",
+        at=job.created_at, until=job.finished_at,
+        summary=f"Maintenance on {host_name}: {maintenance.summary(job)}",
+        hosts=[host_name], subject=host_name,
+        detail={"state": job.state, "packages": job.packages, "rebooted": job.back_at is not None,
+                "note": job.detail, "url": f"/hosts/{job.host_id}"},
+        actor=await username(session, job.created_by) or "Steward")
+
+
+_DOWNTIME_TYPES = {"reboot": "host.rebooted", "unclean": "host.rebooted",
+                   "lost_contact": "host.lost_contact", "agent_stop": "host.agent_restarted"}
+
+
+async def downtime_entry(session, row, host_name: str | None = None):
+    """The entry for one stretch a host was away, written or brought up to date."""
+    from steward.core.formatting import human_duration
+    host_name = host_name or await _host_name(session, row.host_id) or row.host_id
+    seen = downtime.describe(row, row.ended_at or row.started_at)
+    said = seen["label"][:1].lower() + seen["label"][1:]
+    said += f" for {human_duration(seen['seconds'])}" if row.ended_at is not None else ", not back yet"
+    return await record_or_update(
+        session, ("host_downtime", row.id),
+        type=_DOWNTIME_TYPES.get(row.kind, "host.rebooted"),
+        at=row.started_at, until=row.ended_at, summary=f"{host_name}: {said}",
+        hosts=[host_name], subject=host_name,
+        detail={"kind": row.kind, "url": f"/hosts/{row.host_id}"}, actor="Steward")
+
+
+async def write(session, what: str, coro) -> None:
+    """Run one entry writer in a savepoint: the source's own row matters more
+    than its entry, so a failure is logged and nothing else is undone."""
+    try:
+        async with session.begin_nested():
+            await coro
+    except Exception:
+        logger.exception("Timeline: the entry for %s could not be written", what)
+
+
+async def backfill(session, since) -> list:
+    """Entries for the jobs and downtime that predate the timeline."""
+    from .models import HostDowntime, HostMaintenanceJob
+    out = []
+    for job in (await session.execute(select(HostMaintenanceJob).where(
+            HostMaintenanceJob.created_at >= since,
+            HostMaintenanceJob.state.in_(maintenance.FINISHED)))).scalars().all():
+        out.append(await job_entry(session, job))
+    for row in (await session.execute(select(HostDowntime).where(
+            HostDowntime.started_at >= since))).scalars().all():
+        out.append(await downtime_entry(session, row))
+    return out