feat(timeline): maintenance, reboots, deploys, crash-loops and reviewed anomalies on the timeline (#6349)

The host agent writes each maintenance job once it ends, and each stretch a
host was away (a reboot, a crash, lost contact, the agent restarting), kept up
to date until it is back. Docker judges each snapshot: a unit on a new image,
or a container recreated, is a deploy; deaths past the Thresholds crash-loop
count are one crash-loop entry that later deaths extend; single restarts are
recorded but hidden. An anomaly has one entry whose type follows its review:
only those judged real are shown by default.

A source whose record changes keeps one entry for it (record_or_update).
Plugins register the records their entries point to, so an aged-out job or
downtime reads "Details no longer kept". The first fill runs once per source
(timeline.filled), the host agent's jobs and downtime included.

get_anomaly's "what else was going on" reads the timeline for the host or
resource instead of querying alerts, runs, maintenance, downtime and container
deaths itself; Docker's own source keeps only starts, stops and health
changes. A new kind, Observed, is for what Steward saw happen with no rule or
person behind it.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

steward/core/anomalies.py

@@ -24,6 +24,7 @@ from sqlalchemy import ARRAY, String, bindparam, case, delete, func, or_, select
 
 from steward.core.norms import LEARNED, Norm, norms_at
 from steward.models.norms import MetricAnomaly, MetricAnomalySpan
+from steward.timeline.record import anomaly_changed
 
 WINDOW = timedelta(minutes=15)   # the level judged is this window's average
 DEPART = 1.25                    # times the slot's usual hold that counts as a departure
@@ -133,6 +134,7 @@ async def detect(session, now: datetime | None = None) -> dict:
             .where(MetricAnomalySpan.ended_at.is_(None)))).all()
     }
     opened = closed = 0
+    changed: list[MetricAnomaly] = []       # opened, joined or ended: their timeline entries follow
     for key, level in levels.items():
         norm = norms.get(key)
         learned = LEARNED.get((key[0], key[2]))
@@ -144,15 +146,19 @@ async def detect(session, now: datetime | None = None) -> dict:
                 _raise_peak(anomaly, span, level, norm, now)
             else:
                 _close(anomaly, span, now)
+                changed.append(anomaly)
                 closed += 1
         elif out:
-            await _occur(session, key, level, norm, now)
+            changed.append(await _occur(session, key, level, norm, now))
             opened += 1
     # An open occurrence whose key stopped reporting, lost its normal or fell
     # below its gate can't be judged any more: it ends now.
     for span, anomaly in open_now.values():
         _close(anomaly, span, now)
+        changed.append(anomaly)
         closed += 1
+    for anomaly in changed:
+        await anomaly_changed(session, anomaly)
     return {"opened": opened, "closed": closed}
 
 
@@ -169,9 +175,9 @@ def _close(anomaly: MetricAnomaly, span: MetricAnomalySpan, now: datetime) -> No
     anomaly.ended_at = now
 
 
-async def _occur(session, key, level: float, norm: Norm, now: datetime) -> None:
+async def _occur(session, key, level: float, norm: Norm, now: datetime) -> MetricAnomaly:
     """A departure on a key with no open occurrence: join the key's anomaly
-    still to review, or open a new one."""
+    still to review, or open a new one. Returns the anomaly."""
     source, resource, metric = key
     start = now - WINDOW
     anomaly = (await session.execute(
@@ -188,7 +194,7 @@ async def _occur(session, key, level: float, norm: Norm, now: datetime) -> None:
         session.add(anomaly)
         await session.flush()
         session.add(MetricAnomalySpan(anomaly_id=anomaly.id, started_at=start, peak_level=level))
-        return
+        return anomaly
     last = (await session.execute(
         select(MetricAnomalySpan).where(MetricAnomalySpan.anomaly_id == anomaly.id)
         .order_by(MetricAnomalySpan.started_at.desc()).limit(1))).scalar_one_or_none()
@@ -202,17 +208,25 @@ async def _occur(session, key, level: float, norm: Norm, now: datetime) -> None:
         anomaly.last_started_at = start
     anomaly.ended_at = None
     _raise_peak(anomaly, span, level, norm, now)
+    return anomaly
 
 
 async def lapse(session, now: datetime, review_days: int) -> int:
     """Ended anomalies still to review after `review_days` lapse. One still
     going never does. reviewed_at records when it lapsed. Returns how many."""
-    result = await session.execute(
+    ids = list((await session.execute(
         update(MetricAnomaly)
         .where(MetricAnomaly.review == TO_REVIEW, MetricAnomaly.ended_at.is_not(None),
                MetricAnomaly.ended_at < now - timedelta(days=review_days))
-        .values(review=LAPSED, reviewed_at=now))
-    return result.rowcount or 0
+        .values(review=LAPSED, reviewed_at=now)
+        .returning(MetricAnomaly.id)
+        .execution_options(synchronize_session=False))).scalars())
+    if ids:
+        for anomaly in (await session.execute(
+                select(MetricAnomaly).where(MetricAnomaly.id.in_(ids))
+                .execution_options(populate_existing=True))).scalars():
+            await anomaly_changed(session, anomaly)
+    return len(ids)
 
 
 async def prune(session, before: datetime) -> int:
@@ -238,6 +252,7 @@ async def mark(session, anomaly_id: str, review: str, by: str,
     anomaly.reviewed_by = by
     anomaly.reviewed_at = now or datetime.now(timezone.utc)
     anomaly.note = (note or "").strip() or None
+    await anomaly_changed(session, anomaly)
     return anomaly
 
 

steward/core/settings.py

@@ -61,8 +61,9 @@ DEFAULTS: dict[str, Any] = {
                        ["monitoring", "Monitoring"]],
     "timeline.types": {},
     "timeline.area_moved": {},
-    # When the timeline was first filled from the records that predate it.
-    "timeline.backfilled_at": "",
+    # {source: when}: each source (core, a plugin) the timeline was filled
+    # from once, from the records that predate it (timeline.service.backfill).
+    "timeline.filled": {},
     "monitors.poll_interval_seconds": 60,
     # Root log level: DEBUG, INFO, WARNING or ERROR (steward/core/logging_setup.py).
     # WARNING also hides the per-request access lines.

steward/mcp/tools/_events.py

@@ -1,11 +1,12 @@
 """What else was going on: events on a host, or one resource, in a window
 (#5837), so an anomaly comes with its circumstances.
 
-Core gathers alert transitions, Ansible runs, other metric anomalies and
-Steward's own gaps in recording. A plugin adds its own through
-`mcp_events(db, host_id, host_name, since, until)` in its __init__.py (the
-host agent its maintenance jobs and downtime, Docker its container events),
-and for a resource with no host through
+Core reads the timeline (alert episodes, Ansible runs, and what plugins
+wrote there: maintenance, downtime, deploys, restarts), other metric
+anomalies, and Steward's own gaps in recording. A plugin adds what the
+timeline doesn't hold through `mcp_events(db, host_id, host_name, since,
+until)` in its __init__.py (Docker its stops and health changes, syslog its
+warnings), and for a resource with no host through
 `mcp_resource_events(db, source, resource, since, until)` (Traefik: the
 containers of the swarm service behind a router, #5862); each returns a list
 of `event(...)`.
@@ -16,15 +17,11 @@ import logging
 from dataclasses import dataclass
 from datetime import datetime, timedelta
 
-from sqlalchemy import and_, or_, select
+from sqlalchemy import and_, func, or_, select
 
-from steward.alerts.service import watches_host
 from steward.core.alerts import host_of_resource
 from steward.core.formatting import human_duration
 from steward.mcp._context import mcp_app
-from steward.models.alerts import AlertEvent, AlertRule
-from steward.models.ansible import AnsibleRun
-from steward.models.ansible_inventory import AnsibleTarget
 from steward.models.hosts import Host
 from steward.models.metrics import PluginMetric
 from steward.models.monitors import Monitor
@@ -80,47 +77,23 @@ def _overlaps(start, end):
     return clause
 
 
-async def alert_events(db, place: Place, since: datetime, until: datetime) -> list[dict]:
-    """Alert transitions on the resource itself, or anything its host
-    carries: its agent series, its containers, its monitors."""
-    monitor_keys = set()
-    if place.host_id:
-        monitor_keys = {(t, n) for t, n in (await db.execute(
-            select(Monitor.type, Monitor.name).where(Monitor.host_id == place.host_id))).all()}
+async def timeline_events(db, place: Place, since: datetime, until: datetime) -> list[dict]:
+    """What the timeline holds about the host, or the resource itself, while
+    the window was open: alert episodes, playbook runs, maintenance, reboots
+    and lost contact, deploys, restarts and crash-loops, settings changes.
+    Hidden types are included: a dry run or a single restart can be what
+    explains a reading. Anomalies come from other_anomalies, by occurrence."""
+    from steward.models.timeline import TimelineEntry
+    about = [TimelineEntry.subject == place.resource]
+    if place.host_name:
+        about.append(TimelineEntry.hosts.contains([place.host_name]))
     rows = (await db.execute(
-        select(AlertEvent, AlertRule).join(AlertRule, AlertRule.id == AlertEvent.rule_id)
-        .where(AlertEvent.transitioned_at >= since, AlertEvent.transitioned_at <= until)
-        .order_by(AlertEvent.transitioned_at).limit(500))).all()
-    out = []
-    for ev, rule in rows:
-        mine = (rule.source_module, rule.resource_name) == (place.source, place.resource)
-        if not mine and not (place.host_name and watches_host(rule, place.host_name, monitor_keys)):
-            continue
-        note = f" ({ev.note})" if ev.note else ""
-        out.append(event(ev.transitioned_at, "alert", f"{rule.name}: {ev.from_state} → {ev.to_state}{note}",
-                         subject=rule.resource_name, ids={"rule_id": rule.id}))
-    return out
-
-
-async def ansible_runs(db, place: Place, since: datetime, until: datetime) -> list[dict]:
-    """Ansible runs that touched the host while the window was open (by the
-    hosts in their recap: a run still going hasn't said yet)."""
-    if not place.host_id:
-        return []
-    names = {place.host_name} | set((await db.execute(
-        select(AnsibleTarget.name).where(AnsibleTarget.host_id == place.host_id))).scalars())
-    runs = (await db.execute(
-        select(AnsibleRun).where(_overlaps(AnsibleRun.started_at, AnsibleRun.finished_at)(since, until))
-        .order_by(AnsibleRun.started_at).limit(200))).scalars()
-    out = []
-    for r in runs:
-        if not names & set(((r.results or {}).get("hosts") or {}).keys()):
-            continue
-        check = " in check mode" if (r.params or {}).get("check") else ""
-        out.append(event(r.started_at, "ansible_run",
-                         f"Ansible {r.playbook_path}{check}: {r.status.value}",
-                         until=r.finished_at, ids={"run_id": r.id}))
-    return out
+        select(TimelineEntry)
+        .where(or_(*about), _overlaps(TimelineEntry.at, func.coalesce(TimelineEntry.until, TimelineEntry.at))
+               (since, until), or_(TimelineEntry.ref_type.is_(None), TimelineEntry.ref_type != "metric_anomaly"))
+        .order_by(TimelineEntry.at).limit(500))).scalars()
+    return [event(e.at, e.type, e.summary, until=e.until, subject=e.subject,
+                  ids={e.ref_type: e.ref_id} if e.ref_type else None) for e in rows]
 
 
 def _same_host(source: str, resource: str, place: Place) -> bool:
@@ -248,8 +221,8 @@ async def around(db, place: Place, anomaly_id: str, since: datetime, until: date
     elsewhere; and how many events were left out, each source keeping its
     PER_SOURCE nearest `focus`."""
     anomalies, elsewhere = await other_anomalies(db, place, anomaly_id, since, until)
-    groups = [await alert_events(db, place, since, until), await ansible_runs(db, place, since, until),
-              anomalies, *await plugin_events(place, since, until),
+    groups = [await timeline_events(db, place, since, until), anomalies,
+              *await plugin_events(place, since, until),
               *await resource_events(place, since, until), await steward_gaps(db, since, until)]
     found, left_out = [], 0
     for group in groups:

steward/mcp/tools/anomalies.py

@@ -157,13 +157,16 @@ async def get_anomaly(anomaly_id: str, occurrences: int = 5) -> dict:
       has one: an agent series, a container, a monitor pinned to a host.
     - `occurrences[]`: `started_at`, `ended_at`, `held_s`, `peak_level`,
       and `usual` for that hour as Steward has learned it now (`usual_hold`,
-      `usual_peak`, `flag_above`), plus `around`: alert transitions, Ansible
-      runs, other anomalies on the host, what plugins saw (container
-      starts, deaths and OOM kills; maintenance jobs; reboots and agent
-      stops; for a Traefik router, its swarm service's containers on any
-      host; syslog warnings and worse from the host's or the SNMP device's
-      own log), and kind "steward": a stretch Steward itself wasn't recording
-      (a restart or deploy), around which any reading can be wrong. From 30
+      `usual_peak`, `flag_above`), plus `around`: what the timeline holds
+      about the host or resource, its kind the entry's type (alert.fired,
+      ansible.changed, maintenance.done, host.rebooted, docker.deployed,
+      docker.restarted, docker.crash_loop, steward.config …, hidden types
+      included); other anomalies on the host; what plugins saw that the
+      timeline doesn't hold (container starts, stops and health changes; for
+      a Traefik router, its swarm service's containers on any host; syslog
+      warnings and worse from the host's or the SNMP device's own log); and
+      kind "steward": a stretch Steward itself wasn't recording (a restart
+      or deploy), around which any reading can be wrong. From 30
       minutes before to 30 minutes after. Each kind keeps the 40 events
       nearest the occurrence's start; `around_left_out` counts the rest
       (0 when nothing was cut). And

steward/mcp/tools/timeline.py

@@ -12,9 +12,11 @@ async def get_timeline(host: str = "", area: str = "", kind: str = "", type: str
                        range: str = "7d", include_hidden: bool = False, limit: int = 50) -> dict:
     """What happened to the infrastructure, newest first, from every source in
     one list: alerts firing, resolving and going silent; playbook runs and
-    what they changed; changes to Steward's own settings, users and backups.
-    Each entry outlives the record it came from, so this reaches further back
-    than get_alert_history, list_ansible_runs or list_audit_events.
+    what they changed; maintenance runs, reboots and lost contact; deploys
+    and crash-loops; anomalies once reviewed as real; changes to Steward's
+    own settings, users and backups. Each entry outlives the record it came
+    from, so this reaches further back than get_alert_history,
+    list_ansible_runs or list_audit_events.
 
     Reach for it first for "what changed on this host lately?" or "what else
     was going on when this broke?". Follow an entry's `url` or ids into the
@@ -24,11 +26,13 @@ async def get_timeline(host: str = "", area: str = "", kind: str = "", type: str
     area: what part of the infrastructure an entry is about, as Settings →
       Timeline names its areas (`areas` in the reply lists them); empty for all.
     kind: how it came about: "change", "automation", "alert", "anomaly",
-      "ticket" or "steward"; empty for all.
+      "ticket", "observed" (seen happening: a reboot, a crash-loop) or
+      "steward"; empty for all.
     type: one entry type (`types` in the reply); it is returned even if hidden.
     range: "1h", "6h", "24h", "7d" (default) or "30d".
     include_hidden: also the types Settings hides from the timeline (dry runs,
-      sign-ins, playbook runs that changed nothing).
+      sign-ins, playbook runs that changed nothing, single container
+      restarts, anomalies not reviewed as real).
     limit: default 50, max 500.
     """
     if kind and kind not in catalog.KINDS:

steward/timeline/catalog.py

@@ -23,6 +23,7 @@ KINDS: dict[str, str] = {
     "alert": "Alert",
     "anomaly": "Anomaly",
     "ticket": "Ticket",
+    "observed": "Observed",      # Steward saw it happen: a reboot, a crash-loop
     "steward": "Steward",        # Steward's own settings and access
 }
 
@@ -53,6 +54,16 @@ def register_type(key: str, label: str, kind: str, area: str, shown: bool, about
     TYPES[key] = EntryType(key, label, kind, area, shown, about)
 
 
+# The full records an entry can point to: its ref_type and the model whose
+# id it names, so the timeline can tell a record still kept from one aged out.
+# A plugin adds its own with register_ref() from its setup().
+REFS: dict[str, object] = {}
+
+
+def register_ref(ref_type: str, model) -> None:
+    REFS[ref_type] = model
+
+
 for _t in (
     ("alert.fired", "Alert fired", "alert", "monitoring", True,
      "An alert rule's condition held long enough to fire."),
@@ -80,6 +91,16 @@ for _t in (
      "Someone signed in, or failed to."),
     ("steward.backup", "Steward backups", "steward", "storage", True,
      "Steward backed itself up, or was restored from a backup."),
+    # An anomaly's entry changes type as it is reviewed; its area is what the
+    # metric is about.
+    ("anomaly.to_review", "Anomaly to review", "anomaly", "monitoring", False,
+     "A metric left its normal and nobody has reviewed it yet."),
+    ("anomaly.real", "Anomaly judged real", "anomaly", "monitoring", True,
+     "An anomaly someone reviewed and marked a real problem."),
+    ("anomaly.normal", "Anomaly accepted as normal", "anomaly", "monitoring", False,
+     "An anomaly someone reviewed and marked normal: its level is learned."),
+    ("anomaly.lapsed", "Anomaly not reviewed", "anomaly", "monitoring", False,
+     "An anomaly nobody reviewed within the review window."),
 ):
     register_type(*_t)
 

steward/timeline/record.py

@@ -19,23 +19,44 @@ from steward.models.timeline import TimelineEntry
 logger = logging.getLogger(__name__)
 
 
-def record(session, *, type: str, at: datetime, summary: str, area: str | None = None,
+def _entry(*, type: str, at: datetime, summary: str, area: str | None = None,
            hosts=(), subject: str | None = None, detail: dict | None = None,
            actor: str = "system", until: datetime | None = None,
            ref: tuple[str, str] | None = None) -> TimelineEntry:
-    """Add one entry to `session`. `area` defaults to its type's; `ref` is
-    (kind, id) of the full record it points to, which is at most one entry."""
     known = catalog.TYPES.get(type)
-    entry = TimelineEntry(
+    return TimelineEntry(
         at=at, until=until, type=type,
         area=area or (known.area if known else catalog.OTHER[0]),
         hosts=sorted({h for h in hosts if h}), subject=subject[:255] if subject else None, summary=summary,
         detail=detail or None, actor=(actor or "system")[:64],
         ref_type=ref[0] if ref else None, ref_id=str(ref[1]) if ref else None)
+
+
+def record(session, **fields) -> TimelineEntry:
+    """Add one entry to `session`: type, at and summary, and optionally
+    area (defaults to its type's), hosts, subject, detail, actor, until and
+    ref, the (kind, id) of the full record it points to, which is at most
+    one entry."""
+    entry = _entry(**fields)
     session.add(entry)
     return entry
 
 
+async def record_or_update(session, ref: tuple[str, str], **fields) -> TimelineEntry:
+    """record() the entry for the record `ref` points to, or, when it has
+    one already, bring that one up to date: a source whose record changes
+    after it is written (a downtime that ends, an anomaly reviewed, a job
+    whose host came back late) keeps one entry for it, not one per change."""
+    entry = (await session.execute(select(TimelineEntry).where(
+        TimelineEntry.ref_type == ref[0], TimelineEntry.ref_id == str(ref[1])))).scalar_one_or_none()
+    if entry is None:
+        return record(session, ref=ref, **fields)
+    fresh = _entry(ref=ref, **fields)
+    for column in ("at", "until", "type", "area", "hosts", "subject", "summary", "detail", "actor"):
+        setattr(entry, column, getattr(fresh, column))
+    return entry
+
+
 async def username(session, user_id: str | None) -> str | None:
     if not user_id:
         return None
@@ -125,6 +146,43 @@ async def from_alert(session, rule, event, actor: str | None = None,
         ref=("alert_event", event.id))
 
 
+# ── Anomalies ───────────────────────────────────────────────────────────────
+
+_REVIEWED = {"to_review": "", "real": ": a real problem", "normal": ": accepted as normal",
+             "lapsed": ": not reviewed"}
+
+
+async def from_anomaly(session, anomaly) -> TimelineEntry:
+    """The entry for one anomaly, written when it opens and brought up to
+    date as it recurs and is reviewed: its type follows its review."""
+    from steward.core.formatting import metric_label, metric_value
+    metric, res = anomaly.metric_name, anomaly.resource_name
+    times = f", {anomaly.occurrences} times" if anomaly.occurrences > 1 else ""
+    return await record_or_update(
+        session, ("metric_anomaly", anomaly.id),
+        type=f"anomaly.{anomaly.review}", at=anomaly.started_at, until=anomaly.ended_at,
+        summary=(f"{metric_label(metric)} on {res} out of its normal (peak "
+                 f"{metric_value(metric, anomaly.peak_level)}, usual "
+                 f"{metric_value(metric, anomaly.usual_hold)}{times}){_REVIEWED.get(anomaly.review, '')}"),
+        area=catalog.area_of_metric(anomaly.source_module, metric),
+        hosts=await hosts_of_rule(session, anomaly.source_module, res), subject=res,
+        detail={"source": anomaly.source_module, "metric": metric, "review": anomaly.review,
+                "note": anomaly.note, "url": "/alerts/anomalies"},
+        actor=anomaly.reviewed_by if anomaly.review in ("real", "normal") and anomaly.reviewed_by
+        else "Steward")
+
+
+async def anomaly_changed(session, anomaly) -> None:
+    """Write an anomaly's entry in a savepoint. Never raises: the anomaly's
+    own record matters more than its entry."""
+    await session.flush()                   # the anomaly's id, and its row before the savepoint
+    try:
+        async with session.begin_nested():
+            await from_anomaly(session, anomaly)
+    except Exception:
+        logger.exception("Anomaly %s: its timeline entry could not be written", anomaly.id)
+
+
 # ── Ansible runs ────────────────────────────────────────────────────────────
 
 def _run_type(run) -> str:

steward/timeline/service.py

@@ -41,12 +41,15 @@ def query(cfg: Config, *, host: str = "", area: str = "", kind: str = "", type_:
     return stmt
 
 
-# The full records an entry can point to, and whether one is still kept.
 def _ref_models() -> dict:
+    """The full records an entry can point to: core's, and the ones loaded
+    plugins registered (catalog.register_ref)."""
     from steward.models.alerts import AlertEvent
     from steward.models.ansible import AnsibleRun
     from steward.models.audit import AuditEvent
-    return {"alert_event": AlertEvent, "ansible_run": AnsibleRun, "audit_event": AuditEvent}
+    from steward.models.norms import MetricAnomaly
+    return {"alert_event": AlertEvent, "ansible_run": AnsibleRun, "audit_event": AuditEvent,
+            "metric_anomaly": MetricAnomaly, **catalog.REFS}
 
 
 async def kept_refs(db, entries) -> set[tuple[str, str]]:
@@ -59,7 +62,7 @@ async def kept_refs(db, entries) -> set[tuple[str, str]]:
     models = _ref_models()
     for ref_type, ids in wanted.items():
         model = models.get(ref_type)
-        if model is None:           # a kind only a plugin knows: assume it is kept
+        if model is None:           # a plugin no longer loaded: assume it is kept
             kept |= {(ref_type, i) for i in ids}
             continue
         found = (await db.execute(select(model.id).where(model.id.in_(ids)))).scalars()
@@ -126,58 +129,87 @@ async def carry_host_rename(session, old: str, new: str) -> int:
     return result.rowcount or 0
 
 
-async def backfill(app, now: datetime | None = None) -> int:
-    """Fill the timeline once from the alert events, Ansible runs and audit
-    rows that predate it, BACKFILL_DAYS back, through the same builders the
-    live writers use. An entry for a record already covered is skipped, so
-    it is safe to run again. Records the run in timeline.backfilled_at."""
-    from steward.core.settings import get_setting, set_setting
+async def _core_entries(session, since: datetime) -> list:
+    """Entries for the alert events, finished Ansible runs and audit rows
+    since `since`, built by the live writers' own builders. One record that
+    can't be read is skipped, not the whole fill."""
     from steward.models.alerts import AlertEvent, AlertRule
     from steward.models.ansible import AnsibleRun
     from steward.models.audit import AuditEvent
     from .record import from_alert, from_audit, from_run
+    built: list = []
+
+    async def build(what: str, coro) -> None:
+        try:
+            built.append(await coro)
+        except Exception:
+            logger.exception("Timeline fill: %s skipped", what)
+
+    for event, rule in (await session.execute(
+            select(AlertEvent, AlertRule).join(AlertRule, AlertEvent.rule_id == AlertRule.id)
+            .where(AlertEvent.transitioned_at >= since))).all():
+        await build(f"alert event {event.id}", from_alert(session, rule, event))
+    for run in (await session.execute(
+            select(AnsibleRun).where(AnsibleRun.started_at >= since,
+                                     AnsibleRun.finished_at.is_not(None)))).scalars().all():
+        await build(f"Ansible run {run.id}", from_run(session, run))
+    for event in (await session.execute(
+            select(AuditEvent).where(AuditEvent.timestamp >= since))).scalars().all():
+        await build(f"audit row {event.id}", from_audit(session, event, _audit_detail(event)))
+    return built
+
+
+def fill_sources() -> dict:
+    """{source: its `(session, since) -> [entries]`}: core's, and each loaded
+    plugin's `timeline_backfill`."""
+    from steward.core.plugin_manager import get_loaded_plugin_modules
+    sources = {"steward": _core_entries}
+    for name, module in sorted(get_loaded_plugin_modules().items()):
+        hook = getattr(module, "timeline_backfill", None)
+        if callable(hook):
+            sources[name] = hook
+    return sources
+
+
+async def backfill(app, now: datetime | None = None, sources: dict | None = None) -> int:
+    """Fill the timeline from the records that predate it, BACKFILL_DAYS
+    back, once per source: core's, then each loaded plugin's. Which sources
+    are done, and when, is kept in timeline.filled, so a plugin loaded later
+    fills on the next turn. An entry for a record already covered is
+    skipped, so a source filled again adds nothing twice. Returns how many
+    entries were added."""
+    from steward.core.settings import get_setting, set_setting
 
     now = now or datetime.now(timezone.utc)
+    since = now - timedelta(days=BACKFILL_DAYS)
+    sources = fill_sources() if sources is None else sources
+    added = 0
     async with app.db_sessionmaker() as session:
         async with session.begin():
-            if await get_setting(session, "timeline.backfilled_at"):
-                return 0
-        since = now - timedelta(days=BACKFILL_DAYS)
-        built: list[TimelineEntry] = []
-        async with session.begin():
-            # The builders add each entry to the session; no flush may write
-            # them before the insert below, which skips any already covered.
-            session.sync_session.autoflush = False
-
-            async def build(what: str, coro) -> None:
-                # One record that can't be read is skipped, not the whole fill.
-                try:
-                    built.append(await coro)
-                except Exception:
-                    logger.exception("Timeline fill: %s skipped", what)
-
-            for event, rule in (await session.execute(
-                    select(AlertEvent, AlertRule).join(AlertRule, AlertEvent.rule_id == AlertRule.id)
-                    .where(AlertEvent.transitioned_at >= since))).all():
-                await build(f"alert event {event.id}", from_alert(session, rule, event))
-            for run in (await session.execute(
-                    select(AnsibleRun).where(AnsibleRun.started_at >= since,
-                                             AnsibleRun.finished_at.is_not(None)))).scalars().all():
-                await build(f"Ansible run {run.id}", from_run(session, run))
-            for event in (await session.execute(
-                    select(AuditEvent).where(AuditEvent.timestamp >= since))).scalars().all():
-                await build(f"audit row {event.id}", from_audit(session, event, _audit_detail(event)))
-            rows = [e for e in built if e is not None]
-            for entry in rows:
-                session.expunge(entry)
-            for chunk in range(0, len(rows), 500):
-                await session.execute(
-                    insert(TimelineEntry).values([_row(e) for e in rows[chunk:chunk + 500]])
-                    .on_conflict_do_nothing(index_elements=["ref_type", "ref_id"],
-                                            index_where=text("ref_id IS NOT NULL")))
-            await set_setting(session, "timeline.backfilled_at", now.isoformat())
-    logger.info("Timeline: filled %d entries from the last %d days of records", len(rows), BACKFILL_DAYS)
-    return len(rows)
+            filled = dict(await get_setting(session, "timeline.filled") or {})
+        # The builders add each entry to the session; no flush may write
+        # them before the insert below, which skips any already covered.
+        session.sync_session.autoflush = False
+        for name, source in sources.items():
+            if name in filled:
+                continue
+            try:
+                async with session.begin():
+                    rows = [e for e in await source(session, since) if e is not None]
+                    for entry in rows:
+                        session.expunge(entry)
+                    for chunk in range(0, len(rows), 500):
+                        result = await session.execute(
+                            insert(TimelineEntry).values([_row(e) for e in rows[chunk:chunk + 500]])
+                            .on_conflict_do_nothing(index_elements=["ref_type", "ref_id"],
+                                                    index_where=text("ref_id IS NOT NULL")))
+                        added += result.rowcount or 0
+                    filled[name] = now.isoformat()
+                    await set_setting(session, "timeline.filled", filled)
+                logger.info("Timeline: filled from %s, the last %d days", name, BACKFILL_DAYS)
+            except Exception:
+                logger.exception("Timeline: the fill from %s failed; it is tried again tomorrow", name)
+    return added
 
 
 def _audit_detail(event) -> dict: