feat(timeline): maintenance, reboots, deploys, crash-loops and reviewed anomalies on the timeline (#6349)

The host agent writes each maintenance job once it ends, and each stretch a
host was away (a reboot, a crash, lost contact, the agent restarting), kept up
to date until it is back. Docker judges each snapshot: a unit on a new image,
or a container recreated, is a deploy; deaths past the Thresholds crash-loop
count are one crash-loop entry that later deaths extend; single restarts are
recorded but hidden. An anomaly has one entry whose type follows its review:
only those judged real are shown by default.

A source whose record changes keeps one entry for it (record_or_update).
Plugins register the records their entries point to, so an aged-out job or
downtime reads "Details no longer kept". The first fill runs once per source
(timeline.filled), the host agent's jobs and downtime included.

get_anomaly's "what else was going on" reads the timeline for the host or
resource instead of querying alerts, runs, maintenance, downtime and container
deaths itself; Docker's own source keeps only starts, stops and health
changes. A new kind, Observed, is for what Steward saw happen with no rule or
person behind it.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

steward/timeline/service.py

@@ -41,12 +41,15 @@ def query(cfg: Config, *, host: str = "", area: str = "", kind: str = "", type_:
     return stmt
 
 
-# The full records an entry can point to, and whether one is still kept.
 def _ref_models() -> dict:
+    """The full records an entry can point to: core's, and the ones loaded
+    plugins registered (catalog.register_ref)."""
     from steward.models.alerts import AlertEvent
     from steward.models.ansible import AnsibleRun
     from steward.models.audit import AuditEvent
-    return {"alert_event": AlertEvent, "ansible_run": AnsibleRun, "audit_event": AuditEvent}
+    from steward.models.norms import MetricAnomaly
+    return {"alert_event": AlertEvent, "ansible_run": AnsibleRun, "audit_event": AuditEvent,
+            "metric_anomaly": MetricAnomaly, **catalog.REFS}
 
 
 async def kept_refs(db, entries) -> set[tuple[str, str]]:
@@ -59,7 +62,7 @@ async def kept_refs(db, entries) -> set[tuple[str, str]]:
     models = _ref_models()
     for ref_type, ids in wanted.items():
         model = models.get(ref_type)
-        if model is None:           # a kind only a plugin knows: assume it is kept
+        if model is None:           # a plugin no longer loaded: assume it is kept
             kept |= {(ref_type, i) for i in ids}
             continue
         found = (await db.execute(select(model.id).where(model.id.in_(ids)))).scalars()
@@ -126,58 +129,87 @@ async def carry_host_rename(session, old: str, new: str) -> int:
     return result.rowcount or 0
 
 
-async def backfill(app, now: datetime | None = None) -> int:
-    """Fill the timeline once from the alert events, Ansible runs and audit
-    rows that predate it, BACKFILL_DAYS back, through the same builders the
-    live writers use. An entry for a record already covered is skipped, so
-    it is safe to run again. Records the run in timeline.backfilled_at."""
-    from steward.core.settings import get_setting, set_setting
+async def _core_entries(session, since: datetime) -> list:
+    """Entries for the alert events, finished Ansible runs and audit rows
+    since `since`, built by the live writers' own builders. One record that
+    can't be read is skipped, not the whole fill."""
     from steward.models.alerts import AlertEvent, AlertRule
     from steward.models.ansible import AnsibleRun
     from steward.models.audit import AuditEvent
     from .record import from_alert, from_audit, from_run
+    built: list = []
+
+    async def build(what: str, coro) -> None:
+        try:
+            built.append(await coro)
+        except Exception:
+            logger.exception("Timeline fill: %s skipped", what)
+
+    for event, rule in (await session.execute(
+            select(AlertEvent, AlertRule).join(AlertRule, AlertEvent.rule_id == AlertRule.id)
+            .where(AlertEvent.transitioned_at >= since))).all():
+        await build(f"alert event {event.id}", from_alert(session, rule, event))
+    for run in (await session.execute(
+            select(AnsibleRun).where(AnsibleRun.started_at >= since,
+                                     AnsibleRun.finished_at.is_not(None)))).scalars().all():
+        await build(f"Ansible run {run.id}", from_run(session, run))
+    for event in (await session.execute(
+            select(AuditEvent).where(AuditEvent.timestamp >= since))).scalars().all():
+        await build(f"audit row {event.id}", from_audit(session, event, _audit_detail(event)))
+    return built
+
+
+def fill_sources() -> dict:
+    """{source: its `(session, since) -> [entries]`}: core's, and each loaded
+    plugin's `timeline_backfill`."""
+    from steward.core.plugin_manager import get_loaded_plugin_modules
+    sources = {"steward": _core_entries}
+    for name, module in sorted(get_loaded_plugin_modules().items()):
+        hook = getattr(module, "timeline_backfill", None)
+        if callable(hook):
+            sources[name] = hook
+    return sources
+
+
+async def backfill(app, now: datetime | None = None, sources: dict | None = None) -> int:
+    """Fill the timeline from the records that predate it, BACKFILL_DAYS
+    back, once per source: core's, then each loaded plugin's. Which sources
+    are done, and when, is kept in timeline.filled, so a plugin loaded later
+    fills on the next turn. An entry for a record already covered is
+    skipped, so a source filled again adds nothing twice. Returns how many
+    entries were added."""
+    from steward.core.settings import get_setting, set_setting
 
     now = now or datetime.now(timezone.utc)
+    since = now - timedelta(days=BACKFILL_DAYS)
+    sources = fill_sources() if sources is None else sources
+    added = 0
     async with app.db_sessionmaker() as session:
         async with session.begin():
-            if await get_setting(session, "timeline.backfilled_at"):
-                return 0
-        since = now - timedelta(days=BACKFILL_DAYS)
-        built: list[TimelineEntry] = []
-        async with session.begin():
-            # The builders add each entry to the session; no flush may write
-            # them before the insert below, which skips any already covered.
-            session.sync_session.autoflush = False
-
-            async def build(what: str, coro) -> None:
-                # One record that can't be read is skipped, not the whole fill.
-                try:
-                    built.append(await coro)
-                except Exception:
-                    logger.exception("Timeline fill: %s skipped", what)
-
-            for event, rule in (await session.execute(
-                    select(AlertEvent, AlertRule).join(AlertRule, AlertEvent.rule_id == AlertRule.id)
-                    .where(AlertEvent.transitioned_at >= since))).all():
-                await build(f"alert event {event.id}", from_alert(session, rule, event))
-            for run in (await session.execute(
-                    select(AnsibleRun).where(AnsibleRun.started_at >= since,
-                                             AnsibleRun.finished_at.is_not(None)))).scalars().all():
-                await build(f"Ansible run {run.id}", from_run(session, run))
-            for event in (await session.execute(
-                    select(AuditEvent).where(AuditEvent.timestamp >= since))).scalars().all():
-                await build(f"audit row {event.id}", from_audit(session, event, _audit_detail(event)))
-            rows = [e for e in built if e is not None]
-            for entry in rows:
-                session.expunge(entry)
-            for chunk in range(0, len(rows), 500):
-                await session.execute(
-                    insert(TimelineEntry).values([_row(e) for e in rows[chunk:chunk + 500]])
-                    .on_conflict_do_nothing(index_elements=["ref_type", "ref_id"],
-                                            index_where=text("ref_id IS NOT NULL")))
-            await set_setting(session, "timeline.backfilled_at", now.isoformat())
-    logger.info("Timeline: filled %d entries from the last %d days of records", len(rows), BACKFILL_DAYS)
-    return len(rows)
+            filled = dict(await get_setting(session, "timeline.filled") or {})
+        # The builders add each entry to the session; no flush may write
+        # them before the insert below, which skips any already covered.
+        session.sync_session.autoflush = False
+        for name, source in sources.items():
+            if name in filled:
+                continue
+            try:
+                async with session.begin():
+                    rows = [e for e in await source(session, since) if e is not None]
+                    for entry in rows:
+                        session.expunge(entry)
+                    for chunk in range(0, len(rows), 500):
+                        result = await session.execute(
+                            insert(TimelineEntry).values([_row(e) for e in rows[chunk:chunk + 500]])
+                            .on_conflict_do_nothing(index_elements=["ref_type", "ref_id"],
+                                                    index_where=text("ref_id IS NOT NULL")))
+                        added += result.rowcount or 0
+                    filled[name] = now.isoformat()
+                    await set_setting(session, "timeline.filled", filled)
+                logger.info("Timeline: filled from %s, the last %d days", name, BACKFILL_DAYS)
+            except Exception:
+                logger.exception("Timeline: the fill from %s failed; it is tried again tomorrow", name)
+    return added
 
 
 def _audit_detail(event) -> dict: