feat(timeline): maintenance, reboots, deploys, crash-loops and reviewed anomalies on the timeline (#6349)

The host agent writes each maintenance job once it ends, and each stretch a
host was away (a reboot, a crash, lost contact, the agent restarting), kept up
to date until it is back. Docker judges each snapshot: a unit on a new image,
or a container recreated, is a deploy; deaths past the Thresholds crash-loop
count are one crash-loop entry that later deaths extend; single restarts are
recorded but hidden. An anomaly has one entry whose type follows its review:
only those judged real are shown by default.

A source whose record changes keeps one entry for it (record_or_update).
Plugins register the records their entries point to, so an aged-out job or
downtime reads "Details no longer kept". The first fill runs once per source
(timeline.filled), the host agent's jobs and downtime included.

get_anomaly's "what else was going on" reads the timeline for the host or
resource instead of querying alerts, runs, maintenance, downtime and container
deaths itself; Docker's own source keeps only starts, stops and health
changes. A new kind, Observed, is for what Steward saw happen with no rule or
person behind it.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

steward/timeline/catalog.py

@@ -23,6 +23,7 @@ KINDS: dict[str, str] = {
     "alert": "Alert",
     "anomaly": "Anomaly",
     "ticket": "Ticket",
+    "observed": "Observed",      # Steward saw it happen: a reboot, a crash-loop
     "steward": "Steward",        # Steward's own settings and access
 }
 
@@ -53,6 +54,16 @@ def register_type(key: str, label: str, kind: str, area: str, shown: bool, about
     TYPES[key] = EntryType(key, label, kind, area, shown, about)
 
 
+# The full records an entry can point to: its ref_type and the model whose
+# id it names, so the timeline can tell a record still kept from one aged out.
+# A plugin adds its own with register_ref() from its setup().
+REFS: dict[str, object] = {}
+
+
+def register_ref(ref_type: str, model) -> None:
+    REFS[ref_type] = model
+
+
 for _t in (
     ("alert.fired", "Alert fired", "alert", "monitoring", True,
      "An alert rule's condition held long enough to fire."),
@@ -80,6 +91,16 @@ for _t in (
      "Someone signed in, or failed to."),
     ("steward.backup", "Steward backups", "steward", "storage", True,
      "Steward backed itself up, or was restored from a backup."),
+    # An anomaly's entry changes type as it is reviewed; its area is what the
+    # metric is about.
+    ("anomaly.to_review", "Anomaly to review", "anomaly", "monitoring", False,
+     "A metric left its normal and nobody has reviewed it yet."),
+    ("anomaly.real", "Anomaly judged real", "anomaly", "monitoring", True,
+     "An anomaly someone reviewed and marked a real problem."),
+    ("anomaly.normal", "Anomaly accepted as normal", "anomaly", "monitoring", False,
+     "An anomaly someone reviewed and marked normal: its level is learned."),
+    ("anomaly.lapsed", "Anomaly not reviewed", "anomaly", "monitoring", False,
+     "An anomaly nobody reviewed within the review window."),
 ):
     register_type(*_t)
 

steward/timeline/record.py

@@ -19,23 +19,44 @@ from steward.models.timeline import TimelineEntry
 logger = logging.getLogger(__name__)
 
 
-def record(session, *, type: str, at: datetime, summary: str, area: str | None = None,
+def _entry(*, type: str, at: datetime, summary: str, area: str | None = None,
            hosts=(), subject: str | None = None, detail: dict | None = None,
            actor: str = "system", until: datetime | None = None,
            ref: tuple[str, str] | None = None) -> TimelineEntry:
-    """Add one entry to `session`. `area` defaults to its type's; `ref` is
-    (kind, id) of the full record it points to, which is at most one entry."""
     known = catalog.TYPES.get(type)
-    entry = TimelineEntry(
+    return TimelineEntry(
         at=at, until=until, type=type,
         area=area or (known.area if known else catalog.OTHER[0]),
         hosts=sorted({h for h in hosts if h}), subject=subject[:255] if subject else None, summary=summary,
         detail=detail or None, actor=(actor or "system")[:64],
         ref_type=ref[0] if ref else None, ref_id=str(ref[1]) if ref else None)
+
+
+def record(session, **fields) -> TimelineEntry:
+    """Add one entry to `session`: type, at and summary, and optionally
+    area (defaults to its type's), hosts, subject, detail, actor, until and
+    ref, the (kind, id) of the full record it points to, which is at most
+    one entry."""
+    entry = _entry(**fields)
     session.add(entry)
     return entry
 
 
+async def record_or_update(session, ref: tuple[str, str], **fields) -> TimelineEntry:
+    """record() the entry for the record `ref` points to, or, when it has
+    one already, bring that one up to date: a source whose record changes
+    after it is written (a downtime that ends, an anomaly reviewed, a job
+    whose host came back late) keeps one entry for it, not one per change."""
+    entry = (await session.execute(select(TimelineEntry).where(
+        TimelineEntry.ref_type == ref[0], TimelineEntry.ref_id == str(ref[1])))).scalar_one_or_none()
+    if entry is None:
+        return record(session, ref=ref, **fields)
+    fresh = _entry(ref=ref, **fields)
+    for column in ("at", "until", "type", "area", "hosts", "subject", "summary", "detail", "actor"):
+        setattr(entry, column, getattr(fresh, column))
+    return entry
+
+
 async def username(session, user_id: str | None) -> str | None:
     if not user_id:
         return None
@@ -125,6 +146,43 @@ async def from_alert(session, rule, event, actor: str | None = None,
         ref=("alert_event", event.id))
 
 
+# ── Anomalies ───────────────────────────────────────────────────────────────
+
+_REVIEWED = {"to_review": "", "real": ": a real problem", "normal": ": accepted as normal",
+             "lapsed": ": not reviewed"}
+
+
+async def from_anomaly(session, anomaly) -> TimelineEntry:
+    """The entry for one anomaly, written when it opens and brought up to
+    date as it recurs and is reviewed: its type follows its review."""
+    from steward.core.formatting import metric_label, metric_value
+    metric, res = anomaly.metric_name, anomaly.resource_name
+    times = f", {anomaly.occurrences} times" if anomaly.occurrences > 1 else ""
+    return await record_or_update(
+        session, ("metric_anomaly", anomaly.id),
+        type=f"anomaly.{anomaly.review}", at=anomaly.started_at, until=anomaly.ended_at,
+        summary=(f"{metric_label(metric)} on {res} out of its normal (peak "
+                 f"{metric_value(metric, anomaly.peak_level)}, usual "
+                 f"{metric_value(metric, anomaly.usual_hold)}{times}){_REVIEWED.get(anomaly.review, '')}"),
+        area=catalog.area_of_metric(anomaly.source_module, metric),
+        hosts=await hosts_of_rule(session, anomaly.source_module, res), subject=res,
+        detail={"source": anomaly.source_module, "metric": metric, "review": anomaly.review,
+                "note": anomaly.note, "url": "/alerts/anomalies"},
+        actor=anomaly.reviewed_by if anomaly.review in ("real", "normal") and anomaly.reviewed_by
+        else "Steward")
+
+
+async def anomaly_changed(session, anomaly) -> None:
+    """Write an anomaly's entry in a savepoint. Never raises: the anomaly's
+    own record matters more than its entry."""
+    await session.flush()                   # the anomaly's id, and its row before the savepoint
+    try:
+        async with session.begin_nested():
+            await from_anomaly(session, anomaly)
+    except Exception:
+        logger.exception("Anomaly %s: its timeline entry could not be written", anomaly.id)
+
+
 # ── Ansible runs ────────────────────────────────────────────────────────────
 
 def _run_type(run) -> str:

steward/timeline/service.py

@@ -41,12 +41,15 @@ def query(cfg: Config, *, host: str = "", area: str = "", kind: str = "", type_:
     return stmt
 
 
-# The full records an entry can point to, and whether one is still kept.
 def _ref_models() -> dict:
+    """The full records an entry can point to: core's, and the ones loaded
+    plugins registered (catalog.register_ref)."""
     from steward.models.alerts import AlertEvent
     from steward.models.ansible import AnsibleRun
     from steward.models.audit import AuditEvent
-    return {"alert_event": AlertEvent, "ansible_run": AnsibleRun, "audit_event": AuditEvent}
+    from steward.models.norms import MetricAnomaly
+    return {"alert_event": AlertEvent, "ansible_run": AnsibleRun, "audit_event": AuditEvent,
+            "metric_anomaly": MetricAnomaly, **catalog.REFS}
 
 
 async def kept_refs(db, entries) -> set[tuple[str, str]]:
@@ -59,7 +62,7 @@ async def kept_refs(db, entries) -> set[tuple[str, str]]:
     models = _ref_models()
     for ref_type, ids in wanted.items():
         model = models.get(ref_type)
-        if model is None:           # a kind only a plugin knows: assume it is kept
+        if model is None:           # a plugin no longer loaded: assume it is kept
             kept |= {(ref_type, i) for i in ids}
             continue
         found = (await db.execute(select(model.id).where(model.id.in_(ids)))).scalars()
@@ -126,58 +129,87 @@ async def carry_host_rename(session, old: str, new: str) -> int:
     return result.rowcount or 0
 
 
-async def backfill(app, now: datetime | None = None) -> int:
-    """Fill the timeline once from the alert events, Ansible runs and audit
-    rows that predate it, BACKFILL_DAYS back, through the same builders the
-    live writers use. An entry for a record already covered is skipped, so
-    it is safe to run again. Records the run in timeline.backfilled_at."""
-    from steward.core.settings import get_setting, set_setting
+async def _core_entries(session, since: datetime) -> list:
+    """Entries for the alert events, finished Ansible runs and audit rows
+    since `since`, built by the live writers' own builders. One record that
+    can't be read is skipped, not the whole fill."""
     from steward.models.alerts import AlertEvent, AlertRule
     from steward.models.ansible import AnsibleRun
     from steward.models.audit import AuditEvent
     from .record import from_alert, from_audit, from_run
+    built: list = []
+
+    async def build(what: str, coro) -> None:
+        try:
+            built.append(await coro)
+        except Exception:
+            logger.exception("Timeline fill: %s skipped", what)
+
+    for event, rule in (await session.execute(
+            select(AlertEvent, AlertRule).join(AlertRule, AlertEvent.rule_id == AlertRule.id)
+            .where(AlertEvent.transitioned_at >= since))).all():
+        await build(f"alert event {event.id}", from_alert(session, rule, event))
+    for run in (await session.execute(
+            select(AnsibleRun).where(AnsibleRun.started_at >= since,
+                                     AnsibleRun.finished_at.is_not(None)))).scalars().all():
+        await build(f"Ansible run {run.id}", from_run(session, run))
+    for event in (await session.execute(
+            select(AuditEvent).where(AuditEvent.timestamp >= since))).scalars().all():
+        await build(f"audit row {event.id}", from_audit(session, event, _audit_detail(event)))
+    return built
+
+
+def fill_sources() -> dict:
+    """{source: its `(session, since) -> [entries]`}: core's, and each loaded
+    plugin's `timeline_backfill`."""
+    from steward.core.plugin_manager import get_loaded_plugin_modules
+    sources = {"steward": _core_entries}
+    for name, module in sorted(get_loaded_plugin_modules().items()):
+        hook = getattr(module, "timeline_backfill", None)
+        if callable(hook):
+            sources[name] = hook
+    return sources
+
+
+async def backfill(app, now: datetime | None = None, sources: dict | None = None) -> int:
+    """Fill the timeline from the records that predate it, BACKFILL_DAYS
+    back, once per source: core's, then each loaded plugin's. Which sources
+    are done, and when, is kept in timeline.filled, so a plugin loaded later
+    fills on the next turn. An entry for a record already covered is
+    skipped, so a source filled again adds nothing twice. Returns how many
+    entries were added."""
+    from steward.core.settings import get_setting, set_setting
 
     now = now or datetime.now(timezone.utc)
+    since = now - timedelta(days=BACKFILL_DAYS)
+    sources = fill_sources() if sources is None else sources
+    added = 0
     async with app.db_sessionmaker() as session:
         async with session.begin():
-            if await get_setting(session, "timeline.backfilled_at"):
-                return 0
-        since = now - timedelta(days=BACKFILL_DAYS)
-        built: list[TimelineEntry] = []
-        async with session.begin():
-            # The builders add each entry to the session; no flush may write
-            # them before the insert below, which skips any already covered.
-            session.sync_session.autoflush = False
-
-            async def build(what: str, coro) -> None:
-                # One record that can't be read is skipped, not the whole fill.
-                try:
-                    built.append(await coro)
-                except Exception:
-                    logger.exception("Timeline fill: %s skipped", what)
-
-            for event, rule in (await session.execute(
-                    select(AlertEvent, AlertRule).join(AlertRule, AlertEvent.rule_id == AlertRule.id)
-                    .where(AlertEvent.transitioned_at >= since))).all():
-                await build(f"alert event {event.id}", from_alert(session, rule, event))
-            for run in (await session.execute(
-                    select(AnsibleRun).where(AnsibleRun.started_at >= since,
-                                             AnsibleRun.finished_at.is_not(None)))).scalars().all():
-                await build(f"Ansible run {run.id}", from_run(session, run))
-            for event in (await session.execute(
-                    select(AuditEvent).where(AuditEvent.timestamp >= since))).scalars().all():
-                await build(f"audit row {event.id}", from_audit(session, event, _audit_detail(event)))
-            rows = [e for e in built if e is not None]
-            for entry in rows:
-                session.expunge(entry)
-            for chunk in range(0, len(rows), 500):
-                await session.execute(
-                    insert(TimelineEntry).values([_row(e) for e in rows[chunk:chunk + 500]])
-                    .on_conflict_do_nothing(index_elements=["ref_type", "ref_id"],
-                                            index_where=text("ref_id IS NOT NULL")))
-            await set_setting(session, "timeline.backfilled_at", now.isoformat())
-    logger.info("Timeline: filled %d entries from the last %d days of records", len(rows), BACKFILL_DAYS)
-    return len(rows)
+            filled = dict(await get_setting(session, "timeline.filled") or {})
+        # The builders add each entry to the session; no flush may write
+        # them before the insert below, which skips any already covered.
+        session.sync_session.autoflush = False
+        for name, source in sources.items():
+            if name in filled:
+                continue
+            try:
+                async with session.begin():
+                    rows = [e for e in await source(session, since) if e is not None]
+                    for entry in rows:
+                        session.expunge(entry)
+                    for chunk in range(0, len(rows), 500):
+                        result = await session.execute(
+                            insert(TimelineEntry).values([_row(e) for e in rows[chunk:chunk + 500]])
+                            .on_conflict_do_nothing(index_elements=["ref_type", "ref_id"],
+                                                    index_where=text("ref_id IS NOT NULL")))
+                        added += result.rowcount or 0
+                    filled[name] = now.isoformat()
+                    await set_setting(session, "timeline.filled", filled)
+                logger.info("Timeline: filled from %s, the last %d days", name, BACKFILL_DAYS)
+            except Exception:
+                logger.exception("Timeline: the fill from %s failed; it is tried again tomorrow", name)
+    return added
 
 
 def _audit_detail(event) -> dict: