feat(timeline): maintenance, reboots, deploys, crash-loops and reviewed anomalies on the timeline (#6349)

The host agent writes each maintenance job once it ends, and each stretch a
host was away (a reboot, a crash, lost contact, the agent restarting), kept up
to date until it is back. Docker judges each snapshot: a unit on a new image,
or a container recreated, is a deploy; deaths past the Thresholds crash-loop
count are one crash-loop entry that later deaths extend; single restarts are
recorded but hidden. An anomaly has one entry whose type follows its review:
only those judged real are shown by default.

A source whose record changes keeps one entry for it (record_or_update).
Plugins register the records their entries point to, so an aged-out job or
downtime reads "Details no longer kept". The first fill runs once per source
(timeline.filled), the host agent's jobs and downtime included.

get_anomaly's "what else was going on" reads the timeline for the host or
resource instead of querying alerts, runs, maintenance, downtime and container
deaths itself; Docker's own source keeps only starts, stops and health
changes. A new kind, Observed, is for what Steward saw happen with no rule or
person behind it.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

steward/timeline/record.py

@@ -19,23 +19,44 @@ from steward.models.timeline import TimelineEntry
 logger = logging.getLogger(__name__)
 
 
-def record(session, *, type: str, at: datetime, summary: str, area: str | None = None,
+def _entry(*, type: str, at: datetime, summary: str, area: str | None = None,
            hosts=(), subject: str | None = None, detail: dict | None = None,
            actor: str = "system", until: datetime | None = None,
            ref: tuple[str, str] | None = None) -> TimelineEntry:
-    """Add one entry to `session`. `area` defaults to its type's; `ref` is
-    (kind, id) of the full record it points to, which is at most one entry."""
     known = catalog.TYPES.get(type)
-    entry = TimelineEntry(
+    return TimelineEntry(
         at=at, until=until, type=type,
         area=area or (known.area if known else catalog.OTHER[0]),
         hosts=sorted({h for h in hosts if h}), subject=subject[:255] if subject else None, summary=summary,
         detail=detail or None, actor=(actor or "system")[:64],
         ref_type=ref[0] if ref else None, ref_id=str(ref[1]) if ref else None)
+
+
+def record(session, **fields) -> TimelineEntry:
+    """Add one entry to `session`: type, at and summary, and optionally
+    area (defaults to its type's), hosts, subject, detail, actor, until and
+    ref, the (kind, id) of the full record it points to, which is at most
+    one entry."""
+    entry = _entry(**fields)
     session.add(entry)
     return entry
 
 
+async def record_or_update(session, ref: tuple[str, str], **fields) -> TimelineEntry:
+    """record() the entry for the record `ref` points to, or, when it has
+    one already, bring that one up to date: a source whose record changes
+    after it is written (a downtime that ends, an anomaly reviewed, a job
+    whose host came back late) keeps one entry for it, not one per change."""
+    entry = (await session.execute(select(TimelineEntry).where(
+        TimelineEntry.ref_type == ref[0], TimelineEntry.ref_id == str(ref[1])))).scalar_one_or_none()
+    if entry is None:
+        return record(session, ref=ref, **fields)
+    fresh = _entry(ref=ref, **fields)
+    for column in ("at", "until", "type", "area", "hosts", "subject", "summary", "detail", "actor"):
+        setattr(entry, column, getattr(fresh, column))
+    return entry
+
+
 async def username(session, user_id: str | None) -> str | None:
     if not user_id:
         return None
@@ -125,6 +146,43 @@ async def from_alert(session, rule, event, actor: str | None = None,
         ref=("alert_event", event.id))
 
 
+# ── Anomalies ───────────────────────────────────────────────────────────────
+
+_REVIEWED = {"to_review": "", "real": ": a real problem", "normal": ": accepted as normal",
+             "lapsed": ": not reviewed"}
+
+
+async def from_anomaly(session, anomaly) -> TimelineEntry:
+    """The entry for one anomaly, written when it opens and brought up to
+    date as it recurs and is reviewed: its type follows its review."""
+    from steward.core.formatting import metric_label, metric_value
+    metric, res = anomaly.metric_name, anomaly.resource_name
+    times = f", {anomaly.occurrences} times" if anomaly.occurrences > 1 else ""
+    return await record_or_update(
+        session, ("metric_anomaly", anomaly.id),
+        type=f"anomaly.{anomaly.review}", at=anomaly.started_at, until=anomaly.ended_at,
+        summary=(f"{metric_label(metric)} on {res} out of its normal (peak "
+                 f"{metric_value(metric, anomaly.peak_level)}, usual "
+                 f"{metric_value(metric, anomaly.usual_hold)}{times}){_REVIEWED.get(anomaly.review, '')}"),
+        area=catalog.area_of_metric(anomaly.source_module, metric),
+        hosts=await hosts_of_rule(session, anomaly.source_module, res), subject=res,
+        detail={"source": anomaly.source_module, "metric": metric, "review": anomaly.review,
+                "note": anomaly.note, "url": "/alerts/anomalies"},
+        actor=anomaly.reviewed_by if anomaly.review in ("real", "normal") and anomaly.reviewed_by
+        else "Steward")
+
+
+async def anomaly_changed(session, anomaly) -> None:
+    """Write an anomaly's entry in a savepoint. Never raises: the anomaly's
+    own record matters more than its entry."""
+    await session.flush()                   # the anomaly's id, and its row before the savepoint
+    try:
+        async with session.begin_nested():
+            await from_anomaly(session, anomaly)
+    except Exception:
+        logger.exception("Anomaly %s: its timeline entry could not be written", anomaly.id)
+
+
 # ── Ansible runs ────────────────────────────────────────────────────────────
 
 def _run_type(run) -> str: