feat(timeline): maintenance, reboots, deploys, crash-loops and reviewed anomalies on the timeline (#6349)

The host agent writes each maintenance job once it ends, and each stretch a
host was away (a reboot, a crash, lost contact, the agent restarting), kept up
to date until it is back. Docker judges each snapshot: a unit on a new image,
or a container recreated, is a deploy; deaths past the Thresholds crash-loop
count are one crash-loop entry that later deaths extend; single restarts are
recorded but hidden. An anomaly has one entry whose type follows its review:
only those judged real are shown by default.

A source whose record changes keeps one entry for it (record_or_update).
Plugins register the records their entries point to, so an aged-out job or
downtime reads "Details no longer kept". The first fill runs once per source
(timeline.filled), the host agent's jobs and downtime included.

get_anomaly's "what else was going on" reads the timeline for the host or
resource instead of querying alerts, runs, maintenance, downtime and container
deaths itself; Docker's own source keeps only starts, stops and health
changes. A new kind, Observed, is for what Steward saw happen with no rule or
person behind it.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

steward/core/anomalies.py

@@ -24,6 +24,7 @@ from sqlalchemy import ARRAY, String, bindparam, case, delete, func, or_, select
 
 from steward.core.norms import LEARNED, Norm, norms_at
 from steward.models.norms import MetricAnomaly, MetricAnomalySpan
+from steward.timeline.record import anomaly_changed
 
 WINDOW = timedelta(minutes=15)   # the level judged is this window's average
 DEPART = 1.25                    # times the slot's usual hold that counts as a departure
@@ -133,6 +134,7 @@ async def detect(session, now: datetime | None = None) -> dict:
             .where(MetricAnomalySpan.ended_at.is_(None)))).all()
     }
     opened = closed = 0
+    changed: list[MetricAnomaly] = []       # opened, joined or ended: their timeline entries follow
     for key, level in levels.items():
         norm = norms.get(key)
         learned = LEARNED.get((key[0], key[2]))
@@ -144,15 +146,19 @@ async def detect(session, now: datetime | None = None) -> dict:
                 _raise_peak(anomaly, span, level, norm, now)
             else:
                 _close(anomaly, span, now)
+                changed.append(anomaly)
                 closed += 1
         elif out:
-            await _occur(session, key, level, norm, now)
+            changed.append(await _occur(session, key, level, norm, now))
             opened += 1
     # An open occurrence whose key stopped reporting, lost its normal or fell
     # below its gate can't be judged any more: it ends now.
     for span, anomaly in open_now.values():
         _close(anomaly, span, now)
+        changed.append(anomaly)
         closed += 1
+    for anomaly in changed:
+        await anomaly_changed(session, anomaly)
     return {"opened": opened, "closed": closed}
 
 
@@ -169,9 +175,9 @@ def _close(anomaly: MetricAnomaly, span: MetricAnomalySpan, now: datetime) -> No
     anomaly.ended_at = now
 
 
-async def _occur(session, key, level: float, norm: Norm, now: datetime) -> None:
+async def _occur(session, key, level: float, norm: Norm, now: datetime) -> MetricAnomaly:
     """A departure on a key with no open occurrence: join the key's anomaly
-    still to review, or open a new one."""
+    still to review, or open a new one. Returns the anomaly."""
     source, resource, metric = key
     start = now - WINDOW
     anomaly = (await session.execute(
@@ -188,7 +194,7 @@ async def _occur(session, key, level: float, norm: Norm, now: datetime) -> None:
         session.add(anomaly)
         await session.flush()
         session.add(MetricAnomalySpan(anomaly_id=anomaly.id, started_at=start, peak_level=level))
-        return
+        return anomaly
     last = (await session.execute(
         select(MetricAnomalySpan).where(MetricAnomalySpan.anomaly_id == anomaly.id)
         .order_by(MetricAnomalySpan.started_at.desc()).limit(1))).scalar_one_or_none()
@@ -202,17 +208,25 @@ async def _occur(session, key, level: float, norm: Norm, now: datetime) -> None:
         anomaly.last_started_at = start
     anomaly.ended_at = None
     _raise_peak(anomaly, span, level, norm, now)
+    return anomaly
 
 
 async def lapse(session, now: datetime, review_days: int) -> int:
     """Ended anomalies still to review after `review_days` lapse. One still
     going never does. reviewed_at records when it lapsed. Returns how many."""
-    result = await session.execute(
+    ids = list((await session.execute(
         update(MetricAnomaly)
         .where(MetricAnomaly.review == TO_REVIEW, MetricAnomaly.ended_at.is_not(None),
                MetricAnomaly.ended_at < now - timedelta(days=review_days))
-        .values(review=LAPSED, reviewed_at=now))
-    return result.rowcount or 0
+        .values(review=LAPSED, reviewed_at=now)
+        .returning(MetricAnomaly.id)
+        .execution_options(synchronize_session=False))).scalars())
+    if ids:
+        for anomaly in (await session.execute(
+                select(MetricAnomaly).where(MetricAnomaly.id.in_(ids))
+                .execution_options(populate_existing=True))).scalars():
+            await anomaly_changed(session, anomaly)
+    return len(ids)
 
 
 async def prune(session, before: datetime) -> int:
@@ -238,6 +252,7 @@ async def mark(session, anomaly_id: str, review: str, by: str,
     anomaly.reviewed_by = by
     anomaly.reviewed_at = now or datetime.now(timezone.utc)
     anomaly.note = (note or "").strip() or None
+    await anomaly_changed(session, anomaly)
     return anomaly