feat(timeline): maintenance, reboots, deploys, crash-loops and reviewed anomalies on the timeline (#6349)

The host agent writes each maintenance job once it ends, and each stretch a
host was away (a reboot, a crash, lost contact, the agent restarting), kept up
to date until it is back. Docker judges each snapshot: a unit on a new image,
or a container recreated, is a deploy; deaths past the Thresholds crash-loop
count are one crash-loop entry that later deaths extend; single restarts are
recorded but hidden. An anomaly has one entry whose type follows its review:
only those judged real are shown by default.

A source whose record changes keeps one entry for it (record_or_update).
Plugins register the records their entries point to, so an aged-out job or
downtime reads "Details no longer kept". The first fill runs once per source
(timeline.filled), the host agent's jobs and downtime included.

get_anomaly's "what else was going on" reads the timeline for the host or
resource instead of querying alerts, runs, maintenance, downtime and container
deaths itself; Docker's own source keeps only starts, stops and health
changes. A new kind, Observed, is for what Steward saw happen with no rule or
person behind it.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

tests/integration/test_timeline.py

@@ -172,14 +172,15 @@ def test_the_first_fill_writes_what_predates_the_timeline_once(app):
     from steward.core.settings import get_setting, set_setting
     from steward.models.alerts import AlertEvent, AlertOperator, AlertRule
     from steward.models.users import User, UserRole
-    from steward.timeline.service import backfill
+    from steward.timeline.record import record
+    from steward.timeline.service import backfill, fill_sources
 
     async def _go():
         tag = uuid.uuid4().hex[:8]
         name = f"tl-{tag}"
         async with app.db_sessionmaker() as s:
             async with s.begin():
-                before = await get_setting(s, "timeline.backfilled_at")
+                before = await get_setting(s, "timeline.filled")
                 user = User(username=f"tl-{tag}", email=f"tl-{tag}@test", password_hash="x",
                             role=UserRole.admin)
                 s.add(user)
@@ -192,24 +193,31 @@ def test_the_first_fill_writes_what_predates_the_timeline_once(app):
                 # Written straight to the table, as before the timeline existed.
                 s.add(AlertEvent(rule_id=rule.id, from_state="pending", to_state="firing", metric_value=95.0,
                                  transitioned_at=datetime.now(timezone.utc) - timedelta(days=3)))
-                await set_setting(s, "timeline.backfilled_at", "")
+                await set_setting(s, "timeline.filled", {})
+
+        # A plugin's source: its entries are added beside core's, once.
+        async def plugin_source(session, since):
+            return [record(session, type="host.rebooted", at=datetime.now(timezone.utc) - timedelta(days=1),
+                           summary=f"{name}: rebooted for 2m", hosts=[name], ref=("test_fill", name))]
+
+        sources = {"steward": fill_sources()["steward"], "test_plugin": plugin_source}
         try:
-            assert await backfill(app) >= 1
-            [entry] = await _entries(app, name)
-            assert entry.type == "alert.fired" and entry.area == "storage"
-            assert entry.subject == f"{name}:/"
-            assert await backfill(app) == 0                         # once
+            assert await backfill(app, sources=sources) >= 2
+            got = {e.type: e for e in await _entries(app, name)}
+            assert set(got) == {"alert.fired", "host.rebooted"}
+            assert got["alert.fired"].area == "storage" and got["alert.fired"].subject == f"{name}:/"
+            async with app.db_sessionmaker() as s:
+                assert set(await get_setting(s, "timeline.filled")) == {"steward", "test_plugin"}
+            assert await backfill(app, sources=sources) == 0                 # once per source
             async with app.db_sessionmaker() as s:
                 async with s.begin():
-                    await set_setting(s, "timeline.backfilled_at", "")
-            await backfill(app)                                      # again: what is covered is skipped
-            assert len(await _entries(app, name)) == 1
+                    await set_setting(s, "timeline.filled", {"test_plugin": "done"})
+            assert await backfill(app, sources=sources) == 0                 # again: what is covered is skipped
+            assert len(await _entries(app, name)) == 2
         finally:
             async with app.db_sessionmaker() as s:
                 async with s.begin():
-                    await set_setting(s, "timeline.backfilled_at", before or "done")
-
-    asyncio.run(_go())
+                    await set_setting(s, "timeline.filled", before or {"steward": "done"})
 
 
 def test_retention_prunes_only_once_it_is_set(app):