feat(timeline): maintenance, reboots, deploys, crash-loops and reviewed anomalies on the timeline (#6349)

The host agent writes each maintenance job once it ends, and each stretch a
host was away (a reboot, a crash, lost contact, the agent restarting), kept up
to date until it is back. Docker judges each snapshot: a unit on a new image,
or a container recreated, is a deploy; deaths past the Thresholds crash-loop
count are one crash-loop entry that later deaths extend; single restarts are
recorded but hidden. An anomaly has one entry whose type follows its review:
only those judged real are shown by default.

A source whose record changes keeps one entry for it (record_or_update).
Plugins register the records their entries point to, so an aged-out job or
downtime reads "Details no longer kept". The first fill runs once per source
(timeline.filled), the host agent's jobs and downtime included.

get_anomaly's "what else was going on" reads the timeline for the host or
resource instead of querying alerts, runs, maintenance, downtime and container
deaths itself; Docker's own source keeps only starts, stops and health
changes. A new kind, Observed, is for what Steward saw happen with no rule or
person behind it.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

tests/integration/test_mcp_anomaly_review.py

@@ -44,6 +44,7 @@ def test_an_anomaly_with_its_circumstances_judged_through_mcp(app):
     from steward.models.metrics import PluginMetric
     from steward.models.norms import MetricAnomaly, MetricAnomalySpan, MetricNorm
     from steward.models.users import User, UserRole
+    from steward.timeline.record import from_alert, from_run, record
 
     tag = uuid.uuid4().hex[:8]
     host_name, other_host = f"an-{tag}", f"an-other-{tag}"
@@ -98,21 +99,28 @@ def test_an_anomaly_with_its_circumstances_judged_through_mcp(app):
                                  created_by=user.id)
                 s.add(rule)
                 await s.flush()
-                s.add(AlertEvent(rule_id=rule.id, from_state="ok", to_state="firing", metric_value=91.0,
-                                 transitioned_at=new_start + timedelta(minutes=20)))
-                s.add(AnsibleRun(playbook_path="maintenance/backup.yml", source_name="steward-builtin",
+                fired = AlertEvent(rule_id=rule.id, from_state="ok", to_state="firing", metric_value=91.0,
+                                   transitioned_at=new_start + timedelta(minutes=20))
+                run = AnsibleRun(playbook_path="maintenance/backup.yml", source_name="steward-builtin",
                                  status=AnsibleRunStatus.success, started_at=new_start - timedelta(minutes=10),
                                  finished_at=new_start + timedelta(minutes=5), params={"check": False},
-                                 results={"hosts": {host_name: {"ok": 3}}, "failures": []}))
+                                 results={"hosts": {host_name: {"ok": 3}}, "failures": []})
+                s.add_all([fired, run])
+                await s.flush()
+                # The timeline, as the alert, the run and the plugins write it.
+                await from_alert(s, rule, fired)
+                await from_run(s, run)
+                record(s, type="docker.restarted", at=new_start + timedelta(minutes=15),
+                       summary=f"web on {host_name}: died (exit 137)", hosts=[host_name],
+                       subject=f"{host_name}/web")
+                record(s, type="host.rebooted", at=new_end - timedelta(minutes=5),
+                       until=new_end + timedelta(minutes=1), summary=f"{host_name}: rebooted for 6m",
+                       hosts=[host_name], subject=host_name)
+                # A container event the timeline doesn't hold, from Docker's own source.
                 await s.execute(text(
                     "INSERT INTO docker_events (id, host_id, container_name, event, detail, at) "
-                    "VALUES (:id, :h, 'web', 'die', 'exit 137', :at)"),
-                    {"id": str(uuid.uuid4()), "h": host.id, "at": new_start + timedelta(minutes=15)})
-                await s.execute(text(
-                    "INSERT INTO host_downtime (id, host_id, kind, started_at, ended_at, created_at) "
-                    "VALUES (:id, :h, 'reboot', :s, :e, now())"),
-                    {"id": str(uuid.uuid4()), "h": host.id, "s": new_end - timedelta(minutes=5),
-                     "e": new_end + timedelta(minutes=1)})
+                    "VALUES (:id, :h, 'web', 'health_change', 'healthy→unhealthy', :at)"),
+                    {"id": str(uuid.uuid4()), "h": host.id, "at": new_start + timedelta(minutes=16)})
 
     async def _go():
         await _seed()
@@ -127,12 +135,14 @@ def test_an_anomaly_with_its_circumstances_judged_through_mcp(app):
         assert newest["held_s"] == 3600 and newest["peak_level"] == 93.0
         assert newest["usual"]["usual_hold"] == 20.0 and newest["usual"]["flag_above"] >= 25.0
         kinds = {e["kind"] for e in newest["around"]}
-        assert {"alert", "ansible_run", "container", "downtime", "anomaly"} <= kinds
+        assert {"alert.fired", "ansible.unchanged", "docker.restarted", "host.rebooted", "container",
+                "anomaly"} <= kinds
         whats = " | ".join(e["what"] for e in newest["around"])
-        assert "web die (exit 137)" in whats and "maintenance/backup.yml" in whats
+        assert "died (exit 137)" in whats and "maintenance/backup.yml" in whats
+        assert "web health_change (healthy→unhealthy)" in whats
         assert any(e["kind"] == "anomaly" and e["ids"]["anomaly_id"] == ids["mem"] for e in newest["around"])
         assert newest["anomalies_elsewhere"] >= 1
-        assert not [e for e in oldest["around"] if e["kind"] in ("container", "ansible_run")]
+        assert not [e for e in oldest["around"] if e["kind"] in ("container", "ansible.unchanged")]
         points = got["series"]["points"]
         assert max(p[1] for p in points) == 90.0 and min(p[1] for p in points) == 18.0
         assert any(p[2] == 20.0 for p in points)

tests/integration/test_timeline.py

@@ -172,14 +172,15 @@ def test_the_first_fill_writes_what_predates_the_timeline_once(app):
     from steward.core.settings import get_setting, set_setting
     from steward.models.alerts import AlertEvent, AlertOperator, AlertRule
     from steward.models.users import User, UserRole
-    from steward.timeline.service import backfill
+    from steward.timeline.record import record
+    from steward.timeline.service import backfill, fill_sources
 
     async def _go():
         tag = uuid.uuid4().hex[:8]
         name = f"tl-{tag}"
         async with app.db_sessionmaker() as s:
             async with s.begin():
-                before = await get_setting(s, "timeline.backfilled_at")
+                before = await get_setting(s, "timeline.filled")
                 user = User(username=f"tl-{tag}", email=f"tl-{tag}@test", password_hash="x",
                             role=UserRole.admin)
                 s.add(user)
@@ -192,24 +193,31 @@ def test_the_first_fill_writes_what_predates_the_timeline_once(app):
                 # Written straight to the table, as before the timeline existed.
                 s.add(AlertEvent(rule_id=rule.id, from_state="pending", to_state="firing", metric_value=95.0,
                                  transitioned_at=datetime.now(timezone.utc) - timedelta(days=3)))
-                await set_setting(s, "timeline.backfilled_at", "")
+                await set_setting(s, "timeline.filled", {})
+
+        # A plugin's source: its entries are added beside core's, once.
+        async def plugin_source(session, since):
+            return [record(session, type="host.rebooted", at=datetime.now(timezone.utc) - timedelta(days=1),
+                           summary=f"{name}: rebooted for 2m", hosts=[name], ref=("test_fill", name))]
+
+        sources = {"steward": fill_sources()["steward"], "test_plugin": plugin_source}
         try:
-            assert await backfill(app) >= 1
-            [entry] = await _entries(app, name)
-            assert entry.type == "alert.fired" and entry.area == "storage"
-            assert entry.subject == f"{name}:/"
-            assert await backfill(app) == 0                         # once
+            assert await backfill(app, sources=sources) >= 2
+            got = {e.type: e for e in await _entries(app, name)}
+            assert set(got) == {"alert.fired", "host.rebooted"}
+            assert got["alert.fired"].area == "storage" and got["alert.fired"].subject == f"{name}:/"
+            async with app.db_sessionmaker() as s:
+                assert set(await get_setting(s, "timeline.filled")) == {"steward", "test_plugin"}
+            assert await backfill(app, sources=sources) == 0                 # once per source
             async with app.db_sessionmaker() as s:
                 async with s.begin():
-                    await set_setting(s, "timeline.backfilled_at", "")
-            await backfill(app)                                      # again: what is covered is skipped
-            assert len(await _entries(app, name)) == 1
+                    await set_setting(s, "timeline.filled", {"test_plugin": "done"})
+            assert await backfill(app, sources=sources) == 0                 # again: what is covered is skipped
+            assert len(await _entries(app, name)) == 2
         finally:
             async with app.db_sessionmaker() as s:
                 async with s.begin():
-                    await set_setting(s, "timeline.backfilled_at", before or "done")
-
-    asyncio.run(_go())
+                    await set_setting(s, "timeline.filled", before or {"steward": "done"})
 
 
 def test_retention_prunes_only_once_it_is_set(app):

tests/integration/test_timeline_sources.py

@@ -0,0 +1,186 @@
+"""Integration: the plugin sources and anomalies write the timeline as things
+happen. A host away is one entry kept up to date until it is back; a
+container on a new image is a deploy; deaths past the crash-loop count are
+one crash-loop entry; an anomaly's entry follows its review. The database is
+shared with other tests, so each test looks for its own uniquely named host.
+Requires STEWARD_DATABASE_URL.
+"""
+from __future__ import annotations
+
+import asyncio
+import os
+import uuid
+from datetime import datetime, timedelta, timezone
+
+import pytest
+
+pytestmark = pytest.mark.integration
+
+
+@pytest.fixture
+def app():
+    if not os.environ.get("STEWARD_DATABASE_URL"):
+        pytest.skip("needs Postgres")
+    from steward.app import create_app
+    return create_app(testing=False)
+
+
+async def _entries(app, host: str):
+    from sqlalchemy import select
+    from steward.models.timeline import TimelineEntry
+    async with app.db_sessionmaker() as s:
+        return (await s.execute(select(TimelineEntry).where(TimelineEntry.hosts.contains([host]))
+                                .order_by(TimelineEntry.at, TimelineEntry.id))).scalars().all()
+
+
+async def _host(app, name: str):
+    from steward.models.hosts import Host
+    async with app.db_sessionmaker() as s:
+        async with s.begin():
+            host = Host(name=name, address="10.7.7.7")
+            s.add(host)
+        return host
+
+
+def test_a_reboot_is_one_entry_that_gets_its_length_when_the_host_is_back(app):
+    from steward.core.plugin_manager import loaded_plugin_submodule
+    downtime = loaded_plugin_submodule("host_agent", "downtime")
+
+    async def _go():
+        name = f"tl-dt-{uuid.uuid4().hex[:8]}"
+        host = await _host(app, name)
+        down = datetime.now(timezone.utc).replace(microsecond=0) - timedelta(minutes=10)
+
+        async def post(events):
+            async with app.db_sessionmaker() as s:
+                async with s.begin():
+                    await downtime.record_events(s, host.id, events, down - timedelta(minutes=1))
+
+        await post([{"kind": "stopping", "at": down.isoformat(), "host_shutdown": True}])
+        [entry] = await _entries(app, name)
+        assert entry.type == "host.rebooted" and entry.until is None
+        assert entry.summary == f"{name}: rebooted, not back yet"
+        back = down + timedelta(minutes=3)
+        await post([{"kind": "started", "at": back.isoformat(),
+                     "boot_at": (down + timedelta(minutes=2)).isoformat()}])
+        [entry] = await _entries(app, name)                      # the same entry, now with its end
+        assert entry.until == back and entry.summary.startswith(f"{name}: rebooted for ")
+        assert entry.ref_type == "host_downtime" and entry.area == "host_os"
+        # Lost contact sorts under Network.
+        await post([{"kind": "lost_contact", "from": (back + timedelta(minutes=1)).isoformat(),
+                     "to": (back + timedelta(minutes=4)).isoformat()}])
+        lost = [e for e in await _entries(app, name) if e.type == "host.lost_contact"]
+        assert len(lost) == 1 and lost[0].area == "network"
+
+    asyncio.run(_go())
+
+
+def test_a_finished_maintenance_job_goes_on_the_timeline(app):
+    from sqlalchemy import text
+    from steward.core.plugin_manager import loaded_plugin_submodule
+    maintenance = loaded_plugin_submodule("host_agent", "maintenance")
+
+    async def _go():
+        name = f"tl-mj-{uuid.uuid4().hex[:8]}"
+        host = await _host(app, name)
+        job_id = str(uuid.uuid4())
+        now = datetime.now(timezone.utc)
+        async with app.db_sessionmaker() as s:
+            async with s.begin():
+                await s.execute(text(
+                    "INSERT INTO host_maintenance_jobs (id, host_id, state, state_at, reboot_policy, "
+                    "step_seen, step_id, packages, reboot_needed, created_at) "
+                    "VALUES (:id, :h, 'updating', :at, 'if_required', true, :step, NULL, NULL, :at)"),
+                    {"id": job_id, "h": host.id, "at": now - timedelta(minutes=5),
+                     "step": f"{job_id}-update"})
+        async with app.db_sessionmaker() as s:
+            async with s.begin():
+                assert await maintenance.apply_events(s, host.id, [
+                    {"kind": "maintenance", "id": f"{job_id}-update", "action": "update",
+                     "state": "done", "ok": True, "packages": 4, "reboot_required": False}], now) == 1
+        [entry] = await _entries(app, name)
+        assert entry.type == "maintenance.done"
+        assert (entry.ref_type, entry.ref_id) == ("maintenance_job", job_id)
+        assert entry.summary == f"Maintenance on {name}: Done: 4 packages updated, no reboot needed"
+        assert entry.until == now and entry.area == "host_os"
+
+    asyncio.run(_go())
+
+
+def test_docker_deploys_and_a_crash_loop_as_one_episode(app):
+    from steward.core.capabilities import get_capability
+
+    persist = get_capability("docker.persist_host_samples").fn
+
+    def c(name, image, cid, restarts=0, **kw):
+        return {"name": name, "container_id": cid, "image": image, "status": "running",
+                "cpu_pct": 1.0, "mem_used_pct": 1.0, "mem_used_bytes": 1, "mem_limit_bytes": 2,
+                "ports": [], "started_at": None, "health": None, "restart_count": restarts,
+                "exit_code": 0, "oom_killed": False, **kw}
+
+    async def _go():
+        name = f"tl-dk-{uuid.uuid4().hex[:8]}"
+        host = await _host(app, name)
+        t = datetime.now(timezone.utc) - timedelta(minutes=30)
+
+        async def snap(minutes, containers):
+            async with app.db_sessionmaker() as s:
+                async with s.begin():
+                    await persist(s, host, [(t + timedelta(minutes=minutes), containers)])
+
+        await snap(0, [c("web", "nginx:1.27", "a1"), c("job", "worker:3", "j1")])   # the baseline: nothing
+        assert await _entries(app, name) == []
+        await snap(1, [c("web", "nginx:1.28", "a2"), c("job", "worker:3", "j1")])
+        [deploy] = await _entries(app, name)
+        assert deploy.type == "docker.deployed" and deploy.subject == f"{name}/web"
+        assert deploy.summary == f"web on {name}: nginx:1.27 → nginx:1.28"
+        for i in (1, 2):
+            await snap(1 + i, [c("web", "nginx:1.28", "a2"), c("job", "worker:3", "j1", restarts=i)])
+        singles = [e for e in await _entries(app, name) if e.type == "docker.restarted"]
+        assert len(singles) == 2                                     # recorded, hidden
+        for i in (3, 4):
+            await snap(1 + i, [c("web", "nginx:1.28", "a2"), c("job", "worker:3", "j1", restarts=i)])
+        [loop] = [e for e in await _entries(app, name) if e.type == "docker.crash_loop"]
+        assert loop.detail["count"] == 4 and loop.until == t + timedelta(minutes=5)
+        assert loop.at == singles[0].at
+        assert loop.summary == f"job on {name} crash-looping: 4 restarts or deaths"
+        assert len([e for e in await _entries(app, name) if e.type == "docker.restarted"]) == 2
+
+    asyncio.run(_go())
+
+
+def test_an_anomaly_s_entry_follows_its_review(app):
+    from steward.core.anomalies import REAL, lapse, mark
+    from steward.models.norms import MetricAnomaly
+    from steward.timeline.record import anomaly_changed
+
+    async def _go():
+        name = f"tl-an-{uuid.uuid4().hex[:8]}"
+        await _host(app, name)
+        start = datetime.now(timezone.utc) - timedelta(days=20)
+
+        def anomaly(metric):
+            return MetricAnomaly(source_module="host_agent", resource_name=name, metric_name=metric,
+                                 started_at=start, last_started_at=start, ended_at=start + timedelta(hours=1),
+                                 occurrences=1, peak_level=93.0, peak_at=start, usual_hold=20.0,
+                                 usual_peak=35.0, review="to_review")
+
+        async with app.db_sessionmaker() as s:
+            async with s.begin():
+                cpu, mem = anomaly("cpu_pct"), anomaly("mem_used_pct")
+                s.add_all([cpu, mem])
+                await anomaly_changed(s, cpu)
+                await anomaly_changed(s, mem)
+        assert {e.type for e in await _entries(app, name)} == {"anomaly.to_review"}
+        async with app.db_sessionmaker() as s:
+            async with s.begin():
+                await mark(s, cpu.id, REAL, "tester", "the backup hung")
+                assert await lapse(s, datetime.now(timezone.utc), 14) >= 1
+        got = {e.ref_id: e for e in await _entries(app, name)}
+        assert len(got) == 2
+        assert got[cpu.id].type == "anomaly.real" and got[cpu.id].actor == "tester"
+        assert got[cpu.id].summary.endswith(": a real problem")
+        assert got[cpu.id].detail["note"] == "the backup hung"
+        assert got[mem.id].type == "anomaly.lapsed"
+
+    asyncio.run(_go())