feat(timeline): maintenance, reboots, deploys, crash-loops and reviewed anomalies on the timeline (#6349)

The host agent writes each maintenance job once it ends, and each stretch a
host was away (a reboot, a crash, lost contact, the agent restarting), kept up
to date until it is back. Docker judges each snapshot: a unit on a new image,
or a container recreated, is a deploy; deaths past the Thresholds crash-loop
count are one crash-loop entry that later deaths extend; single restarts are
recorded but hidden. An anomaly has one entry whose type follows its review:
only those judged real are shown by default.

A source whose record changes keeps one entry for it (record_or_update).
Plugins register the records their entries point to, so an aged-out job or
downtime reads "Details no longer kept". The first fill runs once per source
(timeline.filled), the host agent's jobs and downtime included.

get_anomaly's "what else was going on" reads the timeline for the host or
resource instead of querying alerts, runs, maintenance, downtime and container
deaths itself; Docker's own source keeps only starts, stops and health
changes. A new kind, Observed, is for what Steward saw happen with no rule or
person behind it.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

tests/integration/test_timeline_sources.py

@@ -0,0 +1,186 @@
+"""Integration: the plugin sources and anomalies write the timeline as things
+happen. A host away is one entry kept up to date until it is back; a
+container on a new image is a deploy; deaths past the crash-loop count are
+one crash-loop entry; an anomaly's entry follows its review. The database is
+shared with other tests, so each test looks for its own uniquely named host.
+Requires STEWARD_DATABASE_URL.
+"""
+from __future__ import annotations
+
+import asyncio
+import os
+import uuid
+from datetime import datetime, timedelta, timezone
+
+import pytest
+
+pytestmark = pytest.mark.integration
+
+
+@pytest.fixture
+def app():
+    if not os.environ.get("STEWARD_DATABASE_URL"):
+        pytest.skip("needs Postgres")
+    from steward.app import create_app
+    return create_app(testing=False)
+
+
+async def _entries(app, host: str):
+    from sqlalchemy import select
+    from steward.models.timeline import TimelineEntry
+    async with app.db_sessionmaker() as s:
+        return (await s.execute(select(TimelineEntry).where(TimelineEntry.hosts.contains([host]))
+                                .order_by(TimelineEntry.at, TimelineEntry.id))).scalars().all()
+
+
+async def _host(app, name: str):
+    from steward.models.hosts import Host
+    async with app.db_sessionmaker() as s:
+        async with s.begin():
+            host = Host(name=name, address="10.7.7.7")
+            s.add(host)
+        return host
+
+
+def test_a_reboot_is_one_entry_that_gets_its_length_when_the_host_is_back(app):
+    from steward.core.plugin_manager import loaded_plugin_submodule
+    downtime = loaded_plugin_submodule("host_agent", "downtime")
+
+    async def _go():
+        name = f"tl-dt-{uuid.uuid4().hex[:8]}"
+        host = await _host(app, name)
+        down = datetime.now(timezone.utc).replace(microsecond=0) - timedelta(minutes=10)
+
+        async def post(events):
+            async with app.db_sessionmaker() as s:
+                async with s.begin():
+                    await downtime.record_events(s, host.id, events, down - timedelta(minutes=1))
+
+        await post([{"kind": "stopping", "at": down.isoformat(), "host_shutdown": True}])
+        [entry] = await _entries(app, name)
+        assert entry.type == "host.rebooted" and entry.until is None
+        assert entry.summary == f"{name}: rebooted, not back yet"
+        back = down + timedelta(minutes=3)
+        await post([{"kind": "started", "at": back.isoformat(),
+                     "boot_at": (down + timedelta(minutes=2)).isoformat()}])
+        [entry] = await _entries(app, name)                      # the same entry, now with its end
+        assert entry.until == back and entry.summary.startswith(f"{name}: rebooted for ")
+        assert entry.ref_type == "host_downtime" and entry.area == "host_os"
+        # Lost contact sorts under Network.
+        await post([{"kind": "lost_contact", "from": (back + timedelta(minutes=1)).isoformat(),
+                     "to": (back + timedelta(minutes=4)).isoformat()}])
+        lost = [e for e in await _entries(app, name) if e.type == "host.lost_contact"]
+        assert len(lost) == 1 and lost[0].area == "network"
+
+    asyncio.run(_go())
+
+
+def test_a_finished_maintenance_job_goes_on_the_timeline(app):
+    from sqlalchemy import text
+    from steward.core.plugin_manager import loaded_plugin_submodule
+    maintenance = loaded_plugin_submodule("host_agent", "maintenance")
+
+    async def _go():
+        name = f"tl-mj-{uuid.uuid4().hex[:8]}"
+        host = await _host(app, name)
+        job_id = str(uuid.uuid4())
+        now = datetime.now(timezone.utc)
+        async with app.db_sessionmaker() as s:
+            async with s.begin():
+                await s.execute(text(
+                    "INSERT INTO host_maintenance_jobs (id, host_id, state, state_at, reboot_policy, "
+                    "step_seen, step_id, packages, reboot_needed, created_at) "
+                    "VALUES (:id, :h, 'updating', :at, 'if_required', true, :step, NULL, NULL, :at)"),
+                    {"id": job_id, "h": host.id, "at": now - timedelta(minutes=5),
+                     "step": f"{job_id}-update"})
+        async with app.db_sessionmaker() as s:
+            async with s.begin():
+                assert await maintenance.apply_events(s, host.id, [
+                    {"kind": "maintenance", "id": f"{job_id}-update", "action": "update",
+                     "state": "done", "ok": True, "packages": 4, "reboot_required": False}], now) == 1
+        [entry] = await _entries(app, name)
+        assert entry.type == "maintenance.done"
+        assert (entry.ref_type, entry.ref_id) == ("maintenance_job", job_id)
+        assert entry.summary == f"Maintenance on {name}: Done: 4 packages updated, no reboot needed"
+        assert entry.until == now and entry.area == "host_os"
+
+    asyncio.run(_go())
+
+
+def test_docker_deploys_and_a_crash_loop_as_one_episode(app):
+    from steward.core.capabilities import get_capability
+
+    persist = get_capability("docker.persist_host_samples").fn
+
+    def c(name, image, cid, restarts=0, **kw):
+        return {"name": name, "container_id": cid, "image": image, "status": "running",
+                "cpu_pct": 1.0, "mem_used_pct": 1.0, "mem_used_bytes": 1, "mem_limit_bytes": 2,
+                "ports": [], "started_at": None, "health": None, "restart_count": restarts,
+                "exit_code": 0, "oom_killed": False, **kw}
+
+    async def _go():
+        name = f"tl-dk-{uuid.uuid4().hex[:8]}"
+        host = await _host(app, name)
+        t = datetime.now(timezone.utc) - timedelta(minutes=30)
+
+        async def snap(minutes, containers):
+            async with app.db_sessionmaker() as s:
+                async with s.begin():
+                    await persist(s, host, [(t + timedelta(minutes=minutes), containers)])
+
+        await snap(0, [c("web", "nginx:1.27", "a1"), c("job", "worker:3", "j1")])   # the baseline: nothing
+        assert await _entries(app, name) == []
+        await snap(1, [c("web", "nginx:1.28", "a2"), c("job", "worker:3", "j1")])
+        [deploy] = await _entries(app, name)
+        assert deploy.type == "docker.deployed" and deploy.subject == f"{name}/web"
+        assert deploy.summary == f"web on {name}: nginx:1.27 → nginx:1.28"
+        for i in (1, 2):
+            await snap(1 + i, [c("web", "nginx:1.28", "a2"), c("job", "worker:3", "j1", restarts=i)])
+        singles = [e for e in await _entries(app, name) if e.type == "docker.restarted"]
+        assert len(singles) == 2                                     # recorded, hidden
+        for i in (3, 4):
+            await snap(1 + i, [c("web", "nginx:1.28", "a2"), c("job", "worker:3", "j1", restarts=i)])
+        [loop] = [e for e in await _entries(app, name) if e.type == "docker.crash_loop"]
+        assert loop.detail["count"] == 4 and loop.until == t + timedelta(minutes=5)
+        assert loop.at == singles[0].at
+        assert loop.summary == f"job on {name} crash-looping: 4 restarts or deaths"
+        assert len([e for e in await _entries(app, name) if e.type == "docker.restarted"]) == 2
+
+    asyncio.run(_go())
+
+
+def test_an_anomaly_s_entry_follows_its_review(app):
+    from steward.core.anomalies import REAL, lapse, mark
+    from steward.models.norms import MetricAnomaly
+    from steward.timeline.record import anomaly_changed
+
+    async def _go():
+        name = f"tl-an-{uuid.uuid4().hex[:8]}"
+        await _host(app, name)
+        start = datetime.now(timezone.utc) - timedelta(days=20)
+
+        def anomaly(metric):
+            return MetricAnomaly(source_module="host_agent", resource_name=name, metric_name=metric,
+                                 started_at=start, last_started_at=start, ended_at=start + timedelta(hours=1),
+                                 occurrences=1, peak_level=93.0, peak_at=start, usual_hold=20.0,
+                                 usual_peak=35.0, review="to_review")
+
+        async with app.db_sessionmaker() as s:
+            async with s.begin():
+                cpu, mem = anomaly("cpu_pct"), anomaly("mem_used_pct")
+                s.add_all([cpu, mem])
+                await anomaly_changed(s, cpu)
+                await anomaly_changed(s, mem)
+        assert {e.type for e in await _entries(app, name)} == {"anomaly.to_review"}
+        async with app.db_sessionmaker() as s:
+            async with s.begin():
+                await mark(s, cpu.id, REAL, "tester", "the backup hung")
+                assert await lapse(s, datetime.now(timezone.utc), 14) >= 1
+        got = {e.ref_id: e for e in await _entries(app, name)}
+        assert len(got) == 2
+        assert got[cpu.id].type == "anomaly.real" and got[cpu.id].actor == "tester"
+        assert got[cpu.id].summary.endswith(": a real problem")
+        assert got[cpu.id].detail["note"] == "the backup hung"
+        assert got[mem.id].type == "anomaly.lapsed"
+
+    asyncio.run(_go())