feat(docker): ingest swarm topology + lifecycle events + health/restart alerts

Wires the agent's enriched + swarm payloads through the docker.persist_host_
samples capability:

  * Swarm topology — persist sample["swarm"] into docker_swarm_services /
    docker_swarm_nodes (upsert + prune stale, host-scoped so two managers don't
    clobber). Migration docker_005 adds services.placement_json for the
    task→node placement the agent now reports.
  * Lifecycle events — _derive_events (pure, unit-tested) diffs the newest
    snapshot against stored per-container state: start / stop / die (non-zero
    exit) / oom / health_change → docker_events rows. Skipped on a host's first
    snapshot so the baseline doesn't emit a start per existing container.
  * Alerts — record restart_count (always) and is_healthy (1.0/0.0, only when a
    HEALTHCHECK exists) alongside cpu/mem, under host-scoped resource names;
    METRIC_CATALOG[docker] gains restart_count + is_healthy so they're alertable.

host_agent ingest captures the newest sample's swarm object and threads it to
the capability (now persist_host_docker(session, host, snapshots, swarm=None));
invoked when containers OR swarm are present, under the same SAVEPOINT. Unit
tests cover the event-diff matrix; integration tests cover event derivation
across two snapshots and swarm topology round-trip (incl. placement).

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_016Jg27rgypiW2efULXJDtMC

tests/integration/test_docker.py

@@ -139,3 +139,83 @@ def test_persist_scopes_containers_by_host(app):
     assert metric_hosts == 2      # time-series rows scoped per host
     assert resources == {"alpha/web", "beta/web"}
     assert enrich == ("healthy", 2, "web", 1000)   # enrichment round-trips
+
+
+@_NEEDS_DB
+def test_lifecycle_events_derived_across_snapshots(app):
+    from sqlalchemy import text
+    from steward.models.hosts import Host
+
+    persist = _persist_fn(app)
+    t1 = datetime(2026, 6, 19, 12, 0, 0, tzinfo=timezone.utc)
+    t2 = datetime(2026, 6, 19, 12, 0, 30, tzinfo=timezone.utc)
+    running = {"name": "web", "container_id": "abc", "image": "nginx",
+               "status": "running", "cpu_pct": 5.0, "mem_pct": 10.0,
+               "restart_count": 0, "exit_code": None, "oom_killed": False,
+               "health": "healthy"}
+    died = {**running, "status": "exited", "cpu_pct": None,
+            "exit_code": 137, "oom_killed": True, "health": None}
+
+    async def _go():
+        async with app.db_sessionmaker() as s:
+            async with s.begin():
+                await s.execute(text("DELETE FROM docker_events"))
+                await s.execute(text("DELETE FROM docker_metrics"))
+                await s.execute(text("DELETE FROM docker_containers"))
+                h = Host(id=str(uuid.uuid4()), name="evt", address="10.9.9.9")
+                s.add(h)
+                await s.flush()
+                # Baseline snapshot establishes state — must NOT emit a start.
+                await persist(s, h, [(t1, [running])])
+                # Second snapshot: the container OOM-died.
+                await persist(s, h, [(t2, [died])])
+                hid = h.id
+            rows = (await s.execute(text(
+                "SELECT event, detail FROM docker_events WHERE host_id = :h "
+                "ORDER BY event"), {"h": hid})).all()
+            return [(r[0], r[1]) for r in rows]
+
+    events = asyncio.run(_go())
+    # baseline emitted nothing; the death emitted both die (exit 137) and oom.
+    assert ("die", "exit 137") in events
+    assert ("oom", None) in events
+    assert not any(e[0] == "start" for e in events)
+
+
+@_NEEDS_DB
+def test_swarm_topology_persisted(app):
+    from sqlalchemy import text
+    from steward.models.hosts import Host
+
+    persist = _persist_fn(app)
+    swarm = {
+        "services": [{"service_name": "web", "mode": "replicated",
+                      "desired": 3, "running": 2, "image": "nginx",
+                      "placement": [{"node_id": "n1", "running": 2}]}],
+        "nodes": [{"node_id": "n1", "hostname": "mgr", "role": "manager",
+                   "availability": "active", "status": "ready", "leader": True}],
+    }
+
+    async def _go():
+        async with app.db_sessionmaker() as s:
+            async with s.begin():
+                await s.execute(text("DELETE FROM docker_swarm_services"))
+                await s.execute(text("DELETE FROM docker_swarm_nodes"))
+                h = Host(id=str(uuid.uuid4()), name="mgr", address="10.8.8.8")
+                s.add(h)
+                await s.flush()
+                # No containers in this sample — swarm must still persist.
+                await persist(s, h, [], swarm)
+                hid = h.id
+            svc = (await s.execute(text(
+                "SELECT mode, desired, running, image, placement_json "
+                "FROM docker_swarm_services WHERE host_id = :h"), {"h": hid})).first()
+            node = (await s.execute(text(
+                "SELECT role, availability, status, leader "
+                "FROM docker_swarm_nodes WHERE host_id = :h"), {"h": hid})).first()
+            return svc, node
+
+    svc, node = asyncio.run(_go())
+    assert svc[0] == "replicated" and svc[1] == 3 and svc[2] == 2 and svc[3] == "nginx"
+    assert "n1" in svc[4]  # placement JSON carries the node id
+    assert node[0] == "manager" and node[2] == "ready" and node[3] is True

tests/plugins/docker/test_ingest_events.py

@@ -0,0 +1,67 @@
+"""Unit tests for the docker plugin's lifecycle-event derivation.
+
+_derive_events is a pure function (no DB), and ingest.py imports its models
+lazily inside functions, so importing it here doesn't register ORM tables —
+safe for the no-DB unit lane.
+"""
+from plugins.docker.ingest import _derive_events
+
+
+def _c(name, status, **kw):
+    return {"name": name, "status": status, **kw}
+
+
+def test_start_on_running_after_stopped():
+    old = {"web": {"status": "exited", "health": None, "oom_killed": False, "exit_code": 0}}
+    events = _derive_events(old, [_c("web", "running", image="nginx")])
+    assert events == [("web", "start", "nginx")]
+
+
+def test_start_for_new_container_when_state_exists():
+    # old_state is non-empty (host already seen), a fresh container appears running.
+    old = {"other": {"status": "running", "health": None, "oom_killed": False, "exit_code": None}}
+    events = _derive_events(old, [
+        _c("other", "running"),
+        _c("new", "running", image="redis"),
+    ])
+    assert ("new", "start", "redis") in events
+    # the unchanged 'other' produces nothing
+    assert all(e[0] != "other" for e in events)
+
+
+def test_stop_on_clean_exit():
+    old = {"job": {"status": "running", "health": None, "oom_killed": False, "exit_code": None}}
+    events = _derive_events(old, [_c("job", "exited", exit_code=0)])
+    assert events == [("job", "stop", None)]
+
+
+def test_die_on_nonzero_exit():
+    old = {"job": {"status": "running", "health": None, "oom_killed": False, "exit_code": None}}
+    events = _derive_events(old, [_c("job", "exited", exit_code=137)])
+    assert events == [("job", "die", "exit 137")]
+
+
+def test_oom_on_flip_to_true():
+    old = {"hog": {"status": "running", "health": None, "oom_killed": False, "exit_code": None}}
+    # OOM-killed → also no longer running with a non-zero exit, so die + oom both fire.
+    events = _derive_events(old, [_c("hog", "exited", exit_code=137, oom_killed=True)])
+    assert ("hog", "oom", None) in events
+    assert ("hog", "die", "exit 137") in events
+
+
+def test_health_change_emitted():
+    old = {"svc": {"status": "running", "health": "healthy", "oom_killed": False, "exit_code": None}}
+    events = _derive_events(old, [_c("svc", "running", health="unhealthy")])
+    assert events == [("svc", "health_change", "healthy→unhealthy")]
+
+
+def test_removed_running_container_stops():
+    old = {"gone": {"status": "running", "health": None, "oom_killed": False, "exit_code": None}}
+    events = _derive_events(old, [])  # vanished from the listing entirely
+    assert events == [("gone", "stop", "removed")]
+
+
+def test_no_event_when_unchanged():
+    old = {"web": {"status": "running", "health": "healthy", "oom_killed": False, "exit_code": None}}
+    events = _derive_events(old, [_c("web", "running", health="healthy")])
+    assert events == []