feat(docker): per-container enrichment — health, restarts, exit code, I/O, grouping

First slice of milestone 77 (Docker monitoring depth). Surfaces real per-container
stats beyond basic state, all read-only on the existing push model.

- agent (→1.4.0): collect_docker now inspects each container (health, restart
  count, exit code, OOM) and reads net + block I/O from the stats payload; pulls
  compose project + swarm service/task/node from container labels. Per-container
  inspect+stats calls run over a small bounded ThreadPool so the ~1s-per-stats
  blocking doesn't stretch the sample on a busy host.
- schema (docker_003): additive columns on docker_containers — health, exit_code,
  oom_killed, compose_project, service_name, task_id, node_id, and BigInteger
  net/blk byte counters.
- ingest: persists the enrichment + restart_count (.get keeps older agents working).
- ui: Docker page rows now show health badge, uptime ("up 3d 4h"), restart count,
  exit code (+OOM) for stopped containers, and compose/service grouping label.
- tests: agent helpers (grouping, inspect fields, net/IO sum) + collect_docker
  assembly incl. inspect; integration asserts enrichment round-trips.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_016Jg27rgypiW2efULXJDtMC

tests/integration/test_docker.py

@@ -75,6 +75,11 @@ def test_persist_scopes_containers_by_host(app):
         "name": "web", "container_id": "abc123", "image": "nginx",
         "status": "running", "cpu_pct": 12.5, "mem_pct": 30.0,
         "mem_usage_bytes": 100, "mem_limit_bytes": 200, "ports": [], "started_at": None,
+        # enrichment (agent ≥ 1.4.0)
+        "health": "healthy", "restart_count": 2, "exit_code": 0, "oom_killed": False,
+        "compose_project": "stack", "service_name": "web",
+        "net_rx_bytes": 1000, "net_tx_bytes": 2000,
+        "blk_read_bytes": 4096, "blk_write_bytes": 8192,
     }])]
 
     async def _go():
@@ -101,9 +106,14 @@ def test_persist_scopes_containers_by_host(app):
             resources = {r[0] for r in (await s.execute(text(
                 "SELECT DISTINCT resource_name FROM plugin_metrics "
                 "WHERE source_module = 'docker'"))).all()}
-            return web_rows, metric_hosts, resources
+            # Enrichment columns persisted (check one host's row).
+            enrich = (await s.execute(text(
+                "SELECT health, restart_count, service_name, net_rx_bytes "
+                "FROM docker_containers WHERE name = 'web' LIMIT 1"))).first()
+            return web_rows, metric_hosts, resources, enrich
 
-    web_rows, metric_hosts, resources = asyncio.run(_go())
+    web_rows, metric_hosts, resources, enrich = asyncio.run(_go())
     assert web_rows == 2          # one 'web' per host, keyed by (host_id, name)
     assert metric_hosts == 2      # time-series rows scoped per host
     assert resources == {"alpha/web", "beta/web"}
+    assert enrich == ("healthy", 2, "web", 1000)   # enrichment round-trips

tests/plugins/host_agent/test_agent_docker.py

@@ -68,6 +68,47 @@ def test_docker_ports_keeps_only_published():
     ]
 
 
+# ── enrichment helpers ────────────────────────────────────────────────────────
+
+def test_docker_grouping_from_labels():
+    g = a._docker_grouping({
+        "com.docker.compose.project": "myproj",
+        "com.docker.swarm.service.name": "web",
+        "com.docker.swarm.task.id": "t123",
+        "com.docker.swarm.node.id": "n456",
+    })
+    assert g == {"compose_project": "myproj", "service_name": "web",
+                 "task_id": "t123", "node_id": "n456"}
+
+
+def test_docker_grouping_empty():
+    assert a._docker_grouping(None) == {
+        "compose_project": None, "service_name": None, "task_id": None, "node_id": None}
+
+
+def test_docker_inspect_fields():
+    insp = {"RestartCount": 3,
+            "State": {"Health": {"Status": "unhealthy"}, "ExitCode": 137, "OOMKilled": True}}
+    assert a._docker_inspect_fields(insp) == ("unhealthy", 3, 137, True)
+
+
+def test_docker_inspect_fields_no_healthcheck():
+    # Containers without a HEALTHCHECK have no State.Health → health is None.
+    health, restarts, exit_code, oom = a._docker_inspect_fields(
+        {"RestartCount": 0, "State": {"ExitCode": 0, "OOMKilled": False}})
+    assert health is None and restarts == 0 and exit_code == 0 and oom is False
+
+
+def test_docker_net_io_sums_counters():
+    stats = {
+        "networks": {"eth0": {"rx_bytes": 1000, "tx_bytes": 2000},
+                     "eth1": {"rx_bytes": 5, "tx_bytes": 7}},
+        "blkio_stats": {"io_service_bytes_recursive": [
+            {"op": "Read", "value": 4096}, {"op": "Write", "value": 8192}]},
+    }
+    assert a._docker_net_io(stats) == (1005, 2007, 4096, 8192)
+
+
 # ── collect_docker assembly + silent skip ─────────────────────────────────────
 
 def test_collect_docker_silent_skip_when_no_socket():
@@ -83,7 +124,11 @@ def test_collect_docker_assembles_container(monkeypatch):
         "State": "running",
         "Created": 1_700_000_000,
         "Ports": [{"PublicPort": 8080, "PrivatePort": 80, "Type": "tcp"}],
+        "Labels": {"com.docker.compose.project": "myproj",
+                   "com.docker.swarm.service.name": "web"},
     }
+    inspect = {"RestartCount": 2,
+               "State": {"Health": {"Status": "healthy"}, "ExitCode": 0, "OOMKilled": False}}
     stats = {
         "cpu_stats": {"cpu_usage": {"total_usage": 200_000_000},
                       "system_cpu_usage": 2_000_000_000, "online_cpus": 4},
@@ -91,10 +136,19 @@ def test_collect_docker_assembles_container(monkeypatch):
                          "system_cpu_usage": 1_000_000_000},
         "memory_stats": {"usage": 200 * 1024 * 1024, "limit": 1000 * 1024 * 1024,
                          "stats": {"cache": 50 * 1024 * 1024}},
+        "networks": {"eth0": {"rx_bytes": 1000, "tx_bytes": 2000}},
+        "blkio_stats": {"io_service_bytes_recursive": [
+            {"op": "Read", "value": 4096}, {"op": "Write", "value": 8192}]},
     }
 
     def fake_request(socket_path, path, timeout=a.DOCKER_API_TIMEOUT):
-        return [container] if path.startswith("/containers/json") else stats
+        if path.startswith("/containers/json"):
+            return [container]
+        if "/stats" in path:
+            return stats
+        if path.endswith("/json"):     # inspect
+            return inspect
+        raise AssertionError(f"unexpected path {path}")
 
     monkeypatch.setattr(a, "_docker_request", fake_request)
     out = a.collect_docker("/var/run/docker.sock")
@@ -108,21 +162,34 @@ def test_collect_docker_assembles_container(monkeypatch):
     assert c["mem_pct"] == 15.0
     assert c["ports"] == [{"host_port": 8080, "container_port": 80, "protocol": "tcp"}]
     assert c["started_at"].startswith("2023-11-14T")  # epoch → ISO-8601
+    # enrichment
+    assert c["health"] == "healthy" and c["restart_count"] == 2
+    assert c["exit_code"] == 0 and c["oom_killed"] is False
+    assert c["net_rx_bytes"] == 1000 and c["net_tx_bytes"] == 2000
+    assert c["blk_read_bytes"] == 4096 and c["blk_write_bytes"] == 8192
+    assert c["compose_project"] == "myproj" and c["service_name"] == "web"
 
 
 def test_collect_docker_skips_stats_for_stopped(monkeypatch):
     container = {"Id": "x" * 16, "Names": ["/db"], "Image": "pg", "State": "exited",
-                 "Created": None, "Ports": []}
+                 "Created": None, "Ports": [], "Labels": {}}
+    inspect = {"RestartCount": 0,
+               "State": {"ExitCode": 137, "OOMKilled": True}}  # no Health block
 
     def fake_request(socket_path, path, timeout=a.DOCKER_API_TIMEOUT):
         if path.startswith("/containers/json"):
             return [container]
+        if path.endswith("/json"):     # inspect — fetched for stopped too (exit code)
+            return inspect
         raise AssertionError("stats must not be fetched for a stopped container")
 
     monkeypatch.setattr(a, "_docker_request", fake_request)
     out = a.collect_docker("/var/run/docker.sock")
     assert out[0]["status"] == "exited"
     assert out[0]["cpu_pct"] is None and out[0]["started_at"] is None
+    # inspect still gives us why it died
+    assert out[0]["exit_code"] == 137 and out[0]["oom_killed"] is True
+    assert out[0]["health"] is None
 
 
 # ── build_sample wiring ───────────────────────────────────────────────────────