feat(docker): per-container enrichment — health, restarts, exit code, I/O, grouping

First slice of milestone 77 (Docker monitoring depth). Surfaces real per-container
stats beyond basic state, all read-only on the existing push model.

- agent (→1.4.0): collect_docker now inspects each container (health, restart
  count, exit code, OOM) and reads net + block I/O from the stats payload; pulls
  compose project + swarm service/task/node from container labels. Per-container
  inspect+stats calls run over a small bounded ThreadPool so the ~1s-per-stats
  blocking doesn't stretch the sample on a busy host.
- schema (docker_003): additive columns on docker_containers — health, exit_code,
  oom_killed, compose_project, service_name, task_id, node_id, and BigInteger
  net/blk byte counters.
- ingest: persists the enrichment + restart_count (.get keeps older agents working).
- ui: Docker page rows now show health badge, uptime ("up 3d 4h"), restart count,
  exit code (+OOM) for stopped containers, and compose/service grouping label.
- tests: agent helpers (grouping, inspect fields, net/IO sum) + collect_docker
  assembly incl. inspect; integration asserts enrichment round-trips.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_016Jg27rgypiW2efULXJDtMC

plugins/docker/ingest.py

@@ -73,6 +73,20 @@ async def persist_host_docker(session, host, snapshots) -> None:
         existing.ports_json = json.dumps(c.get("ports") or [])
         existing.started_at = _parse_started_at(c.get("started_at"))
         existing.scraped_at = latest_at
+        # Enrichment (agent ≥ 1.4.0; .get keeps older agents working — fields
+        # stay None/0 when absent from the payload).
+        existing.restart_count = c.get("restart_count", 0) or 0
+        existing.health = c.get("health")
+        existing.exit_code = c.get("exit_code")
+        existing.oom_killed = bool(c.get("oom_killed", False))
+        existing.compose_project = c.get("compose_project")
+        existing.service_name = c.get("service_name")
+        existing.task_id = c.get("task_id")
+        existing.node_id = c.get("node_id")
+        existing.net_rx_bytes = c.get("net_rx_bytes")
+        existing.net_tx_bytes = c.get("net_tx_bytes")
+        existing.blk_read_bytes = c.get("blk_read_bytes")
+        existing.blk_write_bytes = c.get("blk_write_bytes")
 
         # Alert pipeline — resource is host-scoped so containers of the same name
         # on different hosts don't collide in the metric/alert namespace.

plugins/docker/migrations/versions/docker_003_container_enrichment.py

@@ -0,0 +1,41 @@
+"""Docker container enrichment: health, exit/restart, grouping, I/O counters
+
+Adds the fields the agent (≥1.4.0) now reports per container beyond the basic
+state: health status, exit code, OOM flag, compose/swarm grouping labels, and
+cumulative network/block I/O counters. Additive columns — no data loss, so no
+DROP+recreate needed here.
+
+Revision ID: docker_003_container_enrichment
+Revises: docker_002_host_scoped
+Create Date: 2026-06-19
+"""
+from typing import Sequence, Union
+from alembic import op
+import sqlalchemy as sa
+
+revision: str = "docker_003_container_enrichment"
+down_revision: Union[str, None] = "docker_002_host_scoped"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    op.add_column("docker_containers", sa.Column("health", sa.String(16), nullable=True))
+    op.add_column("docker_containers", sa.Column("exit_code", sa.Integer, nullable=True))
+    op.add_column("docker_containers",
+                  sa.Column("oom_killed", sa.Boolean, nullable=False, server_default=sa.false()))
+    op.add_column("docker_containers", sa.Column("compose_project", sa.String(255), nullable=True))
+    op.add_column("docker_containers", sa.Column("service_name", sa.String(255), nullable=True))
+    op.add_column("docker_containers", sa.Column("task_id", sa.String(64), nullable=True))
+    op.add_column("docker_containers", sa.Column("node_id", sa.String(64), nullable=True))
+    op.add_column("docker_containers", sa.Column("net_rx_bytes", sa.BigInteger, nullable=True))
+    op.add_column("docker_containers", sa.Column("net_tx_bytes", sa.BigInteger, nullable=True))
+    op.add_column("docker_containers", sa.Column("blk_read_bytes", sa.BigInteger, nullable=True))
+    op.add_column("docker_containers", sa.Column("blk_write_bytes", sa.BigInteger, nullable=True))
+
+
+def downgrade() -> None:
+    for col in ("blk_write_bytes", "blk_read_bytes", "net_tx_bytes", "net_rx_bytes",
+                "node_id", "task_id", "service_name", "compose_project",
+                "oom_killed", "exit_code", "health"):
+        op.drop_column("docker_containers", col)

plugins/docker/models.py

@@ -2,7 +2,9 @@
 from __future__ import annotations
 import uuid
 from datetime import datetime, timezone
-from sqlalchemy import DateTime, Float, ForeignKey, Index, Integer, String, Text
+from sqlalchemy import (
+    BigInteger, Boolean, DateTime, Float, ForeignKey, Index, Integer, String, Text,
+)
 from sqlalchemy.orm import Mapped, mapped_column
 from steward.models.base import Base
 
@@ -38,6 +40,23 @@ class DockerContainer(Base):
         default=lambda: datetime.now(timezone.utc),
     )
 
+    # ── Enrichment (agent ≥ 1.4.0) ────────────────────────────────────────────
+    # Health/exit/restart come from `docker inspect` (not the list endpoint);
+    # exit_code is only meaningful for stopped containers.
+    health: Mapped[str | None] = mapped_column(String(16), nullable=True)  # healthy|unhealthy|starting
+    exit_code: Mapped[int | None] = mapped_column(Integer, nullable=True)
+    oom_killed: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
+    # Grouping: compose project + swarm placement, read off container labels.
+    compose_project: Mapped[str | None] = mapped_column(String(255), nullable=True)
+    service_name: Mapped[str | None] = mapped_column(String(255), nullable=True)
+    task_id: Mapped[str | None] = mapped_column(String(64), nullable=True)
+    node_id: Mapped[str | None] = mapped_column(String(64), nullable=True)
+    # Cumulative-since-start I/O counters (BigInteger — they exceed 2^31 quickly).
+    net_rx_bytes: Mapped[int | None] = mapped_column(BigInteger, nullable=True)
+    net_tx_bytes: Mapped[int | None] = mapped_column(BigInteger, nullable=True)
+    blk_read_bytes: Mapped[int | None] = mapped_column(BigInteger, nullable=True)
+    blk_write_bytes: Mapped[int | None] = mapped_column(BigInteger, nullable=True)
+
 
 class DockerMetric(Base):
     """Time-series CPU/memory per container — one row per sample per running

plugins/docker/routes.py

@@ -1,6 +1,8 @@
 # plugins/docker/routes.py
 from __future__ import annotations
 import json
+from datetime import datetime, timezone
+
 from quart import Blueprint, current_app, render_template, request
 from sqlalchemy import select
 
@@ -13,6 +15,25 @@ from .models import DockerContainer, DockerMetric
 docker_bp = Blueprint("docker", __name__, template_folder="templates")
 
 
+def _human_uptime(started_at: datetime | None) -> str | None:
+    """Compact 'how long running' string (e.g. '3d 4h', '5h 12m', '8m')."""
+    if started_at is None:
+        return None
+    if started_at.tzinfo is None:
+        started_at = started_at.replace(tzinfo=timezone.utc)
+    secs = int((datetime.now(timezone.utc) - started_at).total_seconds())
+    if secs < 0:
+        return None
+    d, rem = divmod(secs, 86400)
+    h, rem = divmod(rem, 3600)
+    m, _ = divmod(rem, 60)
+    if d:
+        return f"{d}d {h}h"
+    if h:
+        return f"{h}h {m}m"
+    return f"{m}m"
+
+
 def _sparkline(values: list[float], width: int = 80, height: int = 20) -> str:
     if len(values) < 2:
         return f'<svg width="{width}" height="{height}"></svg>'
@@ -119,6 +140,7 @@ async def rows():
             g["containers"].append({
                 "container": c,
                 "ports": json.loads(c.ports_json) if c.ports_json else [],
+                "uptime": _human_uptime(c.started_at) if c.status == "running" else None,
                 "sparkline_cpu": _sparkline(cpu_hist),
                 "sparkline_mem": _sparkline(mem_hist),
             })

plugins/docker/templates/docker/rows.html

@@ -54,8 +54,24 @@
           <div style="display:flex;align-items:center;gap:0.5rem;">
             <span class="dot {% if c.status == 'running' %}dot-up{% elif c.status == 'paused' %}dot-warn{% else %}dot-down{% endif %}"></span>
             <div>
-              <div style="font-weight:500;font-size:0.9rem;">{{ c.name }}</div>
-              <div style="font-size:0.73rem;color:var(--text-muted);">{{ c.status }}</div>
+              <div style="font-weight:500;font-size:0.9rem;">
+                {{ c.name }}
+                {% if c.health == 'healthy' %}<span title="healthy" style="color:var(--green);font-size:0.7rem;">●</span>
+                {% elif c.health == 'unhealthy' %}<span title="unhealthy" style="color:var(--red);font-size:0.7rem;">●</span>
+                {% elif c.health == 'starting' %}<span title="health: starting" style="color:var(--orange);font-size:0.7rem;">◐</span>{% endif %}
+              </div>
+              <div style="font-size:0.73rem;color:var(--text-muted);">
+                {{ c.status }}{% if item.uptime %} · up {{ item.uptime }}{% endif %}
+                {% if c.status != 'running' and c.exit_code is not none and c.exit_code != 0 %}
+                  · <span style="color:var(--red);">exit {{ c.exit_code }}{% if c.oom_killed %} (OOM){% endif %}</span>
+                {% endif %}
+                {% if c.restart_count %} · <span title="restart count" style="color:var(--orange);">⟳{{ c.restart_count }}</span>{% endif %}
+              </div>
+              {% if c.service_name or c.compose_project %}
+              <div style="font-size:0.68rem;color:var(--text-dim);margin-top:0.1rem;">
+                {{ c.service_name or c.compose_project }}
+              </div>
+              {% endif %}
             </div>
           </div>
         </td>