feat(docker): per-host collection via the host agent; drop central scrape

Docker collection moves off the central single-socket scrape onto the host
agent, giving Docker a real per-host dimension. The Steward host now reports
its own containers like any other host, and same-named containers on different
hosts no longer collide.

- agent: stdlib UDS Docker client (AF_UNIX HTTP/1.1, Connection: close,
  chunked-aware), collect_docker() ports the cpu%/mem math; sample["docker"]
  added best-effort (silent-skip on absent/unreadable socket). AGENT_VERSION
  1.2.0 → 1.3.0; optional docker_socket config key.
- ingest: host_agent ingest hands per-host container snapshots to the docker
  plugin via a new "docker.persist_host_samples" capability (no hard import,
  no-op when docker disabled), inside a SAVEPOINT so a docker failure never
  sinks the host metrics. Resource names are host-scoped ("<host>/<name>").
- schema: docker_containers re-keyed (host_id, name); docker_metrics gains
  host_id; docker_002 migration DROP+recreates (dev-only, rule 122).
- ui: Docker page + widgets grouped by host with host links; new per-host
  Docker panel embedded on the Hosts hub (gated on docker enabled via a new
  enabled_plugins template context). Replaces the SQLite-only strftime
  bucketing with DB-agnostic Python bucketing.
- provisioning: install/provision playbooks add steward-agent to the docker
  group (best-effort) so the agent can read the socket.
- removed central scrape: docker scheduler.py + scraper.py deleted; plugin.yaml
  socket_path/scrape_interval_seconds/include_stopped dropped (plugin 2.0.0).
- tests: agent docker collector units (math, chunked decode, silent-skip,
  sample shape, config) + integration (host-scoped schema + persistence).

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_016Jg27rgypiW2efULXJDtMC

tests/integration/test_docker.py

@@ -0,0 +1,109 @@
+"""Integration: per-host Docker schema + host-scoped persistence.
+
+Validates the docker_002 migration (host_id columns + composite key) applied and
+that persist_host_docker scopes containers/metrics by host so same-named
+containers on different hosts no longer collide. Requires STEWARD_DATABASE_URL.
+"""
+from __future__ import annotations
+
+import asyncio
+import os
+import uuid
+from datetime import datetime, timezone
+
+import pytest
+
+pytestmark = pytest.mark.integration
+
+_NEEDS_DB = pytest.mark.skipif(
+    not os.environ.get("STEWARD_DATABASE_URL"),
+    reason="integration test needs a live Postgres (STEWARD_DATABASE_URL)",
+)
+
+
+@pytest.fixture
+def app():
+    if not os.environ.get("STEWARD_DATABASE_URL"):
+        pytest.skip("needs Postgres")
+    from steward.app import create_app
+    return create_app(testing=False)
+
+
+@_NEEDS_DB
+def test_docker_tables_are_host_scoped(app):
+    from sqlalchemy import text
+
+    async def _go():
+        async with app.db_sessionmaker() as s:
+            # Both tables carry a host_id column after docker_002.
+            for tbl in ("docker_containers", "docker_metrics"):
+                cols = {r[0] for r in (await s.execute(text(
+                    "SELECT column_name FROM information_schema.columns "
+                    "WHERE table_name = :t"), {"t": tbl})).all()}
+                assert "host_id" in cols, f"{tbl} missing host_id"
+            # docker_containers is keyed by (host_id, name), not name alone.
+            pk_cols = {r[0] for r in (await s.execute(text(
+                "SELECT a.attname FROM pg_index i "
+                "JOIN pg_attribute a ON a.attrelid = i.indrelid "
+                " AND a.attnum = ANY(i.indkey) "
+                "WHERE i.indrelid = 'docker_containers'::regclass AND i.indisprimary"
+            ))).all()}
+            assert pk_cols == {"host_id", "name"}
+
+    asyncio.run(_go())
+
+
+def _persist_fn(app):
+    """Resolve persist_host_docker via the registered capability if the docker
+    plugin is loaded, else import it directly (the import is safe only when the
+    plugin is NOT loaded — otherwise its models are already mapped)."""
+    from steward.core.capabilities import has_capability, get_capability
+    if has_capability("docker.persist_host_samples"):
+        return get_capability("docker.persist_host_samples").fn
+    from plugins.docker.ingest import persist_host_docker
+    return persist_host_docker
+
+
+@_NEEDS_DB
+def test_persist_scopes_containers_by_host(app):
+    from sqlalchemy import text
+    from steward.models.hosts import Host
+
+    persist = _persist_fn(app)
+    now = datetime.now(timezone.utc)
+    snapshot = [(now, [{
+        "name": "web", "container_id": "abc123", "image": "nginx",
+        "status": "running", "cpu_pct": 12.5, "mem_pct": 30.0,
+        "mem_usage_bytes": 100, "mem_limit_bytes": 200, "ports": [], "started_at": None,
+    }])]
+
+    async def _go():
+        async with app.db_sessionmaker() as s:
+            async with s.begin():
+                await s.execute(text("DELETE FROM docker_metrics"))
+                await s.execute(text("DELETE FROM docker_containers"))
+                await s.execute(text(
+                    "DELETE FROM plugin_metrics WHERE source_module = 'docker'"))
+                h1 = Host(id=str(uuid.uuid4()), name="alpha", address="10.0.0.1")
+                h2 = Host(id=str(uuid.uuid4()), name="beta", address="10.0.0.2")
+                s.add_all([h1, h2])
+                await s.flush()
+                # Same container name on two hosts — must not collide now.
+                await persist(s, h1, snapshot)
+                await persist(s, h2, snapshot)
+
+            web_rows = (await s.execute(text(
+                "SELECT COUNT(*) FROM docker_containers WHERE name = 'web'"))).scalar()
+            metric_hosts = (await s.execute(text(
+                "SELECT COUNT(DISTINCT host_id) FROM docker_metrics "
+                "WHERE container_name = 'web'"))).scalar()
+            # Alert pipeline received host-scoped resource names.
+            resources = {r[0] for r in (await s.execute(text(
+                "SELECT DISTINCT resource_name FROM plugin_metrics "
+                "WHERE source_module = 'docker'"))).all()}
+            return web_rows, metric_hosts, resources
+
+    web_rows, metric_hosts, resources = asyncio.run(_go())
+    assert web_rows == 2          # one 'web' per host, keyed by (host_id, name)
+    assert metric_hosts == 2      # time-series rows scoped per host
+    assert resources == {"alpha/web", "beta/web"}

tests/plugins/host_agent/test_agent_docker.py

@@ -0,0 +1,157 @@
+# tests/plugins/host_agent/test_agent_docker.py
+"""Unit tests for the agent's stdlib Docker collector.
+
+No socket / no network — the UDS request layer is monkeypatched so we exercise
+the cpu/mem math, chunked-body decoding, sample assembly, and the silent-skip
+contract in isolation.
+"""
+from plugins.host_agent import agent as a
+
+
+# ── chunked transfer decoding ─────────────────────────────────────────────────
+
+def test_dechunk_reassembles_body():
+    chunked = b"4\r\nWiki\r\n5\r\npedia\r\n0\r\n\r\n"
+    assert a._dechunk(chunked) == b"Wikipedia"
+
+
+def test_dechunk_ignores_chunk_extensions():
+    assert a._dechunk(b"3;name=v\r\nabc\r\n0\r\n\r\n") == b"abc"
+
+
+# ── cpu / mem math (ported from the old central scraper) ──────────────────────
+
+def test_docker_cpu_pct():
+    stats = {
+        "cpu_stats": {
+            "cpu_usage": {"total_usage": 200_000_000},
+            "system_cpu_usage": 2_000_000_000,
+            "online_cpus": 4,
+        },
+        "precpu_stats": {
+            "cpu_usage": {"total_usage": 100_000_000},
+            "system_cpu_usage": 1_000_000_000,
+        },
+    }
+    # cpu_delta=1e8, sys_delta=1e9 → 0.1 * 4 cpus * 100 = 40.0%
+    assert a._docker_cpu_pct(stats) == 40.0
+
+
+def test_docker_cpu_pct_guards_bad_stats():
+    assert a._docker_cpu_pct({}) == 0.0
+    # No forward progress in system time → 0, never a divide error.
+    assert a._docker_cpu_pct({
+        "cpu_stats": {"cpu_usage": {"total_usage": 5}, "system_cpu_usage": 10},
+        "precpu_stats": {"cpu_usage": {"total_usage": 5}, "system_cpu_usage": 10},
+    }) == 0.0
+
+
+def test_docker_mem_subtracts_cache():
+    stats = {"memory_stats": {
+        "usage": 200 * 1024 * 1024,
+        "limit": 1000 * 1024 * 1024,
+        "stats": {"cache": 50 * 1024 * 1024},
+    }}
+    usage, limit, pct = a._docker_mem(stats)
+    assert usage == 150 * 1024 * 1024          # working set = usage − cache
+    assert limit == 1000 * 1024 * 1024
+    assert pct == 15.0
+
+
+def test_docker_ports_keeps_only_published():
+    ports = [
+        {"PublicPort": 8080, "PrivatePort": 80, "Type": "tcp"},
+        {"PrivatePort": 5432, "Type": "tcp"},  # unpublished → dropped
+    ]
+    assert a._docker_ports(ports) == [
+        {"host_port": 8080, "container_port": 80, "protocol": "tcp"},
+    ]
+
+
+# ── collect_docker assembly + silent skip ─────────────────────────────────────
+
+def test_collect_docker_silent_skip_when_no_socket():
+    # Absent socket path must degrade to [] (zero-config on Docker-less hosts).
+    assert a.collect_docker("/nonexistent/does-not-exist.sock") == []
+
+
+def test_collect_docker_assembles_container(monkeypatch):
+    container = {
+        "Id": "abc123def4567890",
+        "Names": ["/web"],
+        "Image": "nginx:latest",
+        "State": "running",
+        "Created": 1_700_000_000,
+        "Ports": [{"PublicPort": 8080, "PrivatePort": 80, "Type": "tcp"}],
+    }
+    stats = {
+        "cpu_stats": {"cpu_usage": {"total_usage": 200_000_000},
+                      "system_cpu_usage": 2_000_000_000, "online_cpus": 4},
+        "precpu_stats": {"cpu_usage": {"total_usage": 100_000_000},
+                         "system_cpu_usage": 1_000_000_000},
+        "memory_stats": {"usage": 200 * 1024 * 1024, "limit": 1000 * 1024 * 1024,
+                         "stats": {"cache": 50 * 1024 * 1024}},
+    }
+
+    def fake_request(socket_path, path, timeout=a.DOCKER_API_TIMEOUT):
+        return [container] if path.startswith("/containers/json") else stats
+
+    monkeypatch.setattr(a, "_docker_request", fake_request)
+    out = a.collect_docker("/var/run/docker.sock")
+    assert len(out) == 1
+    c = out[0]
+    assert c["name"] == "web"
+    assert c["container_id"] == "abc123def456"   # 12-char short id
+    assert c["image"] == "nginx:latest"
+    assert c["status"] == "running"
+    assert c["cpu_pct"] == 40.0
+    assert c["mem_pct"] == 15.0
+    assert c["ports"] == [{"host_port": 8080, "container_port": 80, "protocol": "tcp"}]
+    assert c["started_at"].startswith("2023-11-14T")  # epoch → ISO-8601
+
+
+def test_collect_docker_skips_stats_for_stopped(monkeypatch):
+    container = {"Id": "x" * 16, "Names": ["/db"], "Image": "pg", "State": "exited",
+                 "Created": None, "Ports": []}
+
+    def fake_request(socket_path, path, timeout=a.DOCKER_API_TIMEOUT):
+        if path.startswith("/containers/json"):
+            return [container]
+        raise AssertionError("stats must not be fetched for a stopped container")
+
+    monkeypatch.setattr(a, "_docker_request", fake_request)
+    out = a.collect_docker("/var/run/docker.sock")
+    assert out[0]["status"] == "exited"
+    assert out[0]["cpu_pct"] is None and out[0]["started_at"] is None
+
+
+# ── build_sample wiring ───────────────────────────────────────────────────────
+
+def test_build_sample_includes_docker_when_present(monkeypatch):
+    monkeypatch.setattr(a, "collect_docker", lambda _s: [{"name": "web", "status": "running"}])
+    sample = a.build_sample(["/"], {}, "/var/run/docker.sock")
+    assert sample["docker"] == [{"name": "web", "status": "running"}]
+
+
+def test_build_sample_omits_docker_when_empty(monkeypatch):
+    monkeypatch.setattr(a, "collect_docker", lambda _s: [])
+    sample = a.build_sample(["/"], {}, "/var/run/docker.sock")
+    assert "docker" not in sample
+
+
+# ── config ────────────────────────────────────────────────────────────────────
+
+def test_read_config_defaults_docker_socket(tmp_path):
+    cfg_file = tmp_path / "agent.conf"
+    cfg_file.write_text("url = https://steward.example\ntoken = abc\n")
+    cfg = a.read_config(str(cfg_file))
+    assert cfg["docker_socket"] == a.DEFAULT_DOCKER_SOCKET
+
+
+def test_read_config_honours_explicit_docker_socket(tmp_path):
+    cfg_file = tmp_path / "agent.conf"
+    cfg_file.write_text(
+        "url = https://steward.example\ntoken = abc\n"
+        "docker_socket = /run/user/1000/docker.sock\n")
+    cfg = a.read_config(str(cfg_file))
+    assert cfg["docker_socket"] == "/run/user/1000/docker.sock"