feat(docker): per-host collection via the host agent; drop central scrape

Docker collection moves off the central single-socket scrape onto the host
agent, giving Docker a real per-host dimension. The Steward host now reports
its own containers like any other host, and same-named containers on different
hosts no longer collide.

- agent: stdlib UDS Docker client (AF_UNIX HTTP/1.1, Connection: close,
  chunked-aware), collect_docker() ports the cpu%/mem math; sample["docker"]
  added best-effort (silent-skip on absent/unreadable socket). AGENT_VERSION
  1.2.0 → 1.3.0; optional docker_socket config key.
- ingest: host_agent ingest hands per-host container snapshots to the docker
  plugin via a new "docker.persist_host_samples" capability (no hard import,
  no-op when docker disabled), inside a SAVEPOINT so a docker failure never
  sinks the host metrics. Resource names are host-scoped ("<host>/<name>").
- schema: docker_containers re-keyed (host_id, name); docker_metrics gains
  host_id; docker_002 migration DROP+recreates (dev-only, rule 122).
- ui: Docker page + widgets grouped by host with host links; new per-host
  Docker panel embedded on the Hosts hub (gated on docker enabled via a new
  enabled_plugins template context). Replaces the SQLite-only strftime
  bucketing with DB-agnostic Python bucketing.
- provisioning: install/provision playbooks add steward-agent to the docker
  group (best-effort) so the agent can read the socket.
- removed central scrape: docker scheduler.py + scraper.py deleted; plugin.yaml
  socket_path/scrape_interval_seconds/include_stopped dropped (plugin 2.0.0).
- tests: agent docker collector units (math, chunked decode, silent-skip,
  sample shape, config) + integration (host-scoped schema + persistence).

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_016Jg27rgypiW2efULXJDtMC

plugins/docker/routes.py

@@ -2,11 +2,12 @@
 from __future__ import annotations
 import json
 from quart import Blueprint, current_app, render_template, request
-from sqlalchemy import Integer, cast, func, select
+from sqlalchemy import select
 
 from steward.auth.middleware import require_role
 from steward.models.users import UserRole
-from steward.core.time_range import parse_range, DEFAULT_RANGE, bucket_seconds
+from steward.models.hosts import Host
+from steward.core.time_range import parse_range, DEFAULT_RANGE
 from .models import DockerContainer, DockerMetric
 
 docker_bp = Blueprint("docker", __name__, template_folder="templates")
@@ -33,6 +34,38 @@ def _sparkline(values: list[float], width: int = 80, height: int = 20) -> str:
     )
 
 
+def _bucket(values: list[float], target: int = 40) -> list[float]:
+    """Bucket-average a series down to ~target points (DB-agnostic, in Python).
+
+    Replaces the old SQL strftime() bucketing, which was SQLite-only and broke
+    on Postgres. Agent cadence is dense, so a multi-hour window is hundreds of
+    rows — averaging into a readable point count keeps the sparkline's shape.
+    """
+    n = len(values)
+    if n <= target:
+        return values
+    size = (n + target - 1) // target
+    out: list[float] = []
+    for i in range(0, n, size):
+        chunk = values[i:i + size]
+        out.append(sum(chunk) / len(chunk))
+    return out
+
+
+async def _container_history(db, host_id: str, name: str, since) -> tuple[list, list]:
+    """Return (cpu_series, mem_series) sparkline-ready for one host's container."""
+    rows = (await db.execute(
+        select(DockerMetric.cpu_pct, DockerMetric.mem_pct)
+        .where(DockerMetric.host_id == host_id)
+        .where(DockerMetric.container_name == name)
+        .where(DockerMetric.scraped_at >= since)
+        .order_by(DockerMetric.scraped_at)
+    )).all()
+    cpu = _bucket([r.cpu_pct or 0.0 for r in rows])
+    mem = _bucket([r.mem_pct or 0.0 for r in rows])
+    return cpu, mem
+
+
 @docker_bp.get("/")
 @require_role(UserRole.viewer)
 async def index():
@@ -45,64 +78,64 @@ async def index():
     )
 
 
+async def _host_map(db, host_ids: set[str]) -> dict[str, Host]:
+    if not host_ids:
+        return {}
+    return {
+        h.id: h for h in (await db.execute(
+            select(Host).where(Host.id.in_(host_ids)))).scalars().all()
+    }
+
+
 @docker_bp.get("/rows")
 @require_role(UserRole.viewer)
 async def rows():
-    """HTMX fragment: container list with status and resource sparklines."""
+    """HTMX fragment: containers grouped by host, with resource sparklines."""
     since, range_key = parse_range(request.args.get("range"))
-    b_secs = bucket_seconds(since)
-
-    bucket_col = (
-        cast(func.strftime('%s', DockerMetric.scraped_at), Integer) / b_secs
-    ).label("bucket")
 
     async with current_app.db_sessionmaker() as db:
-        # All known containers ordered by running first, then name
-        result = await db.execute(
-            select(DockerContainer)
-            .order_by(
-                # running first
+        containers = list((await db.execute(
+            select(DockerContainer).order_by(
                 (DockerContainer.status == "running").desc(),
                 DockerContainer.name,
             )
-        )
-        containers = list(result.scalars())
+        )).scalars())
+        hosts = await _host_map(db, {c.host_id for c in containers})
 
-        # Build per-container sparkline histories
-        histories: dict[str, list] = {}
+        # Group by host so each container is clearly attributed to the box it
+        # runs on (names are only unique within a host now).
+        groups: dict[str, dict] = {}
         for c in containers:
-            result = await db.execute(
-                select(
-                    func.avg(DockerMetric.cpu_pct).label("cpu_pct"),
-                    func.avg(DockerMetric.mem_pct).label("mem_pct"),
-                    bucket_col,
-                )
-                .where(DockerMetric.container_name == c.name)
-                .where(DockerMetric.scraped_at >= since)
-                .group_by(bucket_col)
-                .order_by(bucket_col)
-            )
-            histories[c.name] = result.all()
-
-    running = sum(1 for c in containers if c.status == "running")
-    stopped = len(containers) - running
-
-    container_data = []
-    for c in containers:
-        hist = histories.get(c.name, [])
-        ports = json.loads(c.ports_json) if c.ports_json else []
-        container_data.append({
-            "container": c,
-            "ports": ports,
-            "sparkline_cpu": _sparkline([r.cpu_pct or 0 for r in hist]),
-            "sparkline_mem": _sparkline([r.mem_pct or 0 for r in hist]),
-        })
+            g = groups.get(c.host_id)
+            if g is None:
+                host = hosts.get(c.host_id)
+                g = groups[c.host_id] = {
+                    "host": host,
+                    "host_id": c.host_id,
+                    "host_name": host.name if host else c.host_id,
+                    "containers": [], "running": 0, "stopped": 0,
+                }
+            cpu_hist, mem_hist = await _container_history(db, c.host_id, c.name, since)
+            g["containers"].append({
+                "container": c,
+                "ports": json.loads(c.ports_json) if c.ports_json else [],
+                "sparkline_cpu": _sparkline(cpu_hist),
+                "sparkline_mem": _sparkline(mem_hist),
+            })
+            if c.status == "running":
+                g["running"] += 1
+            else:
+                g["stopped"] += 1
 
+    host_groups = sorted(groups.values(), key=lambda g: g["host_name"].lower())
+    running = sum(g["running"] for g in host_groups)
+    total = len(containers)
     return await render_template(
         "docker/rows.html",
-        container_data=container_data,
+        host_groups=host_groups,
         running=running,
-        stopped=stopped,
+        stopped=total - running,
+        total=total,
         range_key=range_key,
     )
 
@@ -110,27 +143,38 @@ async def rows():
 @docker_bp.get("/widget")
 @require_role(UserRole.viewer)
 async def widget():
-    """HTMX dashboard widget: container status overview."""
+    """HTMX dashboard widget: container status overview, grouped by host."""
     show_stopped = request.args.get("show_stopped", "no") == "yes"
     widget_id = request.args.get("wid", "0")
 
     async with current_app.db_sessionmaker() as db:
-        result = await db.execute(
-            select(DockerContainer)
-            .order_by(
+        all_containers = list((await db.execute(
+            select(DockerContainer).order_by(
                 (DockerContainer.status == "running").desc(),
                 DockerContainer.name,
             )
-        )
-        all_containers = list(result.scalars())
+        )).scalars())
+        hosts = await _host_map(db, {c.host_id for c in all_containers})
 
     running = [c for c in all_containers if c.status == "running"]
     stopped = [c for c in all_containers if c.status != "running"]
     display = all_containers if show_stopped else running
 
+    # Group for display so multi-host fleets read clearly; single-host stays flat.
+    groups: dict[str, dict] = {}
+    for c in display:
+        g = groups.setdefault(c.host_id, {
+            "host_id": c.host_id,
+            "host_name": hosts[c.host_id].name if c.host_id in hosts else c.host_id,
+            "containers": [],
+        })
+        g["containers"].append(c)
+    host_groups = sorted(groups.values(), key=lambda g: g["host_name"].lower())
+
     return await render_template(
         "docker/widget.html",
-        containers=display,
+        host_groups=host_groups,
+        multi_host=len(host_groups) > 1,
         running_count=len(running),
         stopped_count=len(stopped),
         show_stopped=show_stopped,
@@ -141,20 +185,54 @@ async def widget():
 @docker_bp.get("/widget/resources")
 @require_role(UserRole.viewer)
 async def widget_resources():
-    """HTMX dashboard widget: CPU + memory usage for running containers."""
+    """HTMX dashboard widget: CPU + memory usage for the busiest containers."""
     limit = max(1, min(20, int(request.args.get("limit", 10) or 10)))
     widget_id = request.args.get("wid", "0")
 
     async with current_app.db_sessionmaker() as db:
-        result = await db.execute(
+        containers = list((await db.execute(
             select(DockerContainer)
             .where(DockerContainer.status == "running")
             .order_by(DockerContainer.cpu_pct.desc().nullslast())
-        )
-        containers = list(result.scalars())[:limit]
+        )).scalars())[:limit]
+        hosts = await _host_map(db, {c.host_id for c in containers})
 
+    rows_data = [
+        {"c": c, "host_name": hosts[c.host_id].name if c.host_id in hosts else c.host_id}
+        for c in containers
+    ]
     return await render_template(
         "docker/widget_resources.html",
-        containers=containers,
+        rows=rows_data,
+        multi_host=len({c.host_id for c in containers}) > 1,
         widget_id=widget_id,
     )
+
+
+@docker_bp.get("/host/<host_id>")
+@require_role(UserRole.viewer)
+async def host_panel(host_id: str):
+    """Per-host Docker fragment embedded on the core Hosts hub via HTMX.
+
+    Renders nothing when the host reports no containers, so hosts without Docker
+    don't carry an empty card.
+    """
+    async with current_app.db_sessionmaker() as db:
+        containers = list((await db.execute(
+            select(DockerContainer)
+            .where(DockerContainer.host_id == host_id)
+            .order_by(
+                (DockerContainer.status == "running").desc(),
+                DockerContainer.name,
+            )
+        )).scalars())
+
+    if not containers:
+        return ""
+    running = sum(1 for c in containers if c.status == "running")
+    return await render_template(
+        "docker/host_panel.html",
+        containers=containers,
+        running=running,
+        stopped=len(containers) - running,
+    )