feat(docker): Docker's own status reaches Steward — an empty answer clears a host, a lost one greys it, a demoted manager's swarm rows go (#5152)

The agent (1.9.0) said nothing when Docker could not be asked and nothing
when Docker had no containers, so Steward could tell neither apart from the
other, and kept a host's last containers on screen as running either way.

- Agent: collect_docker returns (status, detail, containers). "ok" sends
  docker: [] when there are none; "unavailable" (no socket, nothing
  listening) and "error" (permission, timeout, bad answer) say why.
  collect_swarm says whether /info answered, and a node that is not a
  manager sends swarm_manager: false. A failed /info says nothing.
- Ingest: an empty docker list is a snapshot; the newest sample's status
  goes to the Docker plugin with the rest.
- docker_host_status (docker_011): per host, status, detail, since and
  last report. A host that never had Docker gets no row.
- An "ok" empty snapshot reaps the host's containers; an error keeps them.
  swarm_manager: false drops the host's swarm services and nodes.
- Docker page: a "Docker not answering" card names each such host, why,
  and since when; its containers read as stale. Duplicate header comment
  in rows.html removed.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

plugins/docker/ingest.py

@@ -251,8 +251,37 @@ async def _persist_disk(session, host, disk: dict) -> None:
     await session.execute(stale)
 
 
+async def _persist_status(session, host, status: dict) -> None:
+    """Record whether Docker answered on this host, and why not.
+
+    A host reporting "unavailable" with no row yet never had Docker: nothing to
+    record. Once a host has had Docker, losing it is recorded, so its last
+    containers read as stale rather than as running."""
+    from .models import DockerHostStatus
+
+    row = await session.get(DockerHostStatus, host.id)
+    if row is None:
+        if status["status"] == "unavailable":
+            return
+        row = DockerHostStatus(host_id=host.id)
+        session.add(row)
+    if row.status != status["status"]:
+        row.since = status["at"]
+    row.status = status["status"]
+    row.detail = (status.get("detail") or "")[:255] or None
+    row.reported_at = status["at"]
+
+
+async def _expire_swarm(session, host) -> None:
+    """Drop the swarm rows a host reported while it was a manager."""
+    from .models import DockerSwarmNode, DockerSwarmService
+
+    await session.execute(delete(DockerSwarmService).where(DockerSwarmService.host_id == host.id))
+    await session.execute(delete(DockerSwarmNode).where(DockerSwarmNode.host_id == host.id))
+
+
 async def persist_host_docker(session, host, snapshots, swarm=None, disk=None,
-                              logs=None) -> None:
+                              logs=None, status=None) -> None:
     """Upsert containers + time-series + lifecycle events + swarm for one host.
 
     `snapshots` is a list of (recorded_at: datetime, containers: list[dict]) —
@@ -264,11 +293,18 @@ async def persist_host_docker(session, host, snapshots, swarm=None, disk=None,
     `disk` is the newest sample's /system/df summary (or None on Docker-less
     hosts) — persisted when present. `logs` is a list of (recorded_at, records)
     log batches (m79) — appended one row per line when present, independent of
-    whether this sample also carried container metrics.
+    whether this sample also carried container metrics. `status` is the newest
+    sample's {"status", "detail", "at", "swarm_manager"} (agent ≥ 1.9.0): an
+    "ok" with an empty snapshot clears the host's containers, and a node that
+    says it is no longer a manager loses the swarm rows it reported as one.
     """
     from steward.core.alerts import MetricBatch
     from .models import DockerContainer, DockerEvent, DockerMetric
 
+    if status is not None:
+        await _persist_status(session, host, status)
+        if status.get("swarm_manager") is False:
+            await _expire_swarm(session, host)
     if swarm is not None:
         await _persist_swarm(session, host, swarm)
     if disk is not None:

plugins/docker/migrations/versions/docker_011_host_status.py

@@ -0,0 +1,36 @@
+"""Docker host status
+
+Adds docker_host_status: per host, whether Docker answered at the agent's
+last report (ok / unavailable / error) and why not. Lets Steward show a host
+whose Docker went away as such, instead of its last containers as running
+(#5152). Additive create_table.
+
+Revision ID: docker_011_host_status
+Revises: docker_010_swarm_tasks
+Create Date: 2026-10-06
+"""
+from typing import Sequence, Union
+from alembic import op
+import sqlalchemy as sa
+
+revision: str = "docker_011_host_status"
+down_revision: Union[str, None] = "docker_010_swarm_tasks"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    op.create_table(
+        "docker_host_status",
+        sa.Column("host_id", sa.String(length=36), nullable=False),
+        sa.Column("status", sa.String(length=16), nullable=False),
+        sa.Column("detail", sa.String(length=255), nullable=True),
+        sa.Column("since", sa.DateTime(timezone=True), nullable=False),
+        sa.Column("reported_at", sa.DateTime(timezone=True), nullable=False),
+        sa.ForeignKeyConstraint(["host_id"], ["hosts.id"], ondelete="CASCADE"),
+        sa.PrimaryKeyConstraint("host_id"),
+    )
+
+
+def downgrade() -> None:
+    op.drop_table("docker_host_status")

plugins/docker/models.py

@@ -304,3 +304,24 @@ class DockerSwarmNode(Base):
         DateTime(timezone=True), nullable=False,
         default=lambda: datetime.now(timezone.utc),
     )
+
+
+class DockerHostStatus(Base):
+    """Whether Docker answered on a host at its agent's last report (agent ≥ 1.9.0).
+
+    "ok" when it did; "unavailable" when there is no Docker to ask (no socket,
+    nothing listening); "error" when Docker is there and the call failed.
+    `detail` says why for the last two. While a host is not "ok" its container
+    rows are what Docker last said, not what is true now, and read as stale.
+    A host that never had Docker gets no row.
+    """
+    __tablename__ = "docker_host_status"
+
+    host_id: Mapped[str] = mapped_column(
+        String(36), ForeignKey("hosts.id", ondelete="CASCADE"), primary_key=True
+    )
+    status: Mapped[str] = mapped_column(String(16), nullable=False)
+    detail: Mapped[str | None] = mapped_column(String(255), nullable=True)
+    # When the status last changed, and the agent's newest report.
+    since: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
+    reported_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)

plugins/docker/routes.py

@@ -18,7 +18,7 @@ from . import service as docker_svc
 from .dedup import dedup_by_container_id
 from .swarm_view import build_swarm_services, parse_tasks, task_summary
 from .models import (
-    DockerContainer, DockerDiskUsage, DockerEvent, DockerImage,
+    DockerContainer, DockerDiskUsage, DockerEvent, DockerHostStatus, DockerImage,
     DockerSwarmNode, DockerSwarmService,
 )
 
@@ -87,6 +87,9 @@ async def rows():
     stale_after = docker_svc.stale_after_seconds(current_app.config)
 
     async with current_app.db_sessionmaker() as db:
+        # Hosts whose Docker is not answering: their rows are what it last said.
+        docker_down = {d.host_id: d for d in (await db.execute(
+            select(DockerHostStatus).where(DockerHostStatus.status != "ok"))).scalars()}
         ranked = await docker_svc.ranked_containers(
             db, now, docker_svc.crash_loop_deaths(current_app.config))
         containers = await docker_svc.list_containers(db)
@@ -95,7 +98,7 @@ async def rows():
         hosts = await docker_svc.host_map(
             db,
             {c.host_id for c in containers}
-            | {n.host_id for n in nodes} | {s.host_id for s in services},
+            | {n.host_id for n in nodes} | {s.host_id for s in services} | set(docker_down),
         )
 
         # Swarm tasks collapse into per-service rows (each replica labelled with
@@ -123,7 +126,7 @@ async def rows():
                 "container": c,
                 "ports": json.loads(c.ports_json) if c.ports_json else [],
                 "uptime": _uptime(c.started_at) if c.status == "running" else None,
-                "stale": _is_stale(c, now, stale_after),
+                "stale": _is_stale(c, now, stale_after, docker_down),
             })
             if c.status == "running":
                 g["running"] += 1
@@ -151,12 +154,16 @@ async def rows():
     swarm_services = swarm_view["services"]
     standalone_running = sum(g["running"] for g in host_groups)
     ghost_total = sum(r["count"] for s in swarm_services for r in s["replicas"] if r["ghost"])
-    attention = [dict(r, stale=_is_stale(r["container"], now, stale_after))
+    attention = [dict(r, stale=_is_stale(r["container"], now, stale_after, docker_down))
                  for r in docker_svc.needs_attention(ranked)]
     return await render_template(
         "docker/rows.html",
         host_groups=host_groups,
         swarm_services=swarm_services,
+        docker_down=sorted(
+            ({"host_id": h, "host_name": hosts[h].name if h in hosts else h, "status": d.status,
+              "detail": d.detail, "since": d.since} for h, d in docker_down.items()),
+            key=lambda d: d["host_name"].lower()),
         attention=attention, says=docker_svc.PROBLEM_SAYS, severity=docker_svc.PROBLEM_SEVERITY,
         unhealthy=sum("unhealthy" in r["problems"] for r in attention),
         restarting=sum(bool({"restarting", "crash_looping"} & set(r["problems"])) for r in attention),
@@ -167,10 +174,12 @@ async def rows():
     )
 
 
-def _is_stale(c, now: datetime, stale_after: int) -> bool:
+def _is_stale(c, now: datetime, stale_after: int, docker_down=()) -> bool:
     """A container whose last report is older than the agent's staleness
-    window: what the page shows may no longer be true."""
-    return c.scraped_at is None or (now - c.scraped_at).total_seconds() > stale_after
+    window, or whose host's Docker is not answering: what the page shows may
+    no longer be true."""
+    return (c.host_id in docker_down or c.scraped_at is None
+            or (now - c.scraped_at).total_seconds() > stale_after)
 
 
 @docker_bp.get("/widget")

plugins/docker/templates/docker/rows.html

@@ -1,5 +1,3 @@
-{# docker/rows.html — HTMX fragment for the Docker main page, grouped by host #}
-
 {# docker/rows.html — HTMX fragment for the Docker page: what needs attention
    first (service.needs_attention, the ranking the MCP tools use), the counts,
    then swarm services and each host's containers. #}
@@ -28,6 +26,22 @@
 </section>
 {% endif %}
 
+{# ── Hosts whose Docker is not answering (agent ≥ 1.9.0) ──────────────────── #}
+{% if docker_down %}
+<section class="card problem-card" aria-labelledby="docker-unreachable">
+  <h2 id="docker-unreachable" class="section-title" style="margin-bottom:0.6rem;">Docker not answering</h2>
+  <ul class="problem-list">
+    {% for d in docker_down %}
+    <li>
+      {{ status_badge('crit', 'Error') if d.status == 'error' else status_badge('stale', 'No Docker') }}
+      <a href="/hosts/{{ d.host_id }}">{{ d.host_name }}</a>
+      <span class="problem-detail">{{ d.detail or "no reason given" }} · since {{ d.since|ago }} · its containers below are what Docker last reported</span>
+    </li>
+    {% endfor %}
+  </ul>
+</section>
+{% endif %}
+
 {# ── Counts ───────────────────────────────────────────────────────────────── #}
 <div class="stat-strip">
   {{ stat_tile("Needs attention", attention|length, state="crit" if attention else "inactive") }}

plugins/host_agent/agent.py

@@ -20,7 +20,7 @@ from collections import deque
 from concurrent.futures import ThreadPoolExecutor
 from datetime import datetime, timezone
 
-AGENT_VERSION = "1.8.0"
+AGENT_VERSION = "1.9.0"
 
 # Default path to the local Docker Engine socket. Overridable via the
 # `docker_socket` config key; collection is silently skipped if it's absent or
@@ -645,19 +645,37 @@ def _collect_one_container(socket_path: str, c: dict) -> dict:
     return record
 
 
-def collect_docker(socket_path: str) -> list:
-    """Per-container state from the local Docker socket, or [] if unavailable.
+def _docker_failure(socket_path: str, exc: Exception) -> tuple:
+    """(status, detail) for a failed Docker call: "unavailable" when there is no
+    Docker here to ask, "error" when there is one and the call went wrong."""
+    if isinstance(exc, FileNotFoundError):
+        return "unavailable", "no Docker socket at " + socket_path
+    if isinstance(exc, ConnectionRefusedError):
+        return "unavailable", "Docker is not accepting connections on " + socket_path
+    if isinstance(exc, PermissionError):
+        return "error", "permission denied on " + socket_path
+    if isinstance(exc, socket.timeout):
+        return "error", "Docker did not answer within %.0f s" % DOCKER_API_TIMEOUT
+    return "error", (str(exc) or type(exc).__name__)[:200]
 
-    Absent socket / permission denied / non-Docker endpoint all yield [] — the
-    agent silently reports no containers rather than erroring, so a Docker-less
-    host needs no configuration.
+
+def collect_docker(socket_path: str) -> tuple:
+    """(status, detail, containers) from the local Docker socket.
+
+    status is "ok" when Docker answered (containers may then be [] on a host
+    with none, which tells Steward to clear its list), "unavailable" when there
+    is no Docker here (no socket, or nothing listening), and "error" when Docker
+    is here but the listing failed. detail says why, for the last two. A
+    Docker-less host still needs no configuration: it reports "unavailable".
     """
     try:
         containers = _docker_request(socket_path, "/containers/json?all=true")
-    except (OSError, ValueError):
-        return []
-    if not isinstance(containers, list) or not containers:
-        return []
+    except (OSError, ValueError) as exc:
+        return _docker_failure(socket_path, exc) + ([],)
+    if not isinstance(containers, list):
+        return "error", "unexpected answer from /containers/json", []
+    if not containers:
+        return "ok", None, []
 
     # Each container needs an inspect (+ a stats read if running), and the stats
     # call blocks ~1s while the daemon computes the cpu delta. Done serially that
@@ -665,7 +683,7 @@ def collect_docker(socket_path: str) -> list:
     # over a small bounded thread pool (I/O-bound → threads are enough).
     workers = min(8, len(containers))
     with ThreadPoolExecutor(max_workers=workers) as ex:
-        return list(ex.map(lambda c: _collect_one_container(socket_path, c), containers))
+        return "ok", None, list(ex.map(lambda c: _collect_one_container(socket_path, c), containers))
 
 
 # ─── container logs (m79) ─────────────────────────────────────────────────────
@@ -978,26 +996,28 @@ def _swarm_nodes(nodes: list) -> list:
     return out
 
 
-def collect_swarm(socket_path: str):
-    """Swarm topology from a manager node, or None on workers / non-swarm hosts.
+def collect_swarm(socket_path: str) -> tuple:
+    """(is_manager, topology) from the local Docker socket.
 
     Self-detects manager via `/info` (one cheap call); only then queries the
-    manager-only endpoints. Any transport failure degrades to None, same silent
-    contract as collect_docker — a non-swarm host adds nothing to the payload.
+    manager-only endpoints. is_manager is None when /info itself failed, so a
+    transient error is never mistaken for a node that stopped being a manager;
+    False tells Steward to drop the swarm rows this host reported as one.
+    topology is None unless this is a manager and every call answered.
     """
     try:
         info = _docker_request(socket_path, "/info")
     except (OSError, ValueError):
-        return None
+        return None, None
     if not _swarm_is_manager(info):
-        return None
+        return False, None
     try:
         services = _docker_request(socket_path, "/services")
         tasks = _docker_request(socket_path, "/tasks")
         nodes = _docker_request(socket_path, "/nodes")
     except (OSError, ValueError):
-        return None
-    return {
+        return True, None
+    return True, {
         "services": _swarm_services(
             services if isinstance(services, list) else [],
             tasks if isinstance(tasks, list) else []),
@@ -1161,26 +1181,32 @@ def build_sample(mounts: list[str], state: dict,
     ]
     state["net"], state["disk"], state["mono"] = net_raw, disk_raw, now_mono
 
+    # Docker's status rides every sample, so Steward can tell "no containers"
+    # (status ok, docker: []) from "couldn't ask" (unavailable / error).
     try:
-        docker = collect_docker(docker_socket)
-    except Exception:
-        docker = []
-    if docker:
+        docker_status, docker_detail, docker = collect_docker(docker_socket)
+    except Exception as exc:
+        docker_status, docker_detail, docker = "error", (str(exc) or type(exc).__name__)[:200], []
+    sample["docker_status"] = docker_status
+    if docker_detail:
+        sample["docker_detail"] = docker_detail
+    if docker_status == "ok":
         sample["docker"] = docker
 
-    # Swarm topology is only emitted by managers (collect_swarm returns None
-    # everywhere else), so the key is absent on workers / non-swarm hosts.
+    # Swarm topology is only emitted by managers. A node whose /info says it
+    # is not a manager says so, so Steward can expire what it once reported.
     try:
-        swarm = collect_swarm(docker_socket)
+        is_manager, swarm = collect_swarm(docker_socket)
     except Exception:
-        swarm = None
+        is_manager, swarm = None, None
     if swarm is not None:
         sample["swarm"] = swarm
+    elif is_manager is False:
+        sample["swarm_manager"] = False
 
-    # Image/disk usage — host-level docker fact (one /system/df call), omitted on
-    # Docker-less hosts. Only meaningful when there's docker here at all, so skip
-    # the call entirely when collect_docker found nothing.
-    if docker:
+    # Image/disk usage — host-level docker fact (one /system/df call), only
+    # when Docker answered.
+    if docker_status == "ok":
         try:
             disk = collect_disk_usage(docker_socket)
         except Exception:

plugins/host_agent/routes.py

@@ -214,15 +214,24 @@ async def ingest():
             latest_swarm_ts: datetime | None = None
             latest_disk: dict | None = None
             latest_disk_ts: datetime | None = None
+            # Whether Docker answered, from the newest sample that says (agent ≥ 1.9.0).
+            docker_status: dict | None = None
             for sample in samples:
                 try:
                     recorded_at = _parse_ts(sample["ts"])
                 except (KeyError, ValueError):
                     continue
                 metric_rows.extend(_expand_sample_to_metrics(sample, host.name, recorded_at))
+                # An empty list is a real answer: Docker is up with no containers.
                 docker = sample.get("docker")
-                if isinstance(docker, list) and docker:
+                if isinstance(docker, list):
                     docker_snapshots.append((recorded_at, docker))
+                status = sample.get("docker_status")
+                if status in ("ok", "unavailable", "error") and (
+                        docker_status is None or recorded_at > docker_status["at"]):
+                    docker_status = {
+                        "status": status, "detail": sample.get("docker_detail"),
+                        "at": recorded_at, "swarm_manager": sample.get("swarm_manager")}
                 # Container logs are time-series (append every line, not newest-only);
                 # each record carries its own Docker ts, recorded_at is the fallback.
                 docker_logs = sample.get("docker_logs")
@@ -257,7 +266,7 @@ async def ingest():
             # (opportunistic synergy via the capability registry — no hard import,
             # no-op when docker is disabled). A failure here must never sink the
             # whole ingest, so the metrics above still land.
-            if (docker_snapshots or latest_swarm is not None
+            if (docker_snapshots or latest_swarm is not None or docker_status is not None
                     or latest_disk is not None or docker_log_batches):
                 from steward.core.capabilities import has_capability, invoke_capability
                 if has_capability("docker.persist_host_samples"):
@@ -268,7 +277,7 @@ async def ingest():
                             await invoke_capability(
                                 "docker.persist_host_samples", UserRole.admin,
                                 session, host, docker_snapshots, latest_swarm,
-                                latest_disk, docker_log_batches,
+                                latest_disk, docker_log_batches, docker_status,
                             )
                     except Exception:
                         current_app.logger.exception(