feat(docker): Docker's own status reaches Steward — an empty answer clears a host, a lost one greys it, a demoted manager's swarm rows go (#5152)

The agent (1.9.0) said nothing when Docker could not be asked and nothing
when Docker had no containers, so Steward could tell neither apart from the
other, and kept a host's last containers on screen as running either way.

- Agent: collect_docker returns (status, detail, containers). "ok" sends
  docker: [] when there are none; "unavailable" (no socket, nothing
  listening) and "error" (permission, timeout, bad answer) say why.
  collect_swarm says whether /info answered, and a node that is not a
  manager sends swarm_manager: false. A failed /info says nothing.
- Ingest: an empty docker list is a snapshot; the newest sample's status
  goes to the Docker plugin with the rest.
- docker_host_status (docker_011): per host, status, detail, since and
  last report. A host that never had Docker gets no row.
- An "ok" empty snapshot reaps the host's containers; an error keeps them.
  swarm_manager: false drops the host's swarm services and nodes.
- Docker page: a "Docker not answering" card names each such host, why,
  and since when; its containers read as stale. Duplicate header comment
  in rows.html removed.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

plugins/docker/ingest.py

@@ -251,8 +251,37 @@ async def _persist_disk(session, host, disk: dict) -> None:
     await session.execute(stale)
 
 
+async def _persist_status(session, host, status: dict) -> None:
+    """Record whether Docker answered on this host, and why not.
+
+    A host reporting "unavailable" with no row yet never had Docker: nothing to
+    record. Once a host has had Docker, losing it is recorded, so its last
+    containers read as stale rather than as running."""
+    from .models import DockerHostStatus
+
+    row = await session.get(DockerHostStatus, host.id)
+    if row is None:
+        if status["status"] == "unavailable":
+            return
+        row = DockerHostStatus(host_id=host.id)
+        session.add(row)
+    if row.status != status["status"]:
+        row.since = status["at"]
+    row.status = status["status"]
+    row.detail = (status.get("detail") or "")[:255] or None
+    row.reported_at = status["at"]
+
+
+async def _expire_swarm(session, host) -> None:
+    """Drop the swarm rows a host reported while it was a manager."""
+    from .models import DockerSwarmNode, DockerSwarmService
+
+    await session.execute(delete(DockerSwarmService).where(DockerSwarmService.host_id == host.id))
+    await session.execute(delete(DockerSwarmNode).where(DockerSwarmNode.host_id == host.id))
+
+
 async def persist_host_docker(session, host, snapshots, swarm=None, disk=None,
-                              logs=None) -> None:
+                              logs=None, status=None) -> None:
     """Upsert containers + time-series + lifecycle events + swarm for one host.
 
     `snapshots` is a list of (recorded_at: datetime, containers: list[dict]) —
@@ -264,11 +293,18 @@ async def persist_host_docker(session, host, snapshots, swarm=None, disk=None,
     `disk` is the newest sample's /system/df summary (or None on Docker-less
     hosts) — persisted when present. `logs` is a list of (recorded_at, records)
     log batches (m79) — appended one row per line when present, independent of
-    whether this sample also carried container metrics.
+    whether this sample also carried container metrics. `status` is the newest
+    sample's {"status", "detail", "at", "swarm_manager"} (agent ≥ 1.9.0): an
+    "ok" with an empty snapshot clears the host's containers, and a node that
+    says it is no longer a manager loses the swarm rows it reported as one.
     """
     from steward.core.alerts import MetricBatch
     from .models import DockerContainer, DockerEvent, DockerMetric
 
+    if status is not None:
+        await _persist_status(session, host, status)
+        if status.get("swarm_manager") is False:
+            await _expire_swarm(session, host)
     if swarm is not None:
         await _persist_swarm(session, host, swarm)
     if disk is not None:

plugins/docker/migrations/versions/docker_011_host_status.py

@@ -0,0 +1,36 @@
+"""Docker host status
+
+Adds docker_host_status: per host, whether Docker answered at the agent's
+last report (ok / unavailable / error) and why not. Lets Steward show a host
+whose Docker went away as such, instead of its last containers as running
+(#5152). Additive create_table.
+
+Revision ID: docker_011_host_status
+Revises: docker_010_swarm_tasks
+Create Date: 2026-10-06
+"""
+from typing import Sequence, Union
+from alembic import op
+import sqlalchemy as sa
+
+revision: str = "docker_011_host_status"
+down_revision: Union[str, None] = "docker_010_swarm_tasks"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    op.create_table(
+        "docker_host_status",
+        sa.Column("host_id", sa.String(length=36), nullable=False),
+        sa.Column("status", sa.String(length=16), nullable=False),
+        sa.Column("detail", sa.String(length=255), nullable=True),
+        sa.Column("since", sa.DateTime(timezone=True), nullable=False),
+        sa.Column("reported_at", sa.DateTime(timezone=True), nullable=False),
+        sa.ForeignKeyConstraint(["host_id"], ["hosts.id"], ondelete="CASCADE"),
+        sa.PrimaryKeyConstraint("host_id"),
+    )
+
+
+def downgrade() -> None:
+    op.drop_table("docker_host_status")

plugins/docker/models.py

@@ -304,3 +304,24 @@ class DockerSwarmNode(Base):
         DateTime(timezone=True), nullable=False,
         default=lambda: datetime.now(timezone.utc),
     )
+
+
+class DockerHostStatus(Base):
+    """Whether Docker answered on a host at its agent's last report (agent ≥ 1.9.0).
+
+    "ok" when it did; "unavailable" when there is no Docker to ask (no socket,
+    nothing listening); "error" when Docker is there and the call failed.
+    `detail` says why for the last two. While a host is not "ok" its container
+    rows are what Docker last said, not what is true now, and read as stale.
+    A host that never had Docker gets no row.
+    """
+    __tablename__ = "docker_host_status"
+
+    host_id: Mapped[str] = mapped_column(
+        String(36), ForeignKey("hosts.id", ondelete="CASCADE"), primary_key=True
+    )
+    status: Mapped[str] = mapped_column(String(16), nullable=False)
+    detail: Mapped[str | None] = mapped_column(String(255), nullable=True)
+    # When the status last changed, and the agent's newest report.
+    since: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
+    reported_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)

plugins/docker/routes.py

@@ -18,7 +18,7 @@ from . import service as docker_svc
 from .dedup import dedup_by_container_id
 from .swarm_view import build_swarm_services, parse_tasks, task_summary
 from .models import (
-    DockerContainer, DockerDiskUsage, DockerEvent, DockerImage,
+    DockerContainer, DockerDiskUsage, DockerEvent, DockerHostStatus, DockerImage,
     DockerSwarmNode, DockerSwarmService,
 )
 
@@ -87,6 +87,9 @@ async def rows():
     stale_after = docker_svc.stale_after_seconds(current_app.config)
 
     async with current_app.db_sessionmaker() as db:
+        # Hosts whose Docker is not answering: their rows are what it last said.
+        docker_down = {d.host_id: d for d in (await db.execute(
+            select(DockerHostStatus).where(DockerHostStatus.status != "ok"))).scalars()}
         ranked = await docker_svc.ranked_containers(
             db, now, docker_svc.crash_loop_deaths(current_app.config))
         containers = await docker_svc.list_containers(db)
@@ -95,7 +98,7 @@ async def rows():
         hosts = await docker_svc.host_map(
             db,
             {c.host_id for c in containers}
-            | {n.host_id for n in nodes} | {s.host_id for s in services},
+            | {n.host_id for n in nodes} | {s.host_id for s in services} | set(docker_down),
         )
 
         # Swarm tasks collapse into per-service rows (each replica labelled with
@@ -123,7 +126,7 @@ async def rows():
                 "container": c,
                 "ports": json.loads(c.ports_json) if c.ports_json else [],
                 "uptime": _uptime(c.started_at) if c.status == "running" else None,
-                "stale": _is_stale(c, now, stale_after),
+                "stale": _is_stale(c, now, stale_after, docker_down),
             })
             if c.status == "running":
                 g["running"] += 1
@@ -151,12 +154,16 @@ async def rows():
     swarm_services = swarm_view["services"]
     standalone_running = sum(g["running"] for g in host_groups)
     ghost_total = sum(r["count"] for s in swarm_services for r in s["replicas"] if r["ghost"])
-    attention = [dict(r, stale=_is_stale(r["container"], now, stale_after))
+    attention = [dict(r, stale=_is_stale(r["container"], now, stale_after, docker_down))
                  for r in docker_svc.needs_attention(ranked)]
     return await render_template(
         "docker/rows.html",
         host_groups=host_groups,
         swarm_services=swarm_services,
+        docker_down=sorted(
+            ({"host_id": h, "host_name": hosts[h].name if h in hosts else h, "status": d.status,
+              "detail": d.detail, "since": d.since} for h, d in docker_down.items()),
+            key=lambda d: d["host_name"].lower()),
         attention=attention, says=docker_svc.PROBLEM_SAYS, severity=docker_svc.PROBLEM_SEVERITY,
         unhealthy=sum("unhealthy" in r["problems"] for r in attention),
         restarting=sum(bool({"restarting", "crash_looping"} & set(r["problems"])) for r in attention),
@@ -167,10 +174,12 @@ async def rows():
     )
 
 
-def _is_stale(c, now: datetime, stale_after: int) -> bool:
+def _is_stale(c, now: datetime, stale_after: int, docker_down=()) -> bool:
     """A container whose last report is older than the agent's staleness
-    window: what the page shows may no longer be true."""
-    return c.scraped_at is None or (now - c.scraped_at).total_seconds() > stale_after
+    window, or whose host's Docker is not answering: what the page shows may
+    no longer be true."""
+    return (c.host_id in docker_down or c.scraped_at is None
+            or (now - c.scraped_at).total_seconds() > stale_after)
 
 
 @docker_bp.get("/widget")

plugins/docker/templates/docker/rows.html

@@ -1,5 +1,3 @@
-{# docker/rows.html — HTMX fragment for the Docker main page, grouped by host #}
-
 {# docker/rows.html — HTMX fragment for the Docker page: what needs attention
    first (service.needs_attention, the ranking the MCP tools use), the counts,
    then swarm services and each host's containers. #}
@@ -28,6 +26,22 @@
 </section>
 {% endif %}
 
+{# ── Hosts whose Docker is not answering (agent ≥ 1.9.0) ──────────────────── #}
+{% if docker_down %}
+<section class="card problem-card" aria-labelledby="docker-unreachable">
+  <h2 id="docker-unreachable" class="section-title" style="margin-bottom:0.6rem;">Docker not answering</h2>
+  <ul class="problem-list">
+    {% for d in docker_down %}
+    <li>
+      {{ status_badge('crit', 'Error') if d.status == 'error' else status_badge('stale', 'No Docker') }}
+      <a href="/hosts/{{ d.host_id }}">{{ d.host_name }}</a>
+      <span class="problem-detail">{{ d.detail or "no reason given" }} · since {{ d.since|ago }} · its containers below are what Docker last reported</span>
+    </li>
+    {% endfor %}
+  </ul>
+</section>
+{% endif %}
+
 {# ── Counts ───────────────────────────────────────────────────────────────── #}
 <div class="stat-strip">
   {{ stat_tile("Needs attention", attention|length, state="crit" if attention else "inactive") }}

plugins/host_agent/agent.py

@@ -20,7 +20,7 @@ from collections import deque
 from concurrent.futures import ThreadPoolExecutor
 from datetime import datetime, timezone
 
-AGENT_VERSION = "1.8.0"
+AGENT_VERSION = "1.9.0"
 
 # Default path to the local Docker Engine socket. Overridable via the
 # `docker_socket` config key; collection is silently skipped if it's absent or
@@ -645,19 +645,37 @@ def _collect_one_container(socket_path: str, c: dict) -> dict:
     return record
 
 
-def collect_docker(socket_path: str) -> list:
-    """Per-container state from the local Docker socket, or [] if unavailable.
+def _docker_failure(socket_path: str, exc: Exception) -> tuple:
+    """(status, detail) for a failed Docker call: "unavailable" when there is no
+    Docker here to ask, "error" when there is one and the call went wrong."""
+    if isinstance(exc, FileNotFoundError):
+        return "unavailable", "no Docker socket at " + socket_path
+    if isinstance(exc, ConnectionRefusedError):
+        return "unavailable", "Docker is not accepting connections on " + socket_path
+    if isinstance(exc, PermissionError):
+        return "error", "permission denied on " + socket_path
+    if isinstance(exc, socket.timeout):
+        return "error", "Docker did not answer within %.0f s" % DOCKER_API_TIMEOUT
+    return "error", (str(exc) or type(exc).__name__)[:200]
 
-    Absent socket / permission denied / non-Docker endpoint all yield [] — the
-    agent silently reports no containers rather than erroring, so a Docker-less
-    host needs no configuration.
+
+def collect_docker(socket_path: str) -> tuple:
+    """(status, detail, containers) from the local Docker socket.
+
+    status is "ok" when Docker answered (containers may then be [] on a host
+    with none, which tells Steward to clear its list), "unavailable" when there
+    is no Docker here (no socket, or nothing listening), and "error" when Docker
+    is here but the listing failed. detail says why, for the last two. A
+    Docker-less host still needs no configuration: it reports "unavailable".
     """
     try:
         containers = _docker_request(socket_path, "/containers/json?all=true")
-    except (OSError, ValueError):
-        return []
-    if not isinstance(containers, list) or not containers:
-        return []
+    except (OSError, ValueError) as exc:
+        return _docker_failure(socket_path, exc) + ([],)
+    if not isinstance(containers, list):
+        return "error", "unexpected answer from /containers/json", []
+    if not containers:
+        return "ok", None, []
 
     # Each container needs an inspect (+ a stats read if running), and the stats
     # call blocks ~1s while the daemon computes the cpu delta. Done serially that
@@ -665,7 +683,7 @@ def collect_docker(socket_path: str) -> list:
     # over a small bounded thread pool (I/O-bound → threads are enough).
     workers = min(8, len(containers))
     with ThreadPoolExecutor(max_workers=workers) as ex:
-        return list(ex.map(lambda c: _collect_one_container(socket_path, c), containers))
+        return "ok", None, list(ex.map(lambda c: _collect_one_container(socket_path, c), containers))
 
 
 # ─── container logs (m79) ─────────────────────────────────────────────────────
@@ -978,26 +996,28 @@ def _swarm_nodes(nodes: list) -> list:
     return out
 
 
-def collect_swarm(socket_path: str):
-    """Swarm topology from a manager node, or None on workers / non-swarm hosts.
+def collect_swarm(socket_path: str) -> tuple:
+    """(is_manager, topology) from the local Docker socket.
 
     Self-detects manager via `/info` (one cheap call); only then queries the
-    manager-only endpoints. Any transport failure degrades to None, same silent
-    contract as collect_docker — a non-swarm host adds nothing to the payload.
+    manager-only endpoints. is_manager is None when /info itself failed, so a
+    transient error is never mistaken for a node that stopped being a manager;
+    False tells Steward to drop the swarm rows this host reported as one.
+    topology is None unless this is a manager and every call answered.
     """
     try:
         info = _docker_request(socket_path, "/info")
     except (OSError, ValueError):
-        return None
+        return None, None
     if not _swarm_is_manager(info):
-        return None
+        return False, None
     try:
         services = _docker_request(socket_path, "/services")
         tasks = _docker_request(socket_path, "/tasks")
         nodes = _docker_request(socket_path, "/nodes")
     except (OSError, ValueError):
-        return None
-    return {
+        return True, None
+    return True, {
         "services": _swarm_services(
             services if isinstance(services, list) else [],
             tasks if isinstance(tasks, list) else []),
@@ -1161,26 +1181,32 @@ def build_sample(mounts: list[str], state: dict,
     ]
     state["net"], state["disk"], state["mono"] = net_raw, disk_raw, now_mono
 
+    # Docker's status rides every sample, so Steward can tell "no containers"
+    # (status ok, docker: []) from "couldn't ask" (unavailable / error).
     try:
-        docker = collect_docker(docker_socket)
-    except Exception:
-        docker = []
-    if docker:
+        docker_status, docker_detail, docker = collect_docker(docker_socket)
+    except Exception as exc:
+        docker_status, docker_detail, docker = "error", (str(exc) or type(exc).__name__)[:200], []
+    sample["docker_status"] = docker_status
+    if docker_detail:
+        sample["docker_detail"] = docker_detail
+    if docker_status == "ok":
         sample["docker"] = docker
 
-    # Swarm topology is only emitted by managers (collect_swarm returns None
-    # everywhere else), so the key is absent on workers / non-swarm hosts.
+    # Swarm topology is only emitted by managers. A node whose /info says it
+    # is not a manager says so, so Steward can expire what it once reported.
     try:
-        swarm = collect_swarm(docker_socket)
+        is_manager, swarm = collect_swarm(docker_socket)
     except Exception:
-        swarm = None
+        is_manager, swarm = None, None
     if swarm is not None:
         sample["swarm"] = swarm
+    elif is_manager is False:
+        sample["swarm_manager"] = False
 
-    # Image/disk usage — host-level docker fact (one /system/df call), omitted on
-    # Docker-less hosts. Only meaningful when there's docker here at all, so skip
-    # the call entirely when collect_docker found nothing.
-    if docker:
+    # Image/disk usage — host-level docker fact (one /system/df call), only
+    # when Docker answered.
+    if docker_status == "ok":
         try:
             disk = collect_disk_usage(docker_socket)
         except Exception:

plugins/host_agent/routes.py

@@ -214,15 +214,24 @@ async def ingest():
             latest_swarm_ts: datetime | None = None
             latest_disk: dict | None = None
             latest_disk_ts: datetime | None = None
+            # Whether Docker answered, from the newest sample that says (agent ≥ 1.9.0).
+            docker_status: dict | None = None
             for sample in samples:
                 try:
                     recorded_at = _parse_ts(sample["ts"])
                 except (KeyError, ValueError):
                     continue
                 metric_rows.extend(_expand_sample_to_metrics(sample, host.name, recorded_at))
+                # An empty list is a real answer: Docker is up with no containers.
                 docker = sample.get("docker")
-                if isinstance(docker, list) and docker:
+                if isinstance(docker, list):
                     docker_snapshots.append((recorded_at, docker))
+                status = sample.get("docker_status")
+                if status in ("ok", "unavailable", "error") and (
+                        docker_status is None or recorded_at > docker_status["at"]):
+                    docker_status = {
+                        "status": status, "detail": sample.get("docker_detail"),
+                        "at": recorded_at, "swarm_manager": sample.get("swarm_manager")}
                 # Container logs are time-series (append every line, not newest-only);
                 # each record carries its own Docker ts, recorded_at is the fallback.
                 docker_logs = sample.get("docker_logs")
@@ -257,7 +266,7 @@ async def ingest():
             # (opportunistic synergy via the capability registry — no hard import,
             # no-op when docker is disabled). A failure here must never sink the
             # whole ingest, so the metrics above still land.
-            if (docker_snapshots or latest_swarm is not None
+            if (docker_snapshots or latest_swarm is not None or docker_status is not None
                     or latest_disk is not None or docker_log_batches):
                 from steward.core.capabilities import has_capability, invoke_capability
                 if has_capability("docker.persist_host_samples"):
@@ -268,7 +277,7 @@ async def ingest():
                             await invoke_capability(
                                 "docker.persist_host_samples", UserRole.admin,
                                 session, host, docker_snapshots, latest_swarm,
-                                latest_disk, docker_log_batches,
+                                latest_disk, docker_log_batches, docker_status,
                             )
                     except Exception:
                         current_app.logger.exception(

tests/integration/test_docker.py

@@ -10,7 +10,7 @@ import asyncio
 import json
 import os
 import uuid
-from datetime import datetime, timezone
+from datetime import datetime, timedelta, timezone
 
 import pytest
 
@@ -383,6 +383,119 @@ def test_vanished_container_is_reaped(app):
     assert after == ["keep"]   # the vanished container was reaped
 
 
+@_NEEDS_DB
+def test_an_empty_snapshot_from_a_docker_that_answered_clears_the_host(app):
+    """Docker answering with no containers removes the host's rows (#5152); a
+    sample with no Docker answer at all removes nothing."""
+    from sqlalchemy import text
+    from steward.models.hosts import Host
+
+    persist = _persist_fn(app)
+    t1 = datetime(2026, 10, 6, 12, 0, 0, tzinfo=timezone.utc)
+    t2 = datetime(2026, 10, 6, 12, 0, 30, tzinfo=timezone.utc)
+    web = {"name": "web", "status": "running", "cpu_pct": 1.0, "mem_pct": 1.0,
+           "restart_count": 0, "exit_code": None, "oom_killed": False, "health": None}
+
+    async def _go():
+        async with app.db_sessionmaker() as s:
+            async with s.begin():
+                h = Host(id=str(uuid.uuid4()), name=f"empty-{uuid.uuid4().hex[:6]}", address="10.9.9.7")
+                s.add(h)
+                await s.flush()
+
+                async def names():
+                    return [r[0] for r in (await s.execute(text(
+                        "SELECT name FROM docker_containers WHERE host_id = :h"), {"h": h.id})).all()]
+
+                await persist(s, h, [(t1, [web])], status={"status": "ok", "detail": None, "at": t1})
+                first = await names()
+                await persist(s, h, [], status={"status": "error", "detail": "timed out", "at": t2})
+                after_error = await names()
+                await persist(s, h, [(t2, [])], status={"status": "ok", "detail": None, "at": t2})
+                return first, after_error, await names()
+
+    first, after_error, after_empty = asyncio.run(_go())
+    assert first == ["web"]
+    assert after_error == ["web"]   # couldn't ask: keep what Docker last said
+    assert after_empty == []        # asked, and there are none
+
+
+@_NEEDS_DB
+def test_docker_status_is_recorded_once_a_host_has_had_docker(app):
+    from steward.models.hosts import Host
+
+    persist = _persist_fn(app)
+    t1 = datetime(2026, 10, 6, 12, 0, 0, tzinfo=timezone.utc)
+    t2 = t1 + timedelta(seconds=30)
+    t3 = t1 + timedelta(seconds=60)
+
+    async def _go():
+        from plugins.docker.models import DockerHostStatus
+        seen = []
+        async with app.db_sessionmaker() as s:
+            async with s.begin():
+                never = Host(id=str(uuid.uuid4()), name=f"plain-{uuid.uuid4().hex[:6]}", address="10.9.9.5")
+                had = Host(id=str(uuid.uuid4()), name=f"had-{uuid.uuid4().hex[:6]}", address="10.9.9.6")
+                s.add_all([never, had])
+                await s.flush()
+                gone = {"status": "unavailable", "detail": "no Docker socket at /var/run/docker.sock"}
+                await persist(s, never, [], status=dict(gone, at=t1))
+                seen.append(await s.get(DockerHostStatus, never.id))
+                for status, at in (({"status": "ok", "detail": None}, t1),
+                                   (gone, t2), (gone, t3),
+                                   ({"status": "error", "detail": "permission denied on /var/run/docker.sock"}, t3)):
+                    await persist(s, had, [], status=dict(status, at=at))
+                    await s.flush()
+                    row = await s.get(DockerHostStatus, had.id)
+                    seen.append((row.status, row.detail, row.since, row.reported_at))
+        return seen
+
+    never, ok, gone1, gone2, err = asyncio.run(_go())
+    assert never is None                 # never had Docker: nothing to say
+    assert ok == ("ok", None, t1, t1)
+    assert gone1 == ("unavailable", "no Docker socket at /var/run/docker.sock", t2, t2)
+    assert gone2[2] == t2 and gone2[3] == t3   # still gone: since holds, the report moves on
+    assert err[:3] == ("error", "permission denied on /var/run/docker.sock", t3)
+
+
+@_NEEDS_DB
+def test_a_node_that_is_no_longer_a_manager_loses_its_swarm_rows(app):
+    from sqlalchemy import text
+    from steward.models.hosts import Host
+
+    persist = _persist_fn(app)
+    t1 = datetime(2026, 10, 6, 12, 0, 0, tzinfo=timezone.utc)
+    swarm = {"services": [{"service_name": "web", "mode": "replicated", "desired": 1, "running": 1}],
+             "nodes": [{"node_id": "n1", "hostname": "mgr", "role": "manager"}]}
+
+    async def _go():
+        async with app.db_sessionmaker() as s:
+            async with s.begin():
+                h = Host(id=str(uuid.uuid4()), name=f"demoted-{uuid.uuid4().hex[:6]}", address="10.9.9.4")
+                s.add(h)
+                await s.flush()
+
+                async def counts():
+                    return tuple((await s.execute(text(
+                        f"SELECT count(*) FROM {t} WHERE host_id = :h"), {"h": h.id})).scalar()
+                        for t in ("docker_swarm_services", "docker_swarm_nodes"))
+
+                await persist(s, h, [], swarm, status={"status": "ok", "detail": None, "at": t1})
+                manager = await counts()
+                # /info failed: not knowing is not "no longer a manager".
+                await persist(s, h, [], status={"status": "error", "detail": "x", "at": t1,
+                                                "swarm_manager": None})
+                unknown = await counts()
+                await persist(s, h, [], status={"status": "ok", "detail": None, "at": t1,
+                                                "swarm_manager": False})
+                return manager, unknown, await counts()
+
+    manager, unknown, demoted = asyncio.run(_go())
+    assert manager == (1, 1)
+    assert unknown == (1, 1)
+    assert demoted == (0, 0)
+
+
 @_NEEDS_DB
 def test_swarm_topology_persisted(app):
     from sqlalchemy import text

tests/integration/test_host_agent_docker_status.py

@@ -0,0 +1,79 @@
+"""Integration: an agent's Docker status reaches Steward through ingest (#5152).
+
+The newest sample's status wins over a backlog's older ones; "error" keeps
+the containers Docker last listed; an "ok" with an empty list clears them.
+
+Rows are read with plain SQL: importing the plugin's models again after the
+app loaded the plugin raises "Table already defined" (see test_mcp_plugin_tools)."""
+from __future__ import annotations
+
+import asyncio
+import hashlib
+import os
+import uuid
+from datetime import datetime, timedelta, timezone
+
+import pytest
+
+pytestmark = pytest.mark.integration
+
+
+@pytest.fixture
+def app():
+    if not os.environ.get("STEWARD_DATABASE_URL"):
+        pytest.skip("needs Postgres")
+    from steward.app import create_app
+    return create_app(testing=False)
+
+
+def test_ingest_records_docker_status_and_clears_on_an_empty_answer(app):
+    from sqlalchemy import text
+    from steward.models.hosts import Host
+
+    web = {"name": "web", "status": "running", "cpu_pct": 1.0, "mem_pct": 1.0,
+           "restart_count": 0, "exit_code": None, "oom_killed": False, "health": None}
+
+    async def _go():
+        tag = uuid.uuid4().hex[:8]
+        token = f"tok-{tag}-{uuid.uuid4().hex}"
+        async with app.db_sessionmaker() as s:
+            async with s.begin():
+                host = Host(name=f"dock-{tag}", address="10.9.9.3")
+                s.add(host)
+                await s.flush()
+                await s.execute(text(
+                    "INSERT INTO host_agent_registrations "
+                    "(id, host_id, token_hash, token_created_at, created_at, updated_at) "
+                    "VALUES (:id, :host, :hash, now(), now(), now())"),
+                    {"id": str(uuid.uuid4()), "host": host.id,
+                     "hash": hashlib.sha256(token.encode()).hexdigest()})
+            host_id = host.id
+
+        client = app.test_client()
+        now = datetime.now(timezone.utc)
+
+        async def post(*samples):
+            resp = await client.post("/plugins/host_agent/ingest", json={"samples": list(samples)},
+                                     headers={"Authorization": f"Bearer {token}"})
+            assert resp.status_code == 200, await resp.get_data(as_text=True)
+            async with app.db_sessionmaker() as s:
+                status = (await s.execute(text(
+                    "SELECT status, detail FROM docker_host_status WHERE host_id = :h"),
+                    {"h": host_id})).first()
+                names = [r[0] for r in (await s.execute(text(
+                    "SELECT name FROM docker_containers WHERE host_id = :h"), {"h": host_id})).all()]
+            return (tuple(status) if status else None), names
+
+        def at(seconds_ago):
+            return (now - timedelta(seconds=seconds_ago)).isoformat()
+
+        # A backlog, newest first: Docker listed web, then stopped answering.
+        backlog = await post(
+            {"ts": at(60), "docker_status": "error", "docker_detail": "permission denied on /s"},
+            {"ts": at(90), "docker_status": "ok", "docker": [web]})
+        emptied = await post({"ts": at(30), "docker_status": "ok", "docker": []})
+        return backlog, emptied
+
+    backlog, emptied = asyncio.run(_go())
+    assert backlog == (("error", "permission denied on /s"), ["web"])
+    assert emptied == (("ok", None), [])

tests/plugins/docker/test_docker_page.py

@@ -33,6 +33,9 @@ def test_stale_is_older_than_the_agents_window():
     assert not _is_stale(_c("a"), NOW, 180)
     assert _is_stale(_c("a", scraped_at=NOW - timedelta(minutes=10)), NOW, 180)
     assert _is_stale(_c("a", scraped_at=None), NOW, 180)
+    # A fresh row on a host whose Docker stopped answering is stale too (#5152).
+    assert _is_stale(_c("a"), NOW, 180, {"h1": object()})
+    assert not _is_stale(_c("a"), NOW, 180, {"h2": object()})
     assert svc.stale_after_seconds({"PLUGINS": {"host_agent": {"stale_after_seconds": 60}}}) == 60
     assert svc.stale_after_seconds({}) == 180
 
@@ -74,3 +77,29 @@ async def test_a_healthy_board_has_no_attention_card(app):
             app, "docker", "docker/rows.html", attention=[], says=svc.PROBLEM_SAYS, severity=svc.PROBLEM_SEVERITY,
             unhealthy=0, restarting=0, stale=0, host_groups=[], swarm_services=[], running=0, stopped=0, total=0)
     assert "Needs attention</h2>" not in page and "No containers reported yet" in page
+
+
+async def test_a_host_whose_docker_stopped_answering_says_so(app):
+    down = [{"host_id": "h1", "host_name": "nas", "status": "unavailable",
+             "detail": "no Docker socket at /var/run/docker.sock", "since": NOW - timedelta(minutes=5)},
+            {"host_id": "h2", "host_name": "web-1", "status": "error",
+             "detail": "permission denied on /var/run/docker.sock", "since": NOW - timedelta(minutes=1)}]
+    async with app.test_request_context("/plugins/docker/rows"):
+        page = await render_plugin_template(
+            app, "docker", "docker/rows.html", docker_down=down, attention=[], says=svc.PROBLEM_SAYS,
+            severity=svc.PROBLEM_SEVERITY, unhealthy=0, restarting=0, stale=0, host_groups=[],
+            swarm_services=[], running=0, stopped=0, total=0)
+    page = re.sub(r">\s+<", "><", " ".join(page.split()))
+    card = page[page.index("Docker not answering"):page.index("</section>")]
+    assert '<span class="badge badge-warn">No Docker</span><a href="/hosts/h1">nas</a>' in card
+    assert "no Docker socket at /var/run/docker.sock · since <time" in card and ">5m ago</time>" in card
+    assert '<span class="badge badge-crit">Error</span><a href="/hosts/h2">web-1</a>' in card
+
+
+async def test_no_card_while_every_docker_answers(app):
+    async with app.test_request_context("/plugins/docker/rows"):
+        page = await render_plugin_template(
+            app, "docker", "docker/rows.html", docker_down=[], attention=[], says=svc.PROBLEM_SAYS,
+            severity=svc.PROBLEM_SEVERITY, unhealthy=0, restarting=0, stale=0, host_groups=[],
+            swarm_services=[], running=0, stopped=0, total=0)
+    assert "Docker not answering" not in page

tests/plugins/host_agent/test_agent_docker.py

@@ -2,9 +2,13 @@
 """Unit tests for the agent's stdlib Docker collector.
 
 No socket / no network — the UDS request layer is monkeypatched so we exercise
-the cpu/mem math, chunked-body decoding, sample assembly, and the silent-skip
-contract in isolation.
+the cpu/mem math, chunked-body decoding, sample assembly, and what the agent
+says when Docker cannot be asked, in isolation.
 """
+import socket
+
+import pytest
+
 from plugins.host_agent import agent as a
 
 
@@ -111,9 +115,31 @@ def test_docker_net_io_sums_counters():
 
 # ── collect_docker assembly + silent skip ─────────────────────────────────────
 
-def test_collect_docker_silent_skip_when_no_socket():
-    # Absent socket path must degrade to [] (zero-config on Docker-less hosts).
-    assert a.collect_docker("/nonexistent/does-not-exist.sock") == []
+def test_collect_docker_says_unavailable_when_no_socket():
+    # A Docker-less host needs no configuration: it reports there is no Docker.
+    assert a.collect_docker("/nonexistent/does-not-exist.sock") == (
+        "unavailable", "no Docker socket at /nonexistent/does-not-exist.sock", [])
+
+
+@pytest.mark.parametrize("exc, status, detail", [
+    (FileNotFoundError(2, "x"), "unavailable", "no Docker socket at /s"),
+    (ConnectionRefusedError(111, "x"), "unavailable", "Docker is not accepting connections on /s"),
+    (PermissionError(13, "x"), "error", "permission denied on /s"),
+    (socket.timeout("timed out"), "error", "Docker did not answer within 5 s"),
+    (OSError("docker API /containers/json → 500"), "error", "docker API /containers/json → 500"),
+    (ValueError("Expecting value"), "error", "Expecting value"),
+])
+def test_collect_docker_says_why_it_could_not_list(monkeypatch, exc, status, detail):
+    def fail(socket_path, path, timeout=a.DOCKER_API_TIMEOUT):
+        raise exc
+
+    monkeypatch.setattr(a, "_docker_request", fail)
+    assert a.collect_docker("/s") == (status, detail, [])
+
+
+def test_collect_docker_with_no_containers_is_ok_and_empty(monkeypatch):
+    monkeypatch.setattr(a, "_docker_request", lambda s, p, timeout=a.DOCKER_API_TIMEOUT: [])
+    assert a.collect_docker("/s") == ("ok", None, [])
 
 
 def test_collect_docker_assembles_container(monkeypatch):
@@ -151,7 +177,8 @@ def test_collect_docker_assembles_container(monkeypatch):
         raise AssertionError(f"unexpected path {path}")
 
     monkeypatch.setattr(a, "_docker_request", fake_request)
-    out = a.collect_docker("/var/run/docker.sock")
+    status, detail, out = a.collect_docker("/var/run/docker.sock")
+    assert (status, detail) == ("ok", None)
     assert len(out) == 1
     c = out[0]
     assert c["name"] == "web"
@@ -184,7 +211,7 @@ def test_collect_docker_skips_stats_for_stopped(monkeypatch):
         raise AssertionError("stats must not be fetched for a stopped container")
 
     monkeypatch.setattr(a, "_docker_request", fake_request)
-    out = a.collect_docker("/var/run/docker.sock")
+    _, _, out = a.collect_docker("/var/run/docker.sock")
     assert out[0]["status"] == "exited"
     assert out[0]["cpu_pct"] is None and out[0]["started_at"] is None
     # inspect still gives us why it died
@@ -289,7 +316,7 @@ def test_collect_swarm_skips_on_worker(monkeypatch):
         raise AssertionError("manager-only endpoint queried on a worker")
 
     monkeypatch.setattr(a, "_docker_request", fake_request)
-    assert a.collect_swarm("/var/run/docker.sock") is None
+    assert a.collect_swarm("/var/run/docker.sock") == (False, None)
 
 
 def test_collect_swarm_assembles_on_manager(monkeypatch):
@@ -309,39 +336,85 @@ def test_collect_swarm_assembles_on_manager(monkeypatch):
         }[path]
 
     monkeypatch.setattr(a, "_docker_request", fake_request)
-    out = a.collect_swarm("/var/run/docker.sock")
+    is_manager, out = a.collect_swarm("/var/run/docker.sock")
+    assert is_manager is True
     assert out["services"][0]["service_name"] == "web"
     assert out["services"][0]["running"] == 1 and out["services"][0]["desired"] == 1
     assert out["nodes"][0]["hostname"] == "mgr" and out["nodes"][0]["leader"] is True
 
 
-def test_collect_swarm_silent_skip_when_no_socket():
-    assert a.collect_swarm("/nonexistent/does-not-exist.sock") is None
+def test_collect_swarm_cannot_say_when_info_fails():
+    # Not knowing is not "no longer a manager": Steward must keep the rows.
+    assert a.collect_swarm("/nonexistent/does-not-exist.sock") == (None, None)
+
+
+def test_collect_swarm_manager_whose_listing_fails_sends_nothing(monkeypatch):
+    def fake_request(socket_path, path, timeout=a.DOCKER_API_TIMEOUT):
+        if path == "/info":
+            return {"Swarm": {"ControlAvailable": True}}
+        raise OSError("docker API /services → 503")
+
+    monkeypatch.setattr(a, "_docker_request", fake_request)
+    assert a.collect_swarm("/var/run/docker.sock") == (True, None)
 
 
 # ── build_sample wiring ───────────────────────────────────────────────────────
 
 def test_build_sample_includes_docker_when_present(monkeypatch):
-    monkeypatch.setattr(a, "collect_docker", lambda _s: [{"name": "web", "status": "running"}])
-    monkeypatch.setattr(a, "collect_swarm", lambda _s: None)
+    monkeypatch.setattr(a, "collect_docker",
+                        lambda _s: ("ok", None, [{"name": "web", "status": "running"}]))
+    monkeypatch.setattr(a, "collect_swarm", lambda _s: (None, None))
     monkeypatch.setattr(a, "collect_disk_usage", lambda _s: None)
     sample = a.build_sample(["/"], {}, "/var/run/docker.sock")
     assert sample["docker"] == [{"name": "web", "status": "running"}]
+    assert sample["docker_status"] == "ok" and "docker_detail" not in sample
 
 
-def test_build_sample_omits_docker_when_empty(monkeypatch):
-    monkeypatch.setattr(a, "collect_docker", lambda _s: [])
-    monkeypatch.setattr(a, "collect_swarm", lambda _s: None)
+def test_build_sample_sends_an_empty_list_when_docker_has_no_containers(monkeypatch):
+    # "No containers" must reach Steward, so it can clear the host's list (#5152).
+    monkeypatch.setattr(a, "collect_docker", lambda _s: ("ok", None, []))
+    monkeypatch.setattr(a, "collect_swarm", lambda _s: (None, None))
+    monkeypatch.setattr(a, "collect_disk_usage", lambda _s: None)
     sample = a.build_sample(["/"], {}, "/var/run/docker.sock")
-    assert "docker" not in sample
+    assert sample["docker"] == [] and sample["docker_status"] == "ok"
+
+
+def test_build_sample_says_why_docker_could_not_be_asked(monkeypatch):
+    monkeypatch.setattr(a, "collect_docker",
+                        lambda _s: ("error", "permission denied on /var/run/docker.sock", []))
+    monkeypatch.setattr(a, "collect_swarm", lambda _s: (None, None))
+
+    def no_df(_s):
+        raise AssertionError("/system/df must not be asked when Docker did not answer")
+
+    monkeypatch.setattr(a, "collect_disk_usage", no_df)
+    sample = a.build_sample(["/"], {}, "/var/run/docker.sock")
+    assert "docker" not in sample   # not "no containers": unknown
+    assert sample["docker_status"] == "error"
+    assert sample["docker_detail"] == "permission denied on /var/run/docker.sock"
 
 
 def test_build_sample_includes_swarm_on_manager(monkeypatch):
-    monkeypatch.setattr(a, "collect_docker", lambda _s: [])
+    monkeypatch.setattr(a, "collect_docker", lambda _s: ("ok", None, []))
     monkeypatch.setattr(a, "collect_swarm",
-                        lambda _s: {"services": [], "nodes": [{"node_id": "n1"}]})
+                        lambda _s: (True, {"services": [], "nodes": [{"node_id": "n1"}]}))
+    monkeypatch.setattr(a, "collect_disk_usage", lambda _s: None)
     sample = a.build_sample(["/"], {}, "/var/run/docker.sock")
     assert sample["swarm"]["nodes"] == [{"node_id": "n1"}]
+    assert "swarm_manager" not in sample
+
+
+@pytest.mark.parametrize("swarm, flag", [
+    ((False, None), False),   # /info says not a manager: say so
+    ((None, None), None),     # /info failed: say nothing
+    ((True, None), None),     # a manager whose listing failed: say nothing
+])
+def test_build_sample_says_when_a_node_is_not_a_manager(monkeypatch, swarm, flag):
+    monkeypatch.setattr(a, "collect_docker", lambda _s: ("ok", None, []))
+    monkeypatch.setattr(a, "collect_swarm", lambda _s: swarm)
+    monkeypatch.setattr(a, "collect_disk_usage", lambda _s: None)
+    sample = a.build_sample(["/"], {}, "/var/run/docker.sock")
+    assert sample.get("swarm_manager") is flag and "swarm" not in sample
 
 
 def test_build_sample_omits_swarm_off_manager(monkeypatch):