feat(docker): Docker's own status reaches Steward — an empty answer clears a host, a lost one greys it, a demoted manager's swarm rows go (#5152)

The agent (1.9.0) said nothing when Docker could not be asked and nothing
when Docker had no containers, so Steward could tell neither apart from the
other, and kept a host's last containers on screen as running either way.

- Agent: collect_docker returns (status, detail, containers). "ok" sends
  docker: [] when there are none; "unavailable" (no socket, nothing
  listening) and "error" (permission, timeout, bad answer) say why.
  collect_swarm says whether /info answered, and a node that is not a
  manager sends swarm_manager: false. A failed /info says nothing.
- Ingest: an empty docker list is a snapshot; the newest sample's status
  goes to the Docker plugin with the rest.
- docker_host_status (docker_011): per host, status, detail, since and
  last report. A host that never had Docker gets no row.
- An "ok" empty snapshot reaps the host's containers; an error keeps them.
  swarm_manager: false drops the host's swarm services and nodes.
- Docker page: a "Docker not answering" card names each such host, why,
  and since when; its containers read as stale. Duplicate header comment
  in rows.html removed.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

plugins/docker/ingest.py

@@ -251,8 +251,37 @@ async def _persist_disk(session, host, disk: dict) -> None:
     await session.execute(stale)
 
 
+async def _persist_status(session, host, status: dict) -> None:
+    """Record whether Docker answered on this host, and why not.
+
+    A host reporting "unavailable" with no row yet never had Docker: nothing to
+    record. Once a host has had Docker, losing it is recorded, so its last
+    containers read as stale rather than as running."""
+    from .models import DockerHostStatus
+
+    row = await session.get(DockerHostStatus, host.id)
+    if row is None:
+        if status["status"] == "unavailable":
+            return
+        row = DockerHostStatus(host_id=host.id)
+        session.add(row)
+    if row.status != status["status"]:
+        row.since = status["at"]
+    row.status = status["status"]
+    row.detail = (status.get("detail") or "")[:255] or None
+    row.reported_at = status["at"]
+
+
+async def _expire_swarm(session, host) -> None:
+    """Drop the swarm rows a host reported while it was a manager."""
+    from .models import DockerSwarmNode, DockerSwarmService
+
+    await session.execute(delete(DockerSwarmService).where(DockerSwarmService.host_id == host.id))
+    await session.execute(delete(DockerSwarmNode).where(DockerSwarmNode.host_id == host.id))
+
+
 async def persist_host_docker(session, host, snapshots, swarm=None, disk=None,
-                              logs=None) -> None:
+                              logs=None, status=None) -> None:
     """Upsert containers + time-series + lifecycle events + swarm for one host.
 
     `snapshots` is a list of (recorded_at: datetime, containers: list[dict]) —
@@ -264,11 +293,18 @@ async def persist_host_docker(session, host, snapshots, swarm=None, disk=None,
     `disk` is the newest sample's /system/df summary (or None on Docker-less
     hosts) — persisted when present. `logs` is a list of (recorded_at, records)
     log batches (m79) — appended one row per line when present, independent of
-    whether this sample also carried container metrics.
+    whether this sample also carried container metrics. `status` is the newest
+    sample's {"status", "detail", "at", "swarm_manager"} (agent ≥ 1.9.0): an
+    "ok" with an empty snapshot clears the host's containers, and a node that
+    says it is no longer a manager loses the swarm rows it reported as one.
     """
     from steward.core.alerts import MetricBatch
     from .models import DockerContainer, DockerEvent, DockerMetric
 
+    if status is not None:
+        await _persist_status(session, host, status)
+        if status.get("swarm_manager") is False:
+            await _expire_swarm(session, host)
     if swarm is not None:
         await _persist_swarm(session, host, swarm)
     if disk is not None:

plugins/docker/migrations/versions/docker_011_host_status.py

@@ -0,0 +1,36 @@
+"""Docker host status
+
+Adds docker_host_status: per host, whether Docker answered at the agent's
+last report (ok / unavailable / error) and why not. Lets Steward show a host
+whose Docker went away as such, instead of its last containers as running
+(#5152). Additive create_table.
+
+Revision ID: docker_011_host_status
+Revises: docker_010_swarm_tasks
+Create Date: 2026-10-06
+"""
+from typing import Sequence, Union
+from alembic import op
+import sqlalchemy as sa
+
+revision: str = "docker_011_host_status"
+down_revision: Union[str, None] = "docker_010_swarm_tasks"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    op.create_table(
+        "docker_host_status",
+        sa.Column("host_id", sa.String(length=36), nullable=False),
+        sa.Column("status", sa.String(length=16), nullable=False),
+        sa.Column("detail", sa.String(length=255), nullable=True),
+        sa.Column("since", sa.DateTime(timezone=True), nullable=False),
+        sa.Column("reported_at", sa.DateTime(timezone=True), nullable=False),
+        sa.ForeignKeyConstraint(["host_id"], ["hosts.id"], ondelete="CASCADE"),
+        sa.PrimaryKeyConstraint("host_id"),
+    )
+
+
+def downgrade() -> None:
+    op.drop_table("docker_host_status")

plugins/docker/models.py

@@ -304,3 +304,24 @@ class DockerSwarmNode(Base):
         DateTime(timezone=True), nullable=False,
         default=lambda: datetime.now(timezone.utc),
     )
+
+
+class DockerHostStatus(Base):
+    """Whether Docker answered on a host at its agent's last report (agent ≥ 1.9.0).
+
+    "ok" when it did; "unavailable" when there is no Docker to ask (no socket,
+    nothing listening); "error" when Docker is there and the call failed.
+    `detail` says why for the last two. While a host is not "ok" its container
+    rows are what Docker last said, not what is true now, and read as stale.
+    A host that never had Docker gets no row.
+    """
+    __tablename__ = "docker_host_status"
+
+    host_id: Mapped[str] = mapped_column(
+        String(36), ForeignKey("hosts.id", ondelete="CASCADE"), primary_key=True
+    )
+    status: Mapped[str] = mapped_column(String(16), nullable=False)
+    detail: Mapped[str | None] = mapped_column(String(255), nullable=True)
+    # When the status last changed, and the agent's newest report.
+    since: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
+    reported_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)

plugins/docker/routes.py

@@ -18,7 +18,7 @@ from . import service as docker_svc
 from .dedup import dedup_by_container_id
 from .swarm_view import build_swarm_services, parse_tasks, task_summary
 from .models import (
-    DockerContainer, DockerDiskUsage, DockerEvent, DockerImage,
+    DockerContainer, DockerDiskUsage, DockerEvent, DockerHostStatus, DockerImage,
     DockerSwarmNode, DockerSwarmService,
 )
 
@@ -87,6 +87,9 @@ async def rows():
     stale_after = docker_svc.stale_after_seconds(current_app.config)
 
     async with current_app.db_sessionmaker() as db:
+        # Hosts whose Docker is not answering: their rows are what it last said.
+        docker_down = {d.host_id: d for d in (await db.execute(
+            select(DockerHostStatus).where(DockerHostStatus.status != "ok"))).scalars()}
         ranked = await docker_svc.ranked_containers(
             db, now, docker_svc.crash_loop_deaths(current_app.config))
         containers = await docker_svc.list_containers(db)
@@ -95,7 +98,7 @@ async def rows():
         hosts = await docker_svc.host_map(
             db,
             {c.host_id for c in containers}
-            | {n.host_id for n in nodes} | {s.host_id for s in services},
+            | {n.host_id for n in nodes} | {s.host_id for s in services} | set(docker_down),
         )
 
         # Swarm tasks collapse into per-service rows (each replica labelled with
@@ -123,7 +126,7 @@ async def rows():
                 "container": c,
                 "ports": json.loads(c.ports_json) if c.ports_json else [],
                 "uptime": _uptime(c.started_at) if c.status == "running" else None,
-                "stale": _is_stale(c, now, stale_after),
+                "stale": _is_stale(c, now, stale_after, docker_down),
             })
             if c.status == "running":
                 g["running"] += 1
@@ -151,12 +154,16 @@ async def rows():
     swarm_services = swarm_view["services"]
     standalone_running = sum(g["running"] for g in host_groups)
     ghost_total = sum(r["count"] for s in swarm_services for r in s["replicas"] if r["ghost"])
-    attention = [dict(r, stale=_is_stale(r["container"], now, stale_after))
+    attention = [dict(r, stale=_is_stale(r["container"], now, stale_after, docker_down))
                  for r in docker_svc.needs_attention(ranked)]
     return await render_template(
         "docker/rows.html",
         host_groups=host_groups,
         swarm_services=swarm_services,
+        docker_down=sorted(
+            ({"host_id": h, "host_name": hosts[h].name if h in hosts else h, "status": d.status,
+              "detail": d.detail, "since": d.since} for h, d in docker_down.items()),
+            key=lambda d: d["host_name"].lower()),
         attention=attention, says=docker_svc.PROBLEM_SAYS, severity=docker_svc.PROBLEM_SEVERITY,
         unhealthy=sum("unhealthy" in r["problems"] for r in attention),
         restarting=sum(bool({"restarting", "crash_looping"} & set(r["problems"])) for r in attention),
@@ -167,10 +174,12 @@ async def rows():
     )
 
 
-def _is_stale(c, now: datetime, stale_after: int) -> bool:
+def _is_stale(c, now: datetime, stale_after: int, docker_down=()) -> bool:
     """A container whose last report is older than the agent's staleness
-    window: what the page shows may no longer be true."""
-    return c.scraped_at is None or (now - c.scraped_at).total_seconds() > stale_after
+    window, or whose host's Docker is not answering: what the page shows may
+    no longer be true."""
+    return (c.host_id in docker_down or c.scraped_at is None
+            or (now - c.scraped_at).total_seconds() > stale_after)
 
 
 @docker_bp.get("/widget")

plugins/docker/templates/docker/rows.html

@@ -1,5 +1,3 @@
-{# docker/rows.html — HTMX fragment for the Docker main page, grouped by host #}
-
 {# docker/rows.html — HTMX fragment for the Docker page: what needs attention
    first (service.needs_attention, the ranking the MCP tools use), the counts,
    then swarm services and each host's containers. #}
@@ -28,6 +26,22 @@
 </section>
 {% endif %}
 
+{# ── Hosts whose Docker is not answering (agent ≥ 1.9.0) ──────────────────── #}
+{% if docker_down %}
+<section class="card problem-card" aria-labelledby="docker-unreachable">
+  <h2 id="docker-unreachable" class="section-title" style="margin-bottom:0.6rem;">Docker not answering</h2>
+  <ul class="problem-list">
+    {% for d in docker_down %}
+    <li>
+      {{ status_badge('crit', 'Error') if d.status == 'error' else status_badge('stale', 'No Docker') }}
+      <a href="/hosts/{{ d.host_id }}">{{ d.host_name }}</a>
+      <span class="problem-detail">{{ d.detail or "no reason given" }} · since {{ d.since|ago }} · its containers below are what Docker last reported</span>
+    </li>
+    {% endfor %}
+  </ul>
+</section>
+{% endif %}
+
 {# ── Counts ───────────────────────────────────────────────────────────────── #}
 <div class="stat-strip">
   {{ stat_tile("Needs attention", attention|length, state="crit" if attention else "inactive") }}