feat(docker): Docker's own status reaches Steward — an empty answer clears a host, a lost one greys it, a demoted manager's swarm rows go (#5152)

The agent (1.9.0) said nothing when Docker could not be asked and nothing
when Docker had no containers, so Steward could tell neither apart from the
other, and kept a host's last containers on screen as running either way.

- Agent: collect_docker returns (status, detail, containers). "ok" sends
  docker: [] when there are none; "unavailable" (no socket, nothing
  listening) and "error" (permission, timeout, bad answer) say why.
  collect_swarm says whether /info answered, and a node that is not a
  manager sends swarm_manager: false. A failed /info says nothing.
- Ingest: an empty docker list is a snapshot; the newest sample's status
  goes to the Docker plugin with the rest.
- docker_host_status (docker_011): per host, status, detail, since and
  last report. A host that never had Docker gets no row.
- An "ok" empty snapshot reaps the host's containers; an error keeps them.
  swarm_manager: false drops the host's swarm services and nodes.
- Docker page: a "Docker not answering" card names each such host, why,
  and since when; its containers read as stale. Duplicate header comment
  in rows.html removed.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

tests/integration/test_docker.py

@@ -10,7 +10,7 @@ import asyncio
 import json
 import os
 import uuid
-from datetime import datetime, timezone
+from datetime import datetime, timedelta, timezone
 
 import pytest
 
@@ -383,6 +383,119 @@ def test_vanished_container_is_reaped(app):
     assert after == ["keep"]   # the vanished container was reaped
 
 
+@_NEEDS_DB
+def test_an_empty_snapshot_from_a_docker_that_answered_clears_the_host(app):
+    """Docker answering with no containers removes the host's rows (#5152); a
+    sample with no Docker answer at all removes nothing."""
+    from sqlalchemy import text
+    from steward.models.hosts import Host
+
+    persist = _persist_fn(app)
+    t1 = datetime(2026, 10, 6, 12, 0, 0, tzinfo=timezone.utc)
+    t2 = datetime(2026, 10, 6, 12, 0, 30, tzinfo=timezone.utc)
+    web = {"name": "web", "status": "running", "cpu_pct": 1.0, "mem_pct": 1.0,
+           "restart_count": 0, "exit_code": None, "oom_killed": False, "health": None}
+
+    async def _go():
+        async with app.db_sessionmaker() as s:
+            async with s.begin():
+                h = Host(id=str(uuid.uuid4()), name=f"empty-{uuid.uuid4().hex[:6]}", address="10.9.9.7")
+                s.add(h)
+                await s.flush()
+
+                async def names():
+                    return [r[0] for r in (await s.execute(text(
+                        "SELECT name FROM docker_containers WHERE host_id = :h"), {"h": h.id})).all()]
+
+                await persist(s, h, [(t1, [web])], status={"status": "ok", "detail": None, "at": t1})
+                first = await names()
+                await persist(s, h, [], status={"status": "error", "detail": "timed out", "at": t2})
+                after_error = await names()
+                await persist(s, h, [(t2, [])], status={"status": "ok", "detail": None, "at": t2})
+                return first, after_error, await names()
+
+    first, after_error, after_empty = asyncio.run(_go())
+    assert first == ["web"]
+    assert after_error == ["web"]   # couldn't ask: keep what Docker last said
+    assert after_empty == []        # asked, and there are none
+
+
+@_NEEDS_DB
+def test_docker_status_is_recorded_once_a_host_has_had_docker(app):
+    from steward.models.hosts import Host
+
+    persist = _persist_fn(app)
+    t1 = datetime(2026, 10, 6, 12, 0, 0, tzinfo=timezone.utc)
+    t2 = t1 + timedelta(seconds=30)
+    t3 = t1 + timedelta(seconds=60)
+
+    async def _go():
+        from plugins.docker.models import DockerHostStatus
+        seen = []
+        async with app.db_sessionmaker() as s:
+            async with s.begin():
+                never = Host(id=str(uuid.uuid4()), name=f"plain-{uuid.uuid4().hex[:6]}", address="10.9.9.5")
+                had = Host(id=str(uuid.uuid4()), name=f"had-{uuid.uuid4().hex[:6]}", address="10.9.9.6")
+                s.add_all([never, had])
+                await s.flush()
+                gone = {"status": "unavailable", "detail": "no Docker socket at /var/run/docker.sock"}
+                await persist(s, never, [], status=dict(gone, at=t1))
+                seen.append(await s.get(DockerHostStatus, never.id))
+                for status, at in (({"status": "ok", "detail": None}, t1),
+                                   (gone, t2), (gone, t3),
+                                   ({"status": "error", "detail": "permission denied on /var/run/docker.sock"}, t3)):
+                    await persist(s, had, [], status=dict(status, at=at))
+                    await s.flush()
+                    row = await s.get(DockerHostStatus, had.id)
+                    seen.append((row.status, row.detail, row.since, row.reported_at))
+        return seen
+
+    never, ok, gone1, gone2, err = asyncio.run(_go())
+    assert never is None                 # never had Docker: nothing to say
+    assert ok == ("ok", None, t1, t1)
+    assert gone1 == ("unavailable", "no Docker socket at /var/run/docker.sock", t2, t2)
+    assert gone2[2] == t2 and gone2[3] == t3   # still gone: since holds, the report moves on
+    assert err[:3] == ("error", "permission denied on /var/run/docker.sock", t3)
+
+
+@_NEEDS_DB
+def test_a_node_that_is_no_longer_a_manager_loses_its_swarm_rows(app):
+    from sqlalchemy import text
+    from steward.models.hosts import Host
+
+    persist = _persist_fn(app)
+    t1 = datetime(2026, 10, 6, 12, 0, 0, tzinfo=timezone.utc)
+    swarm = {"services": [{"service_name": "web", "mode": "replicated", "desired": 1, "running": 1}],
+             "nodes": [{"node_id": "n1", "hostname": "mgr", "role": "manager"}]}
+
+    async def _go():
+        async with app.db_sessionmaker() as s:
+            async with s.begin():
+                h = Host(id=str(uuid.uuid4()), name=f"demoted-{uuid.uuid4().hex[:6]}", address="10.9.9.4")
+                s.add(h)
+                await s.flush()
+
+                async def counts():
+                    return tuple((await s.execute(text(
+                        f"SELECT count(*) FROM {t} WHERE host_id = :h"), {"h": h.id})).scalar()
+                        for t in ("docker_swarm_services", "docker_swarm_nodes"))
+
+                await persist(s, h, [], swarm, status={"status": "ok", "detail": None, "at": t1})
+                manager = await counts()
+                # /info failed: not knowing is not "no longer a manager".
+                await persist(s, h, [], status={"status": "error", "detail": "x", "at": t1,
+                                                "swarm_manager": None})
+                unknown = await counts()
+                await persist(s, h, [], status={"status": "ok", "detail": None, "at": t1,
+                                                "swarm_manager": False})
+                return manager, unknown, await counts()
+
+    manager, unknown, demoted = asyncio.run(_go())
+    assert manager == (1, 1)
+    assert unknown == (1, 1)
+    assert demoted == (0, 0)
+
+
 @_NEEDS_DB
 def test_swarm_topology_persisted(app):
     from sqlalchemy import text

tests/integration/test_host_agent_docker_status.py

@@ -0,0 +1,79 @@
+"""Integration: an agent's Docker status reaches Steward through ingest (#5152).
+
+The newest sample's status wins over a backlog's older ones; "error" keeps
+the containers Docker last listed; an "ok" with an empty list clears them.
+
+Rows are read with plain SQL: importing the plugin's models again after the
+app loaded the plugin raises "Table already defined" (see test_mcp_plugin_tools)."""
+from __future__ import annotations
+
+import asyncio
+import hashlib
+import os
+import uuid
+from datetime import datetime, timedelta, timezone
+
+import pytest
+
+pytestmark = pytest.mark.integration
+
+
+@pytest.fixture
+def app():
+    if not os.environ.get("STEWARD_DATABASE_URL"):
+        pytest.skip("needs Postgres")
+    from steward.app import create_app
+    return create_app(testing=False)
+
+
+def test_ingest_records_docker_status_and_clears_on_an_empty_answer(app):
+    from sqlalchemy import text
+    from steward.models.hosts import Host
+
+    web = {"name": "web", "status": "running", "cpu_pct": 1.0, "mem_pct": 1.0,
+           "restart_count": 0, "exit_code": None, "oom_killed": False, "health": None}
+
+    async def _go():
+        tag = uuid.uuid4().hex[:8]
+        token = f"tok-{tag}-{uuid.uuid4().hex}"
+        async with app.db_sessionmaker() as s:
+            async with s.begin():
+                host = Host(name=f"dock-{tag}", address="10.9.9.3")
+                s.add(host)
+                await s.flush()
+                await s.execute(text(
+                    "INSERT INTO host_agent_registrations "
+                    "(id, host_id, token_hash, token_created_at, created_at, updated_at) "
+                    "VALUES (:id, :host, :hash, now(), now(), now())"),
+                    {"id": str(uuid.uuid4()), "host": host.id,
+                     "hash": hashlib.sha256(token.encode()).hexdigest()})
+            host_id = host.id
+
+        client = app.test_client()
+        now = datetime.now(timezone.utc)
+
+        async def post(*samples):
+            resp = await client.post("/plugins/host_agent/ingest", json={"samples": list(samples)},
+                                     headers={"Authorization": f"Bearer {token}"})
+            assert resp.status_code == 200, await resp.get_data(as_text=True)
+            async with app.db_sessionmaker() as s:
+                status = (await s.execute(text(
+                    "SELECT status, detail FROM docker_host_status WHERE host_id = :h"),
+                    {"h": host_id})).first()
+                names = [r[0] for r in (await s.execute(text(
+                    "SELECT name FROM docker_containers WHERE host_id = :h"), {"h": host_id})).all()]
+            return (tuple(status) if status else None), names
+
+        def at(seconds_ago):
+            return (now - timedelta(seconds=seconds_ago)).isoformat()
+
+        # A backlog, newest first: Docker listed web, then stopped answering.
+        backlog = await post(
+            {"ts": at(60), "docker_status": "error", "docker_detail": "permission denied on /s"},
+            {"ts": at(90), "docker_status": "ok", "docker": [web]})
+        emptied = await post({"ts": at(30), "docker_status": "ok", "docker": []})
+        return backlog, emptied
+
+    backlog, emptied = asyncio.run(_go())
+    assert backlog == (("error", "permission denied on /s"), ["web"])
+    assert emptied == (("ok", None), [])

tests/plugins/docker/test_docker_page.py

@@ -33,6 +33,9 @@ def test_stale_is_older_than_the_agents_window():
     assert not _is_stale(_c("a"), NOW, 180)
     assert _is_stale(_c("a", scraped_at=NOW - timedelta(minutes=10)), NOW, 180)
     assert _is_stale(_c("a", scraped_at=None), NOW, 180)
+    # A fresh row on a host whose Docker stopped answering is stale too (#5152).
+    assert _is_stale(_c("a"), NOW, 180, {"h1": object()})
+    assert not _is_stale(_c("a"), NOW, 180, {"h2": object()})
     assert svc.stale_after_seconds({"PLUGINS": {"host_agent": {"stale_after_seconds": 60}}}) == 60
     assert svc.stale_after_seconds({}) == 180
 
@@ -74,3 +77,29 @@ async def test_a_healthy_board_has_no_attention_card(app):
             app, "docker", "docker/rows.html", attention=[], says=svc.PROBLEM_SAYS, severity=svc.PROBLEM_SEVERITY,
             unhealthy=0, restarting=0, stale=0, host_groups=[], swarm_services=[], running=0, stopped=0, total=0)
     assert "Needs attention</h2>" not in page and "No containers reported yet" in page
+
+
+async def test_a_host_whose_docker_stopped_answering_says_so(app):
+    down = [{"host_id": "h1", "host_name": "nas", "status": "unavailable",
+             "detail": "no Docker socket at /var/run/docker.sock", "since": NOW - timedelta(minutes=5)},
+            {"host_id": "h2", "host_name": "web-1", "status": "error",
+             "detail": "permission denied on /var/run/docker.sock", "since": NOW - timedelta(minutes=1)}]
+    async with app.test_request_context("/plugins/docker/rows"):
+        page = await render_plugin_template(
+            app, "docker", "docker/rows.html", docker_down=down, attention=[], says=svc.PROBLEM_SAYS,
+            severity=svc.PROBLEM_SEVERITY, unhealthy=0, restarting=0, stale=0, host_groups=[],
+            swarm_services=[], running=0, stopped=0, total=0)
+    page = re.sub(r">\s+<", "><", " ".join(page.split()))
+    card = page[page.index("Docker not answering"):page.index("</section>")]
+    assert '<span class="badge badge-warn">No Docker</span><a href="/hosts/h1">nas</a>' in card
+    assert "no Docker socket at /var/run/docker.sock · since <time" in card and ">5m ago</time>" in card
+    assert '<span class="badge badge-crit">Error</span><a href="/hosts/h2">web-1</a>' in card
+
+
+async def test_no_card_while_every_docker_answers(app):
+    async with app.test_request_context("/plugins/docker/rows"):
+        page = await render_plugin_template(
+            app, "docker", "docker/rows.html", docker_down=[], attention=[], says=svc.PROBLEM_SAYS,
+            severity=svc.PROBLEM_SEVERITY, unhealthy=0, restarting=0, stale=0, host_groups=[],
+            swarm_services=[], running=0, stopped=0, total=0)
+    assert "Docker not answering" not in page

tests/plugins/host_agent/test_agent_docker.py

@@ -2,9 +2,13 @@
 """Unit tests for the agent's stdlib Docker collector.
 
 No socket / no network — the UDS request layer is monkeypatched so we exercise
-the cpu/mem math, chunked-body decoding, sample assembly, and the silent-skip
-contract in isolation.
+the cpu/mem math, chunked-body decoding, sample assembly, and what the agent
+says when Docker cannot be asked, in isolation.
 """
+import socket
+
+import pytest
+
 from plugins.host_agent import agent as a
 
 
@@ -111,9 +115,31 @@ def test_docker_net_io_sums_counters():
 
 # ── collect_docker assembly + silent skip ─────────────────────────────────────
 
-def test_collect_docker_silent_skip_when_no_socket():
-    # Absent socket path must degrade to [] (zero-config on Docker-less hosts).
-    assert a.collect_docker("/nonexistent/does-not-exist.sock") == []
+def test_collect_docker_says_unavailable_when_no_socket():
+    # A Docker-less host needs no configuration: it reports there is no Docker.
+    assert a.collect_docker("/nonexistent/does-not-exist.sock") == (
+        "unavailable", "no Docker socket at /nonexistent/does-not-exist.sock", [])
+
+
+@pytest.mark.parametrize("exc, status, detail", [
+    (FileNotFoundError(2, "x"), "unavailable", "no Docker socket at /s"),
+    (ConnectionRefusedError(111, "x"), "unavailable", "Docker is not accepting connections on /s"),
+    (PermissionError(13, "x"), "error", "permission denied on /s"),
+    (socket.timeout("timed out"), "error", "Docker did not answer within 5 s"),
+    (OSError("docker API /containers/json → 500"), "error", "docker API /containers/json → 500"),
+    (ValueError("Expecting value"), "error", "Expecting value"),
+])
+def test_collect_docker_says_why_it_could_not_list(monkeypatch, exc, status, detail):
+    def fail(socket_path, path, timeout=a.DOCKER_API_TIMEOUT):
+        raise exc
+
+    monkeypatch.setattr(a, "_docker_request", fail)
+    assert a.collect_docker("/s") == (status, detail, [])
+
+
+def test_collect_docker_with_no_containers_is_ok_and_empty(monkeypatch):
+    monkeypatch.setattr(a, "_docker_request", lambda s, p, timeout=a.DOCKER_API_TIMEOUT: [])
+    assert a.collect_docker("/s") == ("ok", None, [])
 
 
 def test_collect_docker_assembles_container(monkeypatch):
@@ -151,7 +177,8 @@ def test_collect_docker_assembles_container(monkeypatch):
         raise AssertionError(f"unexpected path {path}")
 
     monkeypatch.setattr(a, "_docker_request", fake_request)
-    out = a.collect_docker("/var/run/docker.sock")
+    status, detail, out = a.collect_docker("/var/run/docker.sock")
+    assert (status, detail) == ("ok", None)
     assert len(out) == 1
     c = out[0]
     assert c["name"] == "web"
@@ -184,7 +211,7 @@ def test_collect_docker_skips_stats_for_stopped(monkeypatch):
         raise AssertionError("stats must not be fetched for a stopped container")
 
     monkeypatch.setattr(a, "_docker_request", fake_request)
-    out = a.collect_docker("/var/run/docker.sock")
+    _, _, out = a.collect_docker("/var/run/docker.sock")
     assert out[0]["status"] == "exited"
     assert out[0]["cpu_pct"] is None and out[0]["started_at"] is None
     # inspect still gives us why it died
@@ -289,7 +316,7 @@ def test_collect_swarm_skips_on_worker(monkeypatch):
         raise AssertionError("manager-only endpoint queried on a worker")
 
     monkeypatch.setattr(a, "_docker_request", fake_request)
-    assert a.collect_swarm("/var/run/docker.sock") is None
+    assert a.collect_swarm("/var/run/docker.sock") == (False, None)
 
 
 def test_collect_swarm_assembles_on_manager(monkeypatch):
@@ -309,39 +336,85 @@ def test_collect_swarm_assembles_on_manager(monkeypatch):
         }[path]
 
     monkeypatch.setattr(a, "_docker_request", fake_request)
-    out = a.collect_swarm("/var/run/docker.sock")
+    is_manager, out = a.collect_swarm("/var/run/docker.sock")
+    assert is_manager is True
     assert out["services"][0]["service_name"] == "web"
     assert out["services"][0]["running"] == 1 and out["services"][0]["desired"] == 1
     assert out["nodes"][0]["hostname"] == "mgr" and out["nodes"][0]["leader"] is True
 
 
-def test_collect_swarm_silent_skip_when_no_socket():
-    assert a.collect_swarm("/nonexistent/does-not-exist.sock") is None
+def test_collect_swarm_cannot_say_when_info_fails():
+    # Not knowing is not "no longer a manager": Steward must keep the rows.
+    assert a.collect_swarm("/nonexistent/does-not-exist.sock") == (None, None)
+
+
+def test_collect_swarm_manager_whose_listing_fails_sends_nothing(monkeypatch):
+    def fake_request(socket_path, path, timeout=a.DOCKER_API_TIMEOUT):
+        if path == "/info":
+            return {"Swarm": {"ControlAvailable": True}}
+        raise OSError("docker API /services → 503")
+
+    monkeypatch.setattr(a, "_docker_request", fake_request)
+    assert a.collect_swarm("/var/run/docker.sock") == (True, None)
 
 
 # ── build_sample wiring ───────────────────────────────────────────────────────
 
 def test_build_sample_includes_docker_when_present(monkeypatch):
-    monkeypatch.setattr(a, "collect_docker", lambda _s: [{"name": "web", "status": "running"}])
-    monkeypatch.setattr(a, "collect_swarm", lambda _s: None)
+    monkeypatch.setattr(a, "collect_docker",
+                        lambda _s: ("ok", None, [{"name": "web", "status": "running"}]))
+    monkeypatch.setattr(a, "collect_swarm", lambda _s: (None, None))
     monkeypatch.setattr(a, "collect_disk_usage", lambda _s: None)
     sample = a.build_sample(["/"], {}, "/var/run/docker.sock")
     assert sample["docker"] == [{"name": "web", "status": "running"}]
+    assert sample["docker_status"] == "ok" and "docker_detail" not in sample
 
 
-def test_build_sample_omits_docker_when_empty(monkeypatch):
-    monkeypatch.setattr(a, "collect_docker", lambda _s: [])
-    monkeypatch.setattr(a, "collect_swarm", lambda _s: None)
+def test_build_sample_sends_an_empty_list_when_docker_has_no_containers(monkeypatch):
+    # "No containers" must reach Steward, so it can clear the host's list (#5152).
+    monkeypatch.setattr(a, "collect_docker", lambda _s: ("ok", None, []))
+    monkeypatch.setattr(a, "collect_swarm", lambda _s: (None, None))
+    monkeypatch.setattr(a, "collect_disk_usage", lambda _s: None)
     sample = a.build_sample(["/"], {}, "/var/run/docker.sock")
-    assert "docker" not in sample
+    assert sample["docker"] == [] and sample["docker_status"] == "ok"
+
+
+def test_build_sample_says_why_docker_could_not_be_asked(monkeypatch):
+    monkeypatch.setattr(a, "collect_docker",
+                        lambda _s: ("error", "permission denied on /var/run/docker.sock", []))
+    monkeypatch.setattr(a, "collect_swarm", lambda _s: (None, None))
+
+    def no_df(_s):
+        raise AssertionError("/system/df must not be asked when Docker did not answer")
+
+    monkeypatch.setattr(a, "collect_disk_usage", no_df)
+    sample = a.build_sample(["/"], {}, "/var/run/docker.sock")
+    assert "docker" not in sample   # not "no containers": unknown
+    assert sample["docker_status"] == "error"
+    assert sample["docker_detail"] == "permission denied on /var/run/docker.sock"
 
 
 def test_build_sample_includes_swarm_on_manager(monkeypatch):
-    monkeypatch.setattr(a, "collect_docker", lambda _s: [])
+    monkeypatch.setattr(a, "collect_docker", lambda _s: ("ok", None, []))
     monkeypatch.setattr(a, "collect_swarm",
-                        lambda _s: {"services": [], "nodes": [{"node_id": "n1"}]})
+                        lambda _s: (True, {"services": [], "nodes": [{"node_id": "n1"}]}))
+    monkeypatch.setattr(a, "collect_disk_usage", lambda _s: None)
     sample = a.build_sample(["/"], {}, "/var/run/docker.sock")
     assert sample["swarm"]["nodes"] == [{"node_id": "n1"}]
+    assert "swarm_manager" not in sample
+
+
+@pytest.mark.parametrize("swarm, flag", [
+    ((False, None), False),   # /info says not a manager: say so
+    ((None, None), None),     # /info failed: say nothing
+    ((True, None), None),     # a manager whose listing failed: say nothing
+])
+def test_build_sample_says_when_a_node_is_not_a_manager(monkeypatch, swarm, flag):
+    monkeypatch.setattr(a, "collect_docker", lambda _s: ("ok", None, []))
+    monkeypatch.setattr(a, "collect_swarm", lambda _s: swarm)
+    monkeypatch.setattr(a, "collect_disk_usage", lambda _s: None)
+    sample = a.build_sample(["/"], {}, "/var/run/docker.sock")
+    assert sample.get("swarm_manager") is flag and "swarm" not in sample
 
 
 def test_build_sample_omits_swarm_off_manager(monkeypatch):