feat(docker): Docker's own status reaches Steward — an empty answer clears a host, a lost one greys it, a demoted manager's swarm rows go (#5152)

The agent (1.9.0) said nothing when Docker could not be asked and nothing
when Docker had no containers, so Steward could tell neither apart from the
other, and kept a host's last containers on screen as running either way.

- Agent: collect_docker returns (status, detail, containers). "ok" sends
  docker: [] when there are none; "unavailable" (no socket, nothing
  listening) and "error" (permission, timeout, bad answer) say why.
  collect_swarm says whether /info answered, and a node that is not a
  manager sends swarm_manager: false. A failed /info says nothing.
- Ingest: an empty docker list is a snapshot; the newest sample's status
  goes to the Docker plugin with the rest.
- docker_host_status (docker_011): per host, status, detail, since and
  last report. A host that never had Docker gets no row.
- An "ok" empty snapshot reaps the host's containers; an error keeps them.
  swarm_manager: false drops the host's swarm services and nodes.
- Docker page: a "Docker not answering" card names each such host, why,
  and since when; its containers read as stale. Duplicate header comment
  in rows.html removed.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

tests/plugins/host_agent/test_agent_docker.py

@@ -2,9 +2,13 @@
 """Unit tests for the agent's stdlib Docker collector.
 
 No socket / no network — the UDS request layer is monkeypatched so we exercise
-the cpu/mem math, chunked-body decoding, sample assembly, and the silent-skip
-contract in isolation.
+the cpu/mem math, chunked-body decoding, sample assembly, and what the agent
+says when Docker cannot be asked, in isolation.
 """
+import socket
+
+import pytest
+
 from plugins.host_agent import agent as a
 
 
@@ -111,9 +115,31 @@ def test_docker_net_io_sums_counters():
 
 # ── collect_docker assembly + silent skip ─────────────────────────────────────
 
-def test_collect_docker_silent_skip_when_no_socket():
-    # Absent socket path must degrade to [] (zero-config on Docker-less hosts).
-    assert a.collect_docker("/nonexistent/does-not-exist.sock") == []
+def test_collect_docker_says_unavailable_when_no_socket():
+    # A Docker-less host needs no configuration: it reports there is no Docker.
+    assert a.collect_docker("/nonexistent/does-not-exist.sock") == (
+        "unavailable", "no Docker socket at /nonexistent/does-not-exist.sock", [])
+
+
+@pytest.mark.parametrize("exc, status, detail", [
+    (FileNotFoundError(2, "x"), "unavailable", "no Docker socket at /s"),
+    (ConnectionRefusedError(111, "x"), "unavailable", "Docker is not accepting connections on /s"),
+    (PermissionError(13, "x"), "error", "permission denied on /s"),
+    (socket.timeout("timed out"), "error", "Docker did not answer within 5 s"),
+    (OSError("docker API /containers/json → 500"), "error", "docker API /containers/json → 500"),
+    (ValueError("Expecting value"), "error", "Expecting value"),
+])
+def test_collect_docker_says_why_it_could_not_list(monkeypatch, exc, status, detail):
+    def fail(socket_path, path, timeout=a.DOCKER_API_TIMEOUT):
+        raise exc
+
+    monkeypatch.setattr(a, "_docker_request", fail)
+    assert a.collect_docker("/s") == (status, detail, [])
+
+
+def test_collect_docker_with_no_containers_is_ok_and_empty(monkeypatch):
+    monkeypatch.setattr(a, "_docker_request", lambda s, p, timeout=a.DOCKER_API_TIMEOUT: [])
+    assert a.collect_docker("/s") == ("ok", None, [])
 
 
 def test_collect_docker_assembles_container(monkeypatch):
@@ -151,7 +177,8 @@ def test_collect_docker_assembles_container(monkeypatch):
         raise AssertionError(f"unexpected path {path}")
 
     monkeypatch.setattr(a, "_docker_request", fake_request)
-    out = a.collect_docker("/var/run/docker.sock")
+    status, detail, out = a.collect_docker("/var/run/docker.sock")
+    assert (status, detail) == ("ok", None)
     assert len(out) == 1
     c = out[0]
     assert c["name"] == "web"
@@ -184,7 +211,7 @@ def test_collect_docker_skips_stats_for_stopped(monkeypatch):
         raise AssertionError("stats must not be fetched for a stopped container")
 
     monkeypatch.setattr(a, "_docker_request", fake_request)
-    out = a.collect_docker("/var/run/docker.sock")
+    _, _, out = a.collect_docker("/var/run/docker.sock")
     assert out[0]["status"] == "exited"
     assert out[0]["cpu_pct"] is None and out[0]["started_at"] is None
     # inspect still gives us why it died
@@ -289,7 +316,7 @@ def test_collect_swarm_skips_on_worker(monkeypatch):
         raise AssertionError("manager-only endpoint queried on a worker")
 
     monkeypatch.setattr(a, "_docker_request", fake_request)
-    assert a.collect_swarm("/var/run/docker.sock") is None
+    assert a.collect_swarm("/var/run/docker.sock") == (False, None)
 
 
 def test_collect_swarm_assembles_on_manager(monkeypatch):
@@ -309,39 +336,85 @@ def test_collect_swarm_assembles_on_manager(monkeypatch):
         }[path]
 
     monkeypatch.setattr(a, "_docker_request", fake_request)
-    out = a.collect_swarm("/var/run/docker.sock")
+    is_manager, out = a.collect_swarm("/var/run/docker.sock")
+    assert is_manager is True
     assert out["services"][0]["service_name"] == "web"
     assert out["services"][0]["running"] == 1 and out["services"][0]["desired"] == 1
     assert out["nodes"][0]["hostname"] == "mgr" and out["nodes"][0]["leader"] is True
 
 
-def test_collect_swarm_silent_skip_when_no_socket():
-    assert a.collect_swarm("/nonexistent/does-not-exist.sock") is None
+def test_collect_swarm_cannot_say_when_info_fails():
+    # Not knowing is not "no longer a manager": Steward must keep the rows.
+    assert a.collect_swarm("/nonexistent/does-not-exist.sock") == (None, None)
+
+
+def test_collect_swarm_manager_whose_listing_fails_sends_nothing(monkeypatch):
+    def fake_request(socket_path, path, timeout=a.DOCKER_API_TIMEOUT):
+        if path == "/info":
+            return {"Swarm": {"ControlAvailable": True}}
+        raise OSError("docker API /services → 503")
+
+    monkeypatch.setattr(a, "_docker_request", fake_request)
+    assert a.collect_swarm("/var/run/docker.sock") == (True, None)
 
 
 # ── build_sample wiring ───────────────────────────────────────────────────────
 
 def test_build_sample_includes_docker_when_present(monkeypatch):
-    monkeypatch.setattr(a, "collect_docker", lambda _s: [{"name": "web", "status": "running"}])
-    monkeypatch.setattr(a, "collect_swarm", lambda _s: None)
+    monkeypatch.setattr(a, "collect_docker",
+                        lambda _s: ("ok", None, [{"name": "web", "status": "running"}]))
+    monkeypatch.setattr(a, "collect_swarm", lambda _s: (None, None))
     monkeypatch.setattr(a, "collect_disk_usage", lambda _s: None)
     sample = a.build_sample(["/"], {}, "/var/run/docker.sock")
     assert sample["docker"] == [{"name": "web", "status": "running"}]
+    assert sample["docker_status"] == "ok" and "docker_detail" not in sample
 
 
-def test_build_sample_omits_docker_when_empty(monkeypatch):
-    monkeypatch.setattr(a, "collect_docker", lambda _s: [])
-    monkeypatch.setattr(a, "collect_swarm", lambda _s: None)
+def test_build_sample_sends_an_empty_list_when_docker_has_no_containers(monkeypatch):
+    # "No containers" must reach Steward, so it can clear the host's list (#5152).
+    monkeypatch.setattr(a, "collect_docker", lambda _s: ("ok", None, []))
+    monkeypatch.setattr(a, "collect_swarm", lambda _s: (None, None))
+    monkeypatch.setattr(a, "collect_disk_usage", lambda _s: None)
     sample = a.build_sample(["/"], {}, "/var/run/docker.sock")
-    assert "docker" not in sample
+    assert sample["docker"] == [] and sample["docker_status"] == "ok"
+
+
+def test_build_sample_says_why_docker_could_not_be_asked(monkeypatch):
+    monkeypatch.setattr(a, "collect_docker",
+                        lambda _s: ("error", "permission denied on /var/run/docker.sock", []))
+    monkeypatch.setattr(a, "collect_swarm", lambda _s: (None, None))
+
+    def no_df(_s):
+        raise AssertionError("/system/df must not be asked when Docker did not answer")
+
+    monkeypatch.setattr(a, "collect_disk_usage", no_df)
+    sample = a.build_sample(["/"], {}, "/var/run/docker.sock")
+    assert "docker" not in sample   # not "no containers": unknown
+    assert sample["docker_status"] == "error"
+    assert sample["docker_detail"] == "permission denied on /var/run/docker.sock"
 
 
 def test_build_sample_includes_swarm_on_manager(monkeypatch):
-    monkeypatch.setattr(a, "collect_docker", lambda _s: [])
+    monkeypatch.setattr(a, "collect_docker", lambda _s: ("ok", None, []))
     monkeypatch.setattr(a, "collect_swarm",
-                        lambda _s: {"services": [], "nodes": [{"node_id": "n1"}]})
+                        lambda _s: (True, {"services": [], "nodes": [{"node_id": "n1"}]}))
+    monkeypatch.setattr(a, "collect_disk_usage", lambda _s: None)
     sample = a.build_sample(["/"], {}, "/var/run/docker.sock")
     assert sample["swarm"]["nodes"] == [{"node_id": "n1"}]
+    assert "swarm_manager" not in sample
+
+
+@pytest.mark.parametrize("swarm, flag", [
+    ((False, None), False),   # /info says not a manager: say so
+    ((None, None), None),     # /info failed: say nothing
+    ((True, None), None),     # a manager whose listing failed: say nothing
+])
+def test_build_sample_says_when_a_node_is_not_a_manager(monkeypatch, swarm, flag):
+    monkeypatch.setattr(a, "collect_docker", lambda _s: ("ok", None, []))
+    monkeypatch.setattr(a, "collect_swarm", lambda _s: swarm)
+    monkeypatch.setattr(a, "collect_disk_usage", lambda _s: None)
+    sample = a.build_sample(["/"], {}, "/var/run/docker.sock")
+    assert sample.get("swarm_manager") is flag and "swarm" not in sample
 
 
 def test_build_sample_omits_swarm_off_manager(monkeypatch):