feat(docker): Docker's own status reaches Steward — an empty answer clears a host, a lost one greys it, a demoted manager's swarm rows go (#5152)
CI / lint (push) Successful in 2s
CI / integration (push) Failing after 53s
CI / unit (push) Failing after 1m17s
CI / publish (push) Skipped

The agent (1.9.0) said nothing when Docker could not be asked and nothing
when Docker had no containers, so Steward could tell neither apart from the
other, and kept a host's last containers on screen as running either way.

- Agent: collect_docker returns (status, detail, containers). "ok" sends
  docker: [] when there are none; "unavailable" (no socket, nothing
  listening) and "error" (permission, timeout, bad answer) say why.
  collect_swarm says whether /info answered, and a node that is not a
  manager sends swarm_manager: false. A failed /info says nothing.
- Ingest: an empty docker list is a snapshot; the newest sample's status
  goes to the Docker plugin with the rest.
- docker_host_status (docker_011): per host, status, detail, since and
  last report. A host that never had Docker gets no row.
- An "ok" empty snapshot reaps the host's containers; an error keeps them.
  swarm_manager: false drops the host's swarm services and nodes.
- Docker page: a "Docker not answering" card names each such host, why,
  and since when; its containers read as stale. Duplicate header comment
  in rows.html removed.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
2026-10-06 18:41:22 -04:00
co-authored by Claude Opus 5.5
parent f802c9f153
commit 8ceb983183
11 changed files with 510 additions and 65 deletions
+114 -1
View File
@@ -10,7 +10,7 @@ import asyncio
import json
import os
import uuid
from datetime import datetime, timezone
from datetime import datetime, timedelta, timezone
import pytest
@@ -383,6 +383,119 @@ def test_vanished_container_is_reaped(app):
assert after == ["keep"] # the vanished container was reaped
@_NEEDS_DB
def test_an_empty_snapshot_from_a_docker_that_answered_clears_the_host(app):
"""Docker answering with no containers removes the host's rows (#5152); a
sample with no Docker answer at all removes nothing."""
from sqlalchemy import text
from steward.models.hosts import Host
persist = _persist_fn(app)
t1 = datetime(2026, 10, 6, 12, 0, 0, tzinfo=timezone.utc)
t2 = datetime(2026, 10, 6, 12, 0, 30, tzinfo=timezone.utc)
web = {"name": "web", "status": "running", "cpu_pct": 1.0, "mem_pct": 1.0,
"restart_count": 0, "exit_code": None, "oom_killed": False, "health": None}
async def _go():
async with app.db_sessionmaker() as s:
async with s.begin():
h = Host(id=str(uuid.uuid4()), name=f"empty-{uuid.uuid4().hex[:6]}", address="10.9.9.7")
s.add(h)
await s.flush()
async def names():
return [r[0] for r in (await s.execute(text(
"SELECT name FROM docker_containers WHERE host_id = :h"), {"h": h.id})).all()]
await persist(s, h, [(t1, [web])], status={"status": "ok", "detail": None, "at": t1})
first = await names()
await persist(s, h, [], status={"status": "error", "detail": "timed out", "at": t2})
after_error = await names()
await persist(s, h, [(t2, [])], status={"status": "ok", "detail": None, "at": t2})
return first, after_error, await names()
first, after_error, after_empty = asyncio.run(_go())
assert first == ["web"]
assert after_error == ["web"] # couldn't ask: keep what Docker last said
assert after_empty == [] # asked, and there are none
@_NEEDS_DB
def test_docker_status_is_recorded_once_a_host_has_had_docker(app):
from steward.models.hosts import Host
persist = _persist_fn(app)
t1 = datetime(2026, 10, 6, 12, 0, 0, tzinfo=timezone.utc)
t2 = t1 + timedelta(seconds=30)
t3 = t1 + timedelta(seconds=60)
async def _go():
from plugins.docker.models import DockerHostStatus
seen = []
async with app.db_sessionmaker() as s:
async with s.begin():
never = Host(id=str(uuid.uuid4()), name=f"plain-{uuid.uuid4().hex[:6]}", address="10.9.9.5")
had = Host(id=str(uuid.uuid4()), name=f"had-{uuid.uuid4().hex[:6]}", address="10.9.9.6")
s.add_all([never, had])
await s.flush()
gone = {"status": "unavailable", "detail": "no Docker socket at /var/run/docker.sock"}
await persist(s, never, [], status=dict(gone, at=t1))
seen.append(await s.get(DockerHostStatus, never.id))
for status, at in (({"status": "ok", "detail": None}, t1),
(gone, t2), (gone, t3),
({"status": "error", "detail": "permission denied on /var/run/docker.sock"}, t3)):
await persist(s, had, [], status=dict(status, at=at))
await s.flush()
row = await s.get(DockerHostStatus, had.id)
seen.append((row.status, row.detail, row.since, row.reported_at))
return seen
never, ok, gone1, gone2, err = asyncio.run(_go())
assert never is None # never had Docker: nothing to say
assert ok == ("ok", None, t1, t1)
assert gone1 == ("unavailable", "no Docker socket at /var/run/docker.sock", t2, t2)
assert gone2[2] == t2 and gone2[3] == t3 # still gone: since holds, the report moves on
assert err[:3] == ("error", "permission denied on /var/run/docker.sock", t3)
@_NEEDS_DB
def test_a_node_that_is_no_longer_a_manager_loses_its_swarm_rows(app):
from sqlalchemy import text
from steward.models.hosts import Host
persist = _persist_fn(app)
t1 = datetime(2026, 10, 6, 12, 0, 0, tzinfo=timezone.utc)
swarm = {"services": [{"service_name": "web", "mode": "replicated", "desired": 1, "running": 1}],
"nodes": [{"node_id": "n1", "hostname": "mgr", "role": "manager"}]}
async def _go():
async with app.db_sessionmaker() as s:
async with s.begin():
h = Host(id=str(uuid.uuid4()), name=f"demoted-{uuid.uuid4().hex[:6]}", address="10.9.9.4")
s.add(h)
await s.flush()
async def counts():
return tuple((await s.execute(text(
f"SELECT count(*) FROM {t} WHERE host_id = :h"), {"h": h.id})).scalar()
for t in ("docker_swarm_services", "docker_swarm_nodes"))
await persist(s, h, [], swarm, status={"status": "ok", "detail": None, "at": t1})
manager = await counts()
# /info failed: not knowing is not "no longer a manager".
await persist(s, h, [], status={"status": "error", "detail": "x", "at": t1,
"swarm_manager": None})
unknown = await counts()
await persist(s, h, [], status={"status": "ok", "detail": None, "at": t1,
"swarm_manager": False})
return manager, unknown, await counts()
manager, unknown, demoted = asyncio.run(_go())
assert manager == (1, 1)
assert unknown == (1, 1)
assert demoted == (0, 0)
@_NEEDS_DB
def test_swarm_topology_persisted(app):
from sqlalchemy import text