feat(docker): Docker's own status reaches Steward — an empty answer clears a host, a lost one greys it, a demoted manager's swarm rows go (#5152)
CI / lint (push) Successful in 2s
CI / integration (push) Failing after 53s
CI / unit (push) Failing after 1m17s
CI / publish (push) Skipped

The agent (1.9.0) said nothing when Docker could not be asked and nothing
when Docker had no containers, so Steward could tell neither apart from the
other, and kept a host's last containers on screen as running either way.

- Agent: collect_docker returns (status, detail, containers). "ok" sends
  docker: [] when there are none; "unavailable" (no socket, nothing
  listening) and "error" (permission, timeout, bad answer) say why.
  collect_swarm says whether /info answered, and a node that is not a
  manager sends swarm_manager: false. A failed /info says nothing.
- Ingest: an empty docker list is a snapshot; the newest sample's status
  goes to the Docker plugin with the rest.
- docker_host_status (docker_011): per host, status, detail, since and
  last report. A host that never had Docker gets no row.
- An "ok" empty snapshot reaps the host's containers; an error keeps them.
  swarm_manager: false drops the host's swarm services and nodes.
- Docker page: a "Docker not answering" card names each such host, why,
  and since when; its containers read as stale. Duplicate header comment
  in rows.html removed.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
2026-10-06 18:41:22 -04:00
co-authored by Claude Opus 5.5
parent f802c9f153
commit 8ceb983183
11 changed files with 510 additions and 65 deletions
+16 -7
View File
@@ -18,7 +18,7 @@ from . import service as docker_svc
from .dedup import dedup_by_container_id
from .swarm_view import build_swarm_services, parse_tasks, task_summary
from .models import (
DockerContainer, DockerDiskUsage, DockerEvent, DockerImage,
DockerContainer, DockerDiskUsage, DockerEvent, DockerHostStatus, DockerImage,
DockerSwarmNode, DockerSwarmService,
)
@@ -87,6 +87,9 @@ async def rows():
stale_after = docker_svc.stale_after_seconds(current_app.config)
async with current_app.db_sessionmaker() as db:
# Hosts whose Docker is not answering: their rows are what it last said.
docker_down = {d.host_id: d for d in (await db.execute(
select(DockerHostStatus).where(DockerHostStatus.status != "ok"))).scalars()}
ranked = await docker_svc.ranked_containers(
db, now, docker_svc.crash_loop_deaths(current_app.config))
containers = await docker_svc.list_containers(db)
@@ -95,7 +98,7 @@ async def rows():
hosts = await docker_svc.host_map(
db,
{c.host_id for c in containers}
| {n.host_id for n in nodes} | {s.host_id for s in services},
| {n.host_id for n in nodes} | {s.host_id for s in services} | set(docker_down),
)
# Swarm tasks collapse into per-service rows (each replica labelled with
@@ -123,7 +126,7 @@ async def rows():
"container": c,
"ports": json.loads(c.ports_json) if c.ports_json else [],
"uptime": _uptime(c.started_at) if c.status == "running" else None,
"stale": _is_stale(c, now, stale_after),
"stale": _is_stale(c, now, stale_after, docker_down),
})
if c.status == "running":
g["running"] += 1
@@ -151,12 +154,16 @@ async def rows():
swarm_services = swarm_view["services"]
standalone_running = sum(g["running"] for g in host_groups)
ghost_total = sum(r["count"] for s in swarm_services for r in s["replicas"] if r["ghost"])
attention = [dict(r, stale=_is_stale(r["container"], now, stale_after))
attention = [dict(r, stale=_is_stale(r["container"], now, stale_after, docker_down))
for r in docker_svc.needs_attention(ranked)]
return await render_template(
"docker/rows.html",
host_groups=host_groups,
swarm_services=swarm_services,
docker_down=sorted(
({"host_id": h, "host_name": hosts[h].name if h in hosts else h, "status": d.status,
"detail": d.detail, "since": d.since} for h, d in docker_down.items()),
key=lambda d: d["host_name"].lower()),
attention=attention, says=docker_svc.PROBLEM_SAYS, severity=docker_svc.PROBLEM_SEVERITY,
unhealthy=sum("unhealthy" in r["problems"] for r in attention),
restarting=sum(bool({"restarting", "crash_looping"} & set(r["problems"])) for r in attention),
@@ -167,10 +174,12 @@ async def rows():
)
def _is_stale(c, now: datetime, stale_after: int) -> bool:
def _is_stale(c, now: datetime, stale_after: int, docker_down=()) -> bool:
"""A container whose last report is older than the agent's staleness
window: what the page shows may no longer be true."""
return c.scraped_at is None or (now - c.scraped_at).total_seconds() > stale_after
window, or whose host's Docker is not answering: what the page shows may
no longer be true."""
return (c.host_id in docker_down or c.scraped_at is None
or (now - c.scraped_at).total_seconds() > stale_after)
@docker_bp.get("/widget")