feat(docker): Docker's own status reaches Steward — an empty answer clears a host, a lost one greys it, a demoted manager's swarm rows go (#5152)
The agent (1.9.0) said nothing when Docker could not be asked and nothing when Docker had no containers, so Steward could tell neither apart from the other, and kept a host's last containers on screen as running either way. - Agent: collect_docker returns (status, detail, containers). "ok" sends docker: [] when there are none; "unavailable" (no socket, nothing listening) and "error" (permission, timeout, bad answer) say why. collect_swarm says whether /info answered, and a node that is not a manager sends swarm_manager: false. A failed /info says nothing. - Ingest: an empty docker list is a snapshot; the newest sample's status goes to the Docker plugin with the rest. - docker_host_status (docker_011): per host, status, detail, since and last report. A host that never had Docker gets no row. - An "ok" empty snapshot reaps the host's containers; an error keeps them. swarm_manager: false drops the host's swarm services and nodes. - Docker page: a "Docker not answering" card names each such host, why, and since when; its containers read as stale. Duplicate header comment in rows.html removed. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -18,7 +18,7 @@ from . import service as docker_svc
|
||||
from .dedup import dedup_by_container_id
|
||||
from .swarm_view import build_swarm_services, parse_tasks, task_summary
|
||||
from .models import (
|
||||
DockerContainer, DockerDiskUsage, DockerEvent, DockerImage,
|
||||
DockerContainer, DockerDiskUsage, DockerEvent, DockerHostStatus, DockerImage,
|
||||
DockerSwarmNode, DockerSwarmService,
|
||||
)
|
||||
|
||||
@@ -87,6 +87,9 @@ async def rows():
|
||||
stale_after = docker_svc.stale_after_seconds(current_app.config)
|
||||
|
||||
async with current_app.db_sessionmaker() as db:
|
||||
# Hosts whose Docker is not answering: their rows are what it last said.
|
||||
docker_down = {d.host_id: d for d in (await db.execute(
|
||||
select(DockerHostStatus).where(DockerHostStatus.status != "ok"))).scalars()}
|
||||
ranked = await docker_svc.ranked_containers(
|
||||
db, now, docker_svc.crash_loop_deaths(current_app.config))
|
||||
containers = await docker_svc.list_containers(db)
|
||||
@@ -95,7 +98,7 @@ async def rows():
|
||||
hosts = await docker_svc.host_map(
|
||||
db,
|
||||
{c.host_id for c in containers}
|
||||
| {n.host_id for n in nodes} | {s.host_id for s in services},
|
||||
| {n.host_id for n in nodes} | {s.host_id for s in services} | set(docker_down),
|
||||
)
|
||||
|
||||
# Swarm tasks collapse into per-service rows (each replica labelled with
|
||||
@@ -123,7 +126,7 @@ async def rows():
|
||||
"container": c,
|
||||
"ports": json.loads(c.ports_json) if c.ports_json else [],
|
||||
"uptime": _uptime(c.started_at) if c.status == "running" else None,
|
||||
"stale": _is_stale(c, now, stale_after),
|
||||
"stale": _is_stale(c, now, stale_after, docker_down),
|
||||
})
|
||||
if c.status == "running":
|
||||
g["running"] += 1
|
||||
@@ -151,12 +154,16 @@ async def rows():
|
||||
swarm_services = swarm_view["services"]
|
||||
standalone_running = sum(g["running"] for g in host_groups)
|
||||
ghost_total = sum(r["count"] for s in swarm_services for r in s["replicas"] if r["ghost"])
|
||||
attention = [dict(r, stale=_is_stale(r["container"], now, stale_after))
|
||||
attention = [dict(r, stale=_is_stale(r["container"], now, stale_after, docker_down))
|
||||
for r in docker_svc.needs_attention(ranked)]
|
||||
return await render_template(
|
||||
"docker/rows.html",
|
||||
host_groups=host_groups,
|
||||
swarm_services=swarm_services,
|
||||
docker_down=sorted(
|
||||
({"host_id": h, "host_name": hosts[h].name if h in hosts else h, "status": d.status,
|
||||
"detail": d.detail, "since": d.since} for h, d in docker_down.items()),
|
||||
key=lambda d: d["host_name"].lower()),
|
||||
attention=attention, says=docker_svc.PROBLEM_SAYS, severity=docker_svc.PROBLEM_SEVERITY,
|
||||
unhealthy=sum("unhealthy" in r["problems"] for r in attention),
|
||||
restarting=sum(bool({"restarting", "crash_looping"} & set(r["problems"])) for r in attention),
|
||||
@@ -167,10 +174,12 @@ async def rows():
|
||||
)
|
||||
|
||||
|
||||
def _is_stale(c, now: datetime, stale_after: int) -> bool:
|
||||
def _is_stale(c, now: datetime, stale_after: int, docker_down=()) -> bool:
|
||||
"""A container whose last report is older than the agent's staleness
|
||||
window: what the page shows may no longer be true."""
|
||||
return c.scraped_at is None or (now - c.scraped_at).total_seconds() > stale_after
|
||||
window, or whose host's Docker is not answering: what the page shows may
|
||||
no longer be true."""
|
||||
return (c.host_id in docker_down or c.scraped_at is None
|
||||
or (now - c.scraped_at).total_seconds() > stale_after)
|
||||
|
||||
|
||||
@docker_bp.get("/widget")
|
||||
|
||||
Reference in New Issue
Block a user