feat(docker): Docker's own status reaches Steward — an empty answer clears a host, a lost one greys it, a demoted manager's swarm rows go (#5152)
The agent (1.9.0) said nothing when Docker could not be asked and nothing when Docker had no containers, so Steward could tell neither apart from the other, and kept a host's last containers on screen as running either way. - Agent: collect_docker returns (status, detail, containers). "ok" sends docker: [] when there are none; "unavailable" (no socket, nothing listening) and "error" (permission, timeout, bad answer) say why. collect_swarm says whether /info answered, and a node that is not a manager sends swarm_manager: false. A failed /info says nothing. - Ingest: an empty docker list is a snapshot; the newest sample's status goes to the Docker plugin with the rest. - docker_host_status (docker_011): per host, status, detail, since and last report. A host that never had Docker gets no row. - An "ok" empty snapshot reaps the host's containers; an error keeps them. swarm_manager: false drops the host's swarm services and nodes. - Docker page: a "Docker not answering" card names each such host, why, and since when; its containers read as stale. Duplicate header comment in rows.html removed. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -10,7 +10,7 @@ import asyncio
|
||||
import json
|
||||
import os
|
||||
import uuid
|
||||
from datetime import datetime, timezone
|
||||
from datetime import datetime, timedelta, timezone
|
||||
|
||||
import pytest
|
||||
|
||||
@@ -383,6 +383,119 @@ def test_vanished_container_is_reaped(app):
|
||||
assert after == ["keep"] # the vanished container was reaped
|
||||
|
||||
|
||||
@_NEEDS_DB
|
||||
def test_an_empty_snapshot_from_a_docker_that_answered_clears_the_host(app):
|
||||
"""Docker answering with no containers removes the host's rows (#5152); a
|
||||
sample with no Docker answer at all removes nothing."""
|
||||
from sqlalchemy import text
|
||||
from steward.models.hosts import Host
|
||||
|
||||
persist = _persist_fn(app)
|
||||
t1 = datetime(2026, 10, 6, 12, 0, 0, tzinfo=timezone.utc)
|
||||
t2 = datetime(2026, 10, 6, 12, 0, 30, tzinfo=timezone.utc)
|
||||
web = {"name": "web", "status": "running", "cpu_pct": 1.0, "mem_pct": 1.0,
|
||||
"restart_count": 0, "exit_code": None, "oom_killed": False, "health": None}
|
||||
|
||||
async def _go():
|
||||
async with app.db_sessionmaker() as s:
|
||||
async with s.begin():
|
||||
h = Host(id=str(uuid.uuid4()), name=f"empty-{uuid.uuid4().hex[:6]}", address="10.9.9.7")
|
||||
s.add(h)
|
||||
await s.flush()
|
||||
|
||||
async def names():
|
||||
return [r[0] for r in (await s.execute(text(
|
||||
"SELECT name FROM docker_containers WHERE host_id = :h"), {"h": h.id})).all()]
|
||||
|
||||
await persist(s, h, [(t1, [web])], status={"status": "ok", "detail": None, "at": t1})
|
||||
first = await names()
|
||||
await persist(s, h, [], status={"status": "error", "detail": "timed out", "at": t2})
|
||||
after_error = await names()
|
||||
await persist(s, h, [(t2, [])], status={"status": "ok", "detail": None, "at": t2})
|
||||
return first, after_error, await names()
|
||||
|
||||
first, after_error, after_empty = asyncio.run(_go())
|
||||
assert first == ["web"]
|
||||
assert after_error == ["web"] # couldn't ask: keep what Docker last said
|
||||
assert after_empty == [] # asked, and there are none
|
||||
|
||||
|
||||
@_NEEDS_DB
|
||||
def test_docker_status_is_recorded_once_a_host_has_had_docker(app):
|
||||
from steward.models.hosts import Host
|
||||
|
||||
persist = _persist_fn(app)
|
||||
t1 = datetime(2026, 10, 6, 12, 0, 0, tzinfo=timezone.utc)
|
||||
t2 = t1 + timedelta(seconds=30)
|
||||
t3 = t1 + timedelta(seconds=60)
|
||||
|
||||
async def _go():
|
||||
from plugins.docker.models import DockerHostStatus
|
||||
seen = []
|
||||
async with app.db_sessionmaker() as s:
|
||||
async with s.begin():
|
||||
never = Host(id=str(uuid.uuid4()), name=f"plain-{uuid.uuid4().hex[:6]}", address="10.9.9.5")
|
||||
had = Host(id=str(uuid.uuid4()), name=f"had-{uuid.uuid4().hex[:6]}", address="10.9.9.6")
|
||||
s.add_all([never, had])
|
||||
await s.flush()
|
||||
gone = {"status": "unavailable", "detail": "no Docker socket at /var/run/docker.sock"}
|
||||
await persist(s, never, [], status=dict(gone, at=t1))
|
||||
seen.append(await s.get(DockerHostStatus, never.id))
|
||||
for status, at in (({"status": "ok", "detail": None}, t1),
|
||||
(gone, t2), (gone, t3),
|
||||
({"status": "error", "detail": "permission denied on /var/run/docker.sock"}, t3)):
|
||||
await persist(s, had, [], status=dict(status, at=at))
|
||||
await s.flush()
|
||||
row = await s.get(DockerHostStatus, had.id)
|
||||
seen.append((row.status, row.detail, row.since, row.reported_at))
|
||||
return seen
|
||||
|
||||
never, ok, gone1, gone2, err = asyncio.run(_go())
|
||||
assert never is None # never had Docker: nothing to say
|
||||
assert ok == ("ok", None, t1, t1)
|
||||
assert gone1 == ("unavailable", "no Docker socket at /var/run/docker.sock", t2, t2)
|
||||
assert gone2[2] == t2 and gone2[3] == t3 # still gone: since holds, the report moves on
|
||||
assert err[:3] == ("error", "permission denied on /var/run/docker.sock", t3)
|
||||
|
||||
|
||||
@_NEEDS_DB
|
||||
def test_a_node_that_is_no_longer_a_manager_loses_its_swarm_rows(app):
|
||||
from sqlalchemy import text
|
||||
from steward.models.hosts import Host
|
||||
|
||||
persist = _persist_fn(app)
|
||||
t1 = datetime(2026, 10, 6, 12, 0, 0, tzinfo=timezone.utc)
|
||||
swarm = {"services": [{"service_name": "web", "mode": "replicated", "desired": 1, "running": 1}],
|
||||
"nodes": [{"node_id": "n1", "hostname": "mgr", "role": "manager"}]}
|
||||
|
||||
async def _go():
|
||||
async with app.db_sessionmaker() as s:
|
||||
async with s.begin():
|
||||
h = Host(id=str(uuid.uuid4()), name=f"demoted-{uuid.uuid4().hex[:6]}", address="10.9.9.4")
|
||||
s.add(h)
|
||||
await s.flush()
|
||||
|
||||
async def counts():
|
||||
return tuple((await s.execute(text(
|
||||
f"SELECT count(*) FROM {t} WHERE host_id = :h"), {"h": h.id})).scalar()
|
||||
for t in ("docker_swarm_services", "docker_swarm_nodes"))
|
||||
|
||||
await persist(s, h, [], swarm, status={"status": "ok", "detail": None, "at": t1})
|
||||
manager = await counts()
|
||||
# /info failed: not knowing is not "no longer a manager".
|
||||
await persist(s, h, [], status={"status": "error", "detail": "x", "at": t1,
|
||||
"swarm_manager": None})
|
||||
unknown = await counts()
|
||||
await persist(s, h, [], status={"status": "ok", "detail": None, "at": t1,
|
||||
"swarm_manager": False})
|
||||
return manager, unknown, await counts()
|
||||
|
||||
manager, unknown, demoted = asyncio.run(_go())
|
||||
assert manager == (1, 1)
|
||||
assert unknown == (1, 1)
|
||||
assert demoted == (0, 0)
|
||||
|
||||
|
||||
@_NEEDS_DB
|
||||
def test_swarm_topology_persisted(app):
|
||||
from sqlalchemy import text
|
||||
|
||||
Reference in New Issue
Block a user