diff --git a/docs/plugins/host-agent-design.md b/docs/plugins/host-agent-design.md index 01dcabf..760c180 100644 --- a/docs/plugins/host-agent-design.md +++ b/docs/plugins/host-agent-design.md @@ -252,6 +252,20 @@ Maintenance** (`/plugins/host_agent/maintenance`). last week, then each plan with its schedule, next slot and last five runs. Each host's Agent panel shows its last maintenance job. +### The swarm's health, alertable (1.19.0, #5788) + +Every manager's agent reports the whole swarm. From each report the docker plugin (`plugins/docker/swarm_health.py`) records three swarm-wide figures on resource `swarm`, source `docker`: + +- `swarm_services_short`: services running fewer tasks than they want. A service scaled to 0 is never short. +- `swarm_nodes_down`: nodes not `ready`. +- `swarm_managers_unreachable`: managers the swarm reports as not `reachable`. Agent 1.19.0 sends `ManagerStatus.Reachability`; before that, a manager whose status isn't `ready` counts. + +Drained nodes are left out of the last two. A node is drained on purpose (VDNT-Util01 always, any node during its maintenance), so its going away is expected, and its host's own "Agent offline" rule still covers it. + +The figures are recorded at most once every 30 s, whichever manager sent them, so three managers make one series and losing one doesn't silence it. + +A maintenance job on a swarm node opens a second window over `docker` / `swarm` beside its host's (`maintenance._swarm_window`). An alert the drain or reboot causes then waits for the job, and one still firing when the window ends is sent then. + ### What happened to the host (1.14.0, #5756) A post may carry `events`, said once each, with the next post that gets through: diff --git a/plugins/docker/__init__.py b/plugins/docker/__init__.py index 0145282..367cdb0 100644 --- a/plugins/docker/__init__.py +++ b/plugins/docker/__init__.py @@ -83,6 +83,10 @@ def get_alert_metrics() -> dict[str, list[str]]: """What ingest.py records per container (resource "/"): restart_count is cumulative (alert on crash-looping); is_healthy is 1.0 / 0.0 from the container's HEALTHCHECK, only for containers that define one. - Per host (resource ""), what Docker keeps on its disk (#5304).""" + Per host (resource ""), what Docker keeps on its disk (#5304). + Swarm-wide (resource "swarm"), from any manager's report: services short + of their replicas, nodes down and managers unreachable, drained nodes + aside (#5788, swarm_health.py).""" from .disk_view import DISK_METRICS - return {"docker": ["cpu_pct", "mem_pct", "restart_count", "is_healthy", *DISK_METRICS]} + from .swarm_health import SWARM_METRICS + return {"docker": ["cpu_pct", "mem_pct", "restart_count", "is_healthy", *DISK_METRICS, *SWARM_METRICS]} diff --git a/plugins/docker/ingest.py b/plugins/docker/ingest.py index 0caa230..9efc563 100644 --- a/plugins/docker/ingest.py +++ b/plugins/docker/ingest.py @@ -292,6 +292,20 @@ async def _persist_status(session, host, status: dict) -> None: row.reported_at = status["at"] +async def _record_swarm_health(session, swarm: dict) -> None: + """The swarm-wide figures alert rules watch (#5788), at most once every + RECORD_EVERY whichever manager reported.""" + from datetime import timezone + from steward.core.alerts import MetricBatch + from . import swarm_health + if not swarm_health.due(datetime.now(timezone.utc)): + return + batch = MetricBatch() + for metric, value in swarm_health.figures(swarm).items(): + batch.add(source_module="docker", resource_name=swarm_health.RESOURCE, metric_name=metric, value=value) + await batch.record(session) + + async def _expire_swarm(session, host) -> None: """Drop the swarm rows a host reported while it was a manager.""" from .models import DockerSwarmNode, DockerSwarmService @@ -338,6 +352,7 @@ async def persist_host_docker(session, host, snapshots, swarm=None, disk=None, await mark("status") if swarm is not None: await _persist_swarm(session, host, swarm) + await _record_swarm_health(session, swarm) await mark("swarm") if disk is not None: await _persist_disk(session, host, disk) diff --git a/plugins/docker/swarm_health.py b/plugins/docker/swarm_health.py new file mode 100644 index 0000000..bc6ad49 --- /dev/null +++ b/plugins/docker/swarm_health.py @@ -0,0 +1,57 @@ +"""The swarm's health as three figures an alert rule can watch (#5788). + +Every manager's agent reports the whole swarm, so the figures are swarm-wide, +on one resource, `swarm`, and are recorded at most once every RECORD_EVERY +whichever manager sent them: three managers then read as one series, and one +manager going silent doesn't silence it. + +- swarm_services_short: services running fewer tasks than they want. A + service scaled to 0 wants none and is never short. +- swarm_nodes_down: nodes whose status isn't ready, other than drained ones. + A drained node is one taken out of service on purpose (VDNT-Util01 always; + any node during its maintenance), so its going away is expected. Its host's + own "Agent offline" rule still covers it. +- swarm_managers_unreachable: managers the swarm can't reach (agent 1.19.0 + sends ManagerStatus.Reachability; before that, a manager whose status + isn't ready), drained ones aside for the same reason. With three managers + one can be lost and the swarm keeps working; two, and it can't change. + +A maintenance job on a swarm node holds these rules' notifications in its +window (host_agent maintenance._swarm_window), since draining and rebooting a +node is exactly what moves them. +""" +from __future__ import annotations + +from datetime import datetime, timedelta + +RESOURCE = "swarm" +SWARM_METRICS = ("swarm_services_short", "swarm_nodes_down", "swarm_managers_unreachable") +RECORD_EVERY = timedelta(seconds=30) + +_last_recorded: datetime | None = None + + +def figures(swarm: dict) -> dict[str, float]: + """The three figures from one manager's swarm report.""" + services = swarm.get("services") or [] + nodes = swarm.get("nodes") or [] + short = sum(1 for s in services + if int(s.get("desired") or 0) > 0 and int(s.get("running") or 0) < int(s.get("desired") or 0)) + in_service = [n for n in nodes if (n.get("availability") or "active") != "drain"] + down = sum(1 for n in in_service if (n.get("status") or "unknown") != "ready") + managers = [n for n in in_service if n.get("role") == "manager"] + unreachable = sum(1 for n in managers + if (n["reachability"] != "reachable" if n.get("reachability") + else (n.get("status") or "unknown") != "ready")) + return {"swarm_services_short": float(short), "swarm_nodes_down": float(down), + "swarm_managers_unreachable": float(unreachable)} + + +def due(now: datetime) -> bool: + """Whether this report should be recorded: the first in RECORD_EVERY, + whichever manager it came from.""" + global _last_recorded + if _last_recorded is not None and now - _last_recorded < RECORD_EVERY: + return False + _last_recorded = now + return True diff --git a/plugins/host_agent/agent.py b/plugins/host_agent/agent.py index 41d6387..9349c6f 100644 --- a/plugins/host_agent/agent.py +++ b/plugins/host_agent/agent.py @@ -22,7 +22,7 @@ from collections import deque from concurrent.futures import ThreadPoolExecutor, wait as wait_futures from datetime import datetime, timezone -AGENT_VERSION = "1.18.0" +AGENT_VERSION = "1.19.0" # Default path to the local Docker Engine socket. Overridable via the # `docker_socket` config key; collection is silently skipped if it's absent or @@ -1170,7 +1170,10 @@ def _swarm_services(services: list, tasks: list) -> list: def _swarm_nodes(nodes: list) -> list: - """Normalise node role / availability / status (+ manager leader flag).""" + """Normalise node role / availability / status (+ manager leader flag and, + from 1.19.0, whether the swarm can reach a manager: "reachable", + "unreachable" or "unknown"; "" for a worker), so Steward can count the + managers the swarm has lost (#5788).""" out = [] for n in (nodes or []): spec = n.get("Spec") or {} @@ -1184,6 +1187,7 @@ def _swarm_nodes(nodes: list) -> list: "availability": (spec.get("Availability") or "active").lower(), "status": (status.get("State") or "unknown").lower(), "leader": bool(mgr.get("Leader", False)), + "reachability": (mgr.get("Reachability") or "").lower(), }) return out diff --git a/plugins/host_agent/maintenance.py b/plugins/host_agent/maintenance.py index 3dc2166..74c3123 100644 --- a/plugins/host_agent/maintenance.py +++ b/plugins/host_agent/maintenance.py @@ -188,21 +188,47 @@ def utcnow() -> datetime: async def _window(session, job, host_name: str, now: datetime) -> None: """Keep the job's maintenance window over the host until the job ends: one - scoped to this host's agent metrics, so an alert the job causes waits.""" + scoped to this host's agent metrics, so an alert the job causes waits. + A swarm node's job keeps a second over the swarm-wide figures (#5788): + draining and rebooting a node takes it down and can leave a service + pinned to it short, which is the job, not news.""" from steward.models.alerts import MaintenanceWindow if job.window_id: w = await session.get(MaintenanceWindow, job.window_id) if w is not None: w.end_at = max(w.start_at, window_end(job, now)) - return - if job.created_by is None or job.state in FINISHED: + elif job.created_by is None or job.state in FINISHED: return # a window needs an owner; none for a finished job - w = MaintenanceWindow(name=f"Maintenance: {host_name}"[:128], start_at=now, - end_at=window_end(job, now), scope_source="host_agent", - scope_resource=host_name, created_by=job.created_by) - session.add(w) - await session.flush() - job.window_id = w.id + else: + w = MaintenanceWindow(name=f"Maintenance: {host_name}"[:128], start_at=now, + end_at=window_end(job, now), scope_source="host_agent", + scope_resource=host_name, created_by=job.created_by) + session.add(w) + await session.flush() + job.window_id = w.id + if job.node_id: + await _swarm_window(session, job, host_name, now) + + +SWARM_SCOPE = ("docker", "swarm") # the swarm-wide figures' source and resource + + +async def _swarm_window(session, job, host_name: str, now: datetime) -> None: + """The swarm half of a swarm node's job window, found again by its name + and the job's start, so it needs no column of its own.""" + from sqlalchemy import select + from steward.models.alerts import MaintenanceWindow + name = f"Maintenance: {host_name} (swarm)"[:128] + w = (await session.execute(select(MaintenanceWindow).where( + MaintenanceWindow.name == name, MaintenanceWindow.scope_source == SWARM_SCOPE[0], + MaintenanceWindow.scope_resource == SWARM_SCOPE[1], + MaintenanceWindow.start_at >= job.created_at))).scalars().first() + if w is not None: + w.end_at = max(w.start_at, window_end(job, now)) + elif job.created_by is not None and job.state not in FINISHED: + session.add(MaintenanceWindow(name=name, start_at=now, end_at=window_end(job, now), + scope_source=SWARM_SCOPE[0], scope_resource=SWARM_SCOPE[1], + created_by=job.created_by)) async def _host_name(session, host_id: str) -> str: diff --git a/tests/integration/test_docker.py b/tests/integration/test_docker.py index 251e9a5..80abc77 100644 --- a/tests/integration/test_docker.py +++ b/tests/integration/test_docker.py @@ -575,17 +575,26 @@ def test_swarm_topology_persisted(app): s.add(h) await s.flush() # No containers in this sample — swarm must still persist. + from steward.core.plugin_manager import loaded_plugin_submodule + loaded_plugin_submodule("docker", "swarm_health")._last_recorded = None + since = datetime.now(timezone.utc) await persist(s, h, [], swarm) hid = h.id + # The swarm-wide figures alert rules watch (#5788): web is 2 of 3. + figures = dict((await s.execute(text( + "SELECT metric_name, value FROM plugin_metrics WHERE source_module = 'docker' " + "AND resource_name = 'swarm' AND recorded_at >= :t"), {"t": since})).all()) svc = (await s.execute(text( "SELECT mode, desired, running, image, placement_json, tasks_json " "FROM docker_swarm_services WHERE host_id = :h"), {"h": hid})).first() node = (await s.execute(text( "SELECT role, availability, status, leader " "FROM docker_swarm_nodes WHERE host_id = :h"), {"h": hid})).first() - return svc, node + return svc, node, figures - svc, node = asyncio.run(_go()) + svc, node, figures = asyncio.run(_go()) + assert figures == {"swarm_services_short": 1.0, "swarm_nodes_down": 0.0, + "swarm_managers_unreachable": 0.0} assert svc[0] == "replicated" and svc[1] == 3 and svc[2] == 2 and svc[3] == "nginx" assert "n1" in svc[4] # placement JSON carries the node id (task,) = json.loads(svc[5]) # docker_010: the manager's task records survive diff --git a/tests/integration/test_host_maintenance_job.py b/tests/integration/test_host_maintenance_job.py index 3fa4ead..0524e46 100644 --- a/tests/integration/test_host_maintenance_job.py +++ b/tests/integration/test_host_maintenance_job.py @@ -116,3 +116,39 @@ def test_a_job_is_followed_through_a_steward_restart_and_the_host_reboot(): assert window_end <= datetime.now(timezone.utc) asyncio.run(_second_half()) + + +def test_a_swarm_node_s_job_also_holds_the_swarm_wide_alerts(): + """Draining and rebooting a node moves the swarm figures; that is the job, + not news (#5788). The window over them follows the job's own, and ends + with it.""" + from types import SimpleNamespace + + from sqlalchemy import text + from steward.core.plugin_manager import loaded_plugin_submodule + from steward.models.users import User + app = _app() + maintenance = loaded_plugin_submodule("host_agent", "maintenance") + tag = uuid.uuid4().hex[:8] + host_name = f"swarm-node-{tag}" + + async def _go(): + now = datetime.now(timezone.utc) + async with app.db_sessionmaker() as s: + async with s.begin(): + user = User(username=f"sw-{tag}", email=f"sw-{tag}@test", password_hash="x") + s.add(user) + await s.flush() + job = SimpleNamespace(window_id=None, created_by=user.id, state="draining", + node_id="node-1", created_at=now - timedelta(seconds=1)) + await maintenance._window(s, job, host_name, now) + await maintenance._window(s, job, host_name, now + timedelta(minutes=1)) # the next tick + job.state = "done" + await maintenance._window(s, job, host_name, now + timedelta(minutes=2)) # finished + return (await s.execute(text( + "SELECT scope_source, scope_resource, end_at FROM maintenance_windows " + "WHERE name LIKE :n ORDER BY scope_source"), {"n": f"Maintenance: {host_name}%"})).all(), now + + rows, now = asyncio.run(_go()) + assert [(r[0], r[1]) for r in rows] == [("docker", "swarm"), ("host_agent", host_name)] + assert all(r[2] == now + timedelta(minutes=2) for r in rows) # ended with the job, not before diff --git a/tests/plugins/docker/test_swarm_health.py b/tests/plugins/docker/test_swarm_health.py new file mode 100644 index 0000000..9c50315 --- /dev/null +++ b/tests/plugins/docker/test_swarm_health.py @@ -0,0 +1,61 @@ +"""The swarm-wide figures alert rules watch (#5788). On 2026-10-14 nine +services sat at 0/1 after VDNT-Docker03 became a manager, and nothing fired.""" +from datetime import datetime, timedelta, timezone + +import pytest + +from plugins.docker import swarm_health as sh + +T0 = datetime(2026, 10, 15, tzinfo=timezone.utc) + + +def _node(name, role="manager", availability="active", status="ready", reachability="reachable"): + return {"node_id": name, "hostname": name, "role": role, "availability": availability, + "status": status, "leader": False, "reachability": reachability if role == "manager" else ""} + + +def test_services_short_of_their_replicas_are_counted(): + swarm = {"services": [{"service_name": "a", "desired": 1, "running": 0}, # the 2026-10-14 shape + {"service_name": "b", "desired": 3, "running": 2}, + {"service_name": "c", "desired": 2, "running": 2}, + {"service_name": "d", "desired": 0, "running": 0}, # scaled to 0: wants none + {"service_name": "e", "desired": 1, "running": 2}]} # an update overlapping + assert sh.figures(swarm)["swarm_services_short"] == 2 + + +def test_a_drained_node_going_away_is_expected(): + """VDNT-Util01 is always drained, and maintenance drains a node before it + reboots: neither is news. Its host's own Agent offline rule covers it.""" + nodes = [_node("util01", availability="drain", status="down", reachability="unreachable"), + _node("docker02"), _node("docker03")] + got = sh.figures({"nodes": nodes}) + assert got["swarm_nodes_down"] == 0 and got["swarm_managers_unreachable"] == 0 + + +def test_a_node_in_service_going_down_is_counted_and_a_manager_lost(): + nodes = [_node("util01", availability="drain"), _node("docker02", status="down", reachability="unreachable"), + _node("docker03"), _node("worker1", role="worker", status="disconnected")] + got = sh.figures({"nodes": nodes}) + assert got["swarm_nodes_down"] == 2 and got["swarm_managers_unreachable"] == 1 + + +@pytest.mark.parametrize("node, unreachable", [ + (_node("m", status="ready", reachability="unreachable"), 1), # the swarm lost it, though Docker says ready + (_node("m", status="down", reachability=""), 1), # agent < 1.19.0: its status decides + (_node("m", status="ready", reachability=""), 0), +]) +def test_a_manager_is_unreachable_by_the_swarm_s_own_word_when_the_agent_sends_it(node, unreachable): + assert sh.figures({"nodes": [node]})["swarm_managers_unreachable"] == unreachable + + +def test_three_managers_reporting_record_once_per_window(monkeypatch): + monkeypatch.setattr(sh, "_last_recorded", None) + assert sh.due(T0) + assert not sh.due(T0 + timedelta(seconds=7)) # the next manager's report + assert not sh.due(T0 + timedelta(seconds=29)) + assert sh.due(T0 + timedelta(seconds=30)) + + +def test_the_figures_are_offered_to_alert_rules(): + from plugins.docker import get_alert_metrics + assert set(sh.SWARM_METRICS) <= set(get_alert_metrics()["docker"]) diff --git a/tests/plugins/host_agent/test_agent_docker.py b/tests/plugins/host_agent/test_agent_docker.py index dbf7044..901c0b2 100644 --- a/tests/plugins/host_agent/test_agent_docker.py +++ b/tests/plugins/host_agent/test_agent_docker.py @@ -298,16 +298,18 @@ def test_swarm_nodes_normalises_fields(): nodes = [ {"ID": "n1", "Description": {"Hostname": "mgr"}, "Spec": {"Role": "manager", "Availability": "active"}, - "Status": {"State": "ready"}, "ManagerStatus": {"Leader": True}}, + "Status": {"State": "ready"}, "ManagerStatus": {"Leader": True, "Reachability": "Reachable"}}, {"ID": "n2", "Description": {"Hostname": "wrk"}, "Spec": {"Role": "worker", "Availability": "drain"}, "Status": {"State": "down"}}, ] out = {n["node_id"]: n for n in a._swarm_nodes(nodes)} assert out["n1"] == {"node_id": "n1", "hostname": "mgr", "role": "manager", - "availability": "active", "status": "ready", "leader": True} + "availability": "active", "status": "ready", "leader": True, + "reachability": "reachable"} assert out["n2"] == {"node_id": "n2", "hostname": "wrk", "role": "worker", - "availability": "drain", "status": "down", "leader": False} + "availability": "drain", "status": "down", "leader": False, + "reachability": ""} def test_collect_swarm_skips_on_worker(monkeypatch):