diff --git a/plugins/docker/mcp_tools.py b/plugins/docker/mcp_tools.py index 4b403e8..6e9ccb8 100644 --- a/plugins/docker/mcp_tools.py +++ b/plugins/docker/mcp_tools.py @@ -107,10 +107,17 @@ def task_error_text(err: dict | None) -> str: return f"{why}{code}{where}" +# Services that started failing within this long of each other read as one +# finding: they are usually one node's event (#5722). +TOGETHER = timedelta(minutes=5) + + def service_findings(services: list[dict]) -> list[dict]: - """Swarm services that are failing: crash-looping, under-replicated, or that - lost a task to an error in the last 24h. Each names Swarm's last task error.""" - out = [] + """Swarm services failing now: crash-looping, or short of replicas, each + naming Swarm's last task error. One back to all its replicas is history, + kept on the Docker page, not a finding (#5722). Several short of replicas + since within TOGETHER of each other are one finding naming them all.""" + out, short = [], [] for s in services: replicas = f"{s['running']} of {s['desired']} replicas running" last = task_error_text(s.get("last_error")) @@ -124,20 +131,40 @@ def service_findings(services: list[dict]) -> list[dict]: f"last hour, {replicas}{tail}", since=since, ids=ids, next="list_swarm_services()")) elif s["state"] != "healthy": + short.append((s, replicas, tail)) + for group in _together(short): + if len(group) == 1: + s, replicas, tail = group[0] out.append(finding( "critical" if s["state"] == "down" else "warning", f"swarm_service_{s['state']}", s["name"], f"swarm service {s['name']} has {replicas}{tail}", - since=since, ids=ids, next="list_swarm_services()")) - elif s.get("failures_24h"): - n = s["failures_24h"] - out.append(finding( - "warning", "swarm_task_failed", s["name"], - f"swarm service {s['name']} had {n} failed task{'s' if n != 1 else ''} in the last " - f"24h and is now {replicas}{tail}", since=since, ids=ids, + since=s.get("trouble_since"), ids={"service_name": s["name"]}, next="list_swarm_services()")) + continue + names = [s["name"] for s, _r, _t in group] + down = any(s["state"] == "down" for s, _r, _t in group) + out.append(finding( + "critical" if down else "warning", "swarm_services_short", f"{len(group)} swarm services", + f"{len(group)} swarm services started failing together and are short of replicas: " + + ", ".join(f"{s['name']} ({r})" for s, r, _t in group), + since=min(s["trouble_since"] for s, _r, _t in group), + ids={"service_names": names}, next="list_swarm_services()")) return out +def _together(short: list[tuple]) -> list[list[tuple]]: + """Short services grouped by when they started failing: each group's + starts lie within TOGETHER of the one before. One without a start stands alone.""" + timed = sorted((x for x in short if x[0].get("trouble_since")), key=lambda x: x[0]["trouble_since"]) + groups: list[list[tuple]] = [] + for x in timed: + if groups and x[0]["trouble_since"] - groups[-1][-1][0]["trouble_since"] <= TOGETHER: + groups[-1].append(x) + else: + groups.append([x]) + return groups + [[x] for x in short if not x[0].get("trouble_since")] + + # ── loading ────────────────────────────────────────────────────────────────── async def _host_names(db) -> dict[str, str]: diff --git a/plugins/docker/routes.py b/plugins/docker/routes.py index 91d09a5..5cb6d78 100644 --- a/plugins/docker/routes.py +++ b/plugins/docker/routes.py @@ -245,7 +245,11 @@ async def widget(): swarm_services = services_problems_first(build_swarm_services( all_containers, services, node_hostname, host_name, crash_loop_deaths=docker_svc.crash_loop_deaths(current_app.config))["services"]) - failures = docker_svc.failures_24h(events, swarm_services, host_name) + # The dashboard shows what is wrong now: a service back to all its + # replicas has its failed tasks on the Docker page, not here (#5722). + still = {s["name"] for s in swarm_services if s["state"] != "healthy" or s.get("crash_looping")} + failures = [f for f in docker_svc.failures_24h(events, swarm_services, host_name) + if f["kind"] != "service" or f["name"] in still] standalone = [c for c in all_containers if not c.service_name] running = [c for c in standalone if c.status == "running"] diff --git a/tests/plugins/docker/test_mcp_tools.py b/tests/plugins/docker/test_mcp_tools.py index 759e9ea..f5fd8c3 100644 --- a/tests/plugins/docker/test_mcp_tools.py +++ b/tests/plugins/docker/test_mcp_tools.py @@ -96,7 +96,7 @@ def test_service_findings_name_the_crash_loop_and_swarms_error(): {"name": "ok", "state": "healthy", "running": 1, "desired": 1, "crash_looping": False, "failures_1h": 0, "failures_24h": 0, "last_error": None}] got = {f["subject"]: f for f in t.service_findings(svcs)} - assert set(got) == {"db", "app", "web"} + assert set(got) == {"db", "app"} # web is back to all its replicas: history (#5722) db = got["db"] assert db["kind"] == "swarm_service_crash_looping" and db["severity"] == "critical" assert "4 tasks failed in the last hour" in db["summary"] @@ -104,9 +104,6 @@ def test_service_findings_name_the_crash_loop_and_swarms_error(): # Down without a loop still names why: here Swarm could not even start it. assert got["app"]["kind"] == "swarm_service_down" assert got["app"]["summary"].endswith("last task error: No such image: app:dev") - # Recovered, but a task died in the last day: worth knowing, not critical. - assert got["web"]["kind"] == "swarm_task_failed" and got["web"]["severity"] == "warning" - assert "had 1 failed task in the last 24h" in got["web"]["summary"] def test_a_replaced_swarm_task_is_marked_and_judged_on_its_service(): @@ -178,3 +175,24 @@ def test_findings_carry_when_the_problem_began(): svc = {"name": "db", "state": "down", "running": 0, "desired": 1, "trouble_since": began} (g,) = t.service_findings([svc]) assert g["since"] == "2026-10-06T11:20:00Z" + + +def test_services_that_failed_together_are_one_finding_and_recovered_ones_none(): + """#5722: nine services lost tasks within three minutes on one node's bad + moment, and stayed on the overview for a day after they were back. Those + back to full replicas are history; those still short and started together + are one finding naming them, dated from the first.""" + at = NOW - timedelta(minutes=30) + svcs = [{"name": "auth", "state": "degraded", "running": 1, "desired": 2, "trouble_since": at}, + {"name": "wiki", "state": "down", "running": 0, "desired": 1, + "trouble_since": at + timedelta(minutes=2)}, + {"name": "later", "state": "degraded", "running": 1, "desired": 2, + "trouble_since": at + timedelta(minutes=20)}, + {"name": "back", "state": "healthy", "running": 1, "desired": 1, "failures_24h": 3, + "trouble_since": at}] + together, alone = t.service_findings(svcs) + assert together["kind"] == "swarm_services_short" and together["severity"] == "critical" + assert together["subject"] == "2 swarm services" + assert "auth (1 of 2 replicas running), wiki (0 of 1 replicas running)" in together["summary"] + assert together["since"] == "2026-10-06T11:30:00Z" + assert alone["subject"] == "later" and alone["kind"] == "swarm_service_degraded"