fix(docker): a swarm service back to all its replicas leaves the overview and dashboard (#5722)

Nine services that lost tasks in one node's bad three minutes stayed on the
overview as warnings for a day, 17 hours after they recovered. Against the
UI ruling (only what is current on the dashboard), a service running all its
replicas is now history: its failed tasks stay on the Docker page, not on
the overview or the dashboard's failed count. Services still short of
replicas that started failing within five minutes of each other read as one
finding naming them, dated from the first.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

plugins/docker/mcp_tools.py

@@ -107,10 +107,17 @@ def task_error_text(err: dict | None) -> str:
     return f"{why}{code}{where}"
 
 
+# Services that started failing within this long of each other read as one
+# finding: they are usually one node's event (#5722).
+TOGETHER = timedelta(minutes=5)
+
+
 def service_findings(services: list[dict]) -> list[dict]:
-    """Swarm services that are failing: crash-looping, under-replicated, or that
+    """Swarm services failing now: crash-looping, or short of replicas, each
-    lost a task to an error in the last 24h. Each names Swarm's last task error."""
+    naming Swarm's last task error. One back to all its replicas is history,
-    out = []
+    kept on the Docker page, not a finding (#5722). Several short of replicas
+    since within TOGETHER of each other are one finding naming them all."""
+    out, short = [], []
     for s in services:
         replicas = f"{s['running']} of {s['desired']} replicas running"
         last = task_error_text(s.get("last_error"))
@@ -124,20 +131,40 @@ def service_findings(services: list[dict]) -> list[dict]:
                 f"last hour, {replicas}{tail}", since=since, ids=ids,
                 next="list_swarm_services()"))
         elif s["state"] != "healthy":
+            short.append((s, replicas, tail))
+    for group in _together(short):
+        if len(group) == 1:
+            s, replicas, tail = group[0]
             out.append(finding(
                 "critical" if s["state"] == "down" else "warning", f"swarm_service_{s['state']}",
                 s["name"], f"swarm service {s['name']} has {replicas}{tail}",
-                since=since, ids=ids, next="list_swarm_services()"))
+                since=s.get("trouble_since"), ids={"service_name": s["name"]},
-        elif s.get("failures_24h"):
-            n = s["failures_24h"]
-            out.append(finding(
-                "warning", "swarm_task_failed", s["name"],
-                f"swarm service {s['name']} had {n} failed task{'s' if n != 1 else ''} in the last "
-                f"24h and is now {replicas}{tail}", since=since, ids=ids,
                 next="list_swarm_services()"))
+            continue
+        names = [s["name"] for s, _r, _t in group]
+        down = any(s["state"] == "down" for s, _r, _t in group)
+        out.append(finding(
+            "critical" if down else "warning", "swarm_services_short", f"{len(group)} swarm services",
+            f"{len(group)} swarm services started failing together and are short of replicas: "
+            + ", ".join(f"{s['name']} ({r})" for s, r, _t in group),
+            since=min(s["trouble_since"] for s, _r, _t in group),
+            ids={"service_names": names}, next="list_swarm_services()"))
     return out
 
 
+def _together(short: list[tuple]) -> list[list[tuple]]:
+    """Short services grouped by when they started failing: each group's
+    starts lie within TOGETHER of the one before. One without a start stands alone."""
+    timed = sorted((x for x in short if x[0].get("trouble_since")), key=lambda x: x[0]["trouble_since"])
+    groups: list[list[tuple]] = []
+    for x in timed:
+        if groups and x[0]["trouble_since"] - groups[-1][-1][0]["trouble_since"] <= TOGETHER:
+            groups[-1].append(x)
+        else:
+            groups.append([x])
+    return groups + [[x] for x in short if not x[0].get("trouble_since")]
+
+
 # ── loading ──────────────────────────────────────────────────────────────────
 
 async def _host_names(db) -> dict[str, str]:

plugins/docker/routes.py

@@ -245,7 +245,11 @@ async def widget():
     swarm_services = services_problems_first(build_swarm_services(
         all_containers, services, node_hostname, host_name,
         crash_loop_deaths=docker_svc.crash_loop_deaths(current_app.config))["services"])
-    failures = docker_svc.failures_24h(events, swarm_services, host_name)
+    # The dashboard shows what is wrong now: a service back to all its
+    # replicas has its failed tasks on the Docker page, not here (#5722).
+    still = {s["name"] for s in swarm_services if s["state"] != "healthy" or s.get("crash_looping")}
+    failures = [f for f in docker_svc.failures_24h(events, swarm_services, host_name)
+                if f["kind"] != "service" or f["name"] in still]
 
     standalone = [c for c in all_containers if not c.service_name]
     running = [c for c in standalone if c.status == "running"]

tests/plugins/docker/test_mcp_tools.py

@@ -96,7 +96,7 @@ def test_service_findings_name_the_crash_loop_and_swarms_error():
             {"name": "ok", "state": "healthy", "running": 1, "desired": 1, "crash_looping": False,
              "failures_1h": 0, "failures_24h": 0, "last_error": None}]
     got = {f["subject"]: f for f in t.service_findings(svcs)}
-    assert set(got) == {"db", "app", "web"}
+    assert set(got) == {"db", "app"}     # web is back to all its replicas: history (#5722)
     db = got["db"]
     assert db["kind"] == "swarm_service_crash_looping" and db["severity"] == "critical"
     assert "4 tasks failed in the last hour" in db["summary"]
@@ -104,9 +104,6 @@ def test_service_findings_name_the_crash_loop_and_swarms_error():
     # Down without a loop still names why: here Swarm could not even start it.
     assert got["app"]["kind"] == "swarm_service_down"
     assert got["app"]["summary"].endswith("last task error: No such image: app:dev")
-    # Recovered, but a task died in the last day: worth knowing, not critical.
-    assert got["web"]["kind"] == "swarm_task_failed" and got["web"]["severity"] == "warning"
-    assert "had 1 failed task in the last 24h" in got["web"]["summary"]
 
 
 def test_a_replaced_swarm_task_is_marked_and_judged_on_its_service():
@@ -178,3 +175,24 @@ def test_findings_carry_when_the_problem_began():
     svc = {"name": "db", "state": "down", "running": 0, "desired": 1, "trouble_since": began}
     (g,) = t.service_findings([svc])
     assert g["since"] == "2026-10-06T11:20:00Z"
+
+
+def test_services_that_failed_together_are_one_finding_and_recovered_ones_none():
+    """#5722: nine services lost tasks within three minutes on one node's bad
+    moment, and stayed on the overview for a day after they were back. Those
+    back to full replicas are history; those still short and started together
+    are one finding naming them, dated from the first."""
+    at = NOW - timedelta(minutes=30)
+    svcs = [{"name": "auth", "state": "degraded", "running": 1, "desired": 2, "trouble_since": at},
+            {"name": "wiki", "state": "down", "running": 0, "desired": 1,
+             "trouble_since": at + timedelta(minutes=2)},
+            {"name": "later", "state": "degraded", "running": 1, "desired": 2,
+             "trouble_since": at + timedelta(minutes=20)},
+            {"name": "back", "state": "healthy", "running": 1, "desired": 1, "failures_24h": 3,
+             "trouble_since": at}]
+    together, alone = t.service_findings(svcs)
+    assert together["kind"] == "swarm_services_short" and together["severity"] == "critical"
+    assert together["subject"] == "2 swarm services"
+    assert "auth (1 of 2 replicas running), wiki (0 of 1 replicas running)" in together["summary"]
+    assert together["since"] == "2026-10-06T11:30:00Z"
+    assert alone["subject"] == "later" and alone["kind"] == "swarm_service_degraded"