fix(docker): a swarm service back to all its replicas leaves the overview and dashboard (#5722)

Nine services that lost tasks in one node's bad three minutes stayed on the
overview as warnings for a day, 17 hours after they recovered. Against the
UI ruling (only what is current on the dashboard), a service running all its
replicas is now history: its failed tasks stay on the Docker page, not on
the overview or the dashboard's failed count. Services still short of
replicas that started failing within five minutes of each other read as one
finding naming them, dated from the first.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

plugins/docker/mcp_tools.py

@@ -107,10 +107,17 @@ def task_error_text(err: dict | None) -> str:
     return f"{why}{code}{where}"
 
 
+# Services that started failing within this long of each other read as one
+# finding: they are usually one node's event (#5722).
+TOGETHER = timedelta(minutes=5)
+
+
 def service_findings(services: list[dict]) -> list[dict]:
-    """Swarm services that are failing: crash-looping, under-replicated, or that
-    lost a task to an error in the last 24h. Each names Swarm's last task error."""
-    out = []
+    """Swarm services failing now: crash-looping, or short of replicas, each
+    naming Swarm's last task error. One back to all its replicas is history,
+    kept on the Docker page, not a finding (#5722). Several short of replicas
+    since within TOGETHER of each other are one finding naming them all."""
+    out, short = [], []
     for s in services:
         replicas = f"{s['running']} of {s['desired']} replicas running"
         last = task_error_text(s.get("last_error"))
@@ -124,20 +131,40 @@ def service_findings(services: list[dict]) -> list[dict]:
                 f"last hour, {replicas}{tail}", since=since, ids=ids,
                 next="list_swarm_services()"))
         elif s["state"] != "healthy":
+            short.append((s, replicas, tail))
+    for group in _together(short):
+        if len(group) == 1:
+            s, replicas, tail = group[0]
             out.append(finding(
                 "critical" if s["state"] == "down" else "warning", f"swarm_service_{s['state']}",
                 s["name"], f"swarm service {s['name']} has {replicas}{tail}",
-                since=since, ids=ids, next="list_swarm_services()"))
-        elif s.get("failures_24h"):
-            n = s["failures_24h"]
-            out.append(finding(
-                "warning", "swarm_task_failed", s["name"],
-                f"swarm service {s['name']} had {n} failed task{'s' if n != 1 else ''} in the last "
-                f"24h and is now {replicas}{tail}", since=since, ids=ids,
+                since=s.get("trouble_since"), ids={"service_name": s["name"]},
                 next="list_swarm_services()"))
+            continue
+        names = [s["name"] for s, _r, _t in group]
+        down = any(s["state"] == "down" for s, _r, _t in group)
+        out.append(finding(
+            "critical" if down else "warning", "swarm_services_short", f"{len(group)} swarm services",
+            f"{len(group)} swarm services started failing together and are short of replicas: "
+            + ", ".join(f"{s['name']} ({r})" for s, r, _t in group),
+            since=min(s["trouble_since"] for s, _r, _t in group),
+            ids={"service_names": names}, next="list_swarm_services()"))
     return out
 
 
+def _together(short: list[tuple]) -> list[list[tuple]]:
+    """Short services grouped by when they started failing: each group's
+    starts lie within TOGETHER of the one before. One without a start stands alone."""
+    timed = sorted((x for x in short if x[0].get("trouble_since")), key=lambda x: x[0]["trouble_since"])
+    groups: list[list[tuple]] = []
+    for x in timed:
+        if groups and x[0]["trouble_since"] - groups[-1][-1][0]["trouble_since"] <= TOGETHER:
+            groups[-1].append(x)
+        else:
+            groups.append([x])
+    return groups + [[x] for x in short if not x[0].get("trouble_since")]
+
+
 # ── loading ──────────────────────────────────────────────────────────────────
 
 async def _host_names(db) -> dict[str, str]:

plugins/docker/routes.py

@@ -245,7 +245,11 @@ async def widget():
     swarm_services = services_problems_first(build_swarm_services(
         all_containers, services, node_hostname, host_name,
         crash_loop_deaths=docker_svc.crash_loop_deaths(current_app.config))["services"])
-    failures = docker_svc.failures_24h(events, swarm_services, host_name)
+    # The dashboard shows what is wrong now: a service back to all its
+    # replicas has its failed tasks on the Docker page, not here (#5722).
+    still = {s["name"] for s in swarm_services if s["state"] != "healthy" or s.get("crash_looping")}
+    failures = [f for f in docker_svc.failures_24h(events, swarm_services, host_name)
+                if f["kind"] != "service" or f["name"] in still]
 
     standalone = [c for c in all_containers if not c.service_name]
     running = [c for c in standalone if c.status == "running"]