fix(docker): a swarm service back to all its replicas leaves the overview and dashboard (#5722)

Nine services that lost tasks in one node's bad three minutes stayed on the
overview as warnings for a day, 17 hours after they recovered. Against the
UI ruling (only what is current on the dashboard), a service running all its
replicas is now history: its failed tasks stay on the Docker page, not on
the overview or the dashboard's failed count. Services still short of
replicas that started failing within five minutes of each other read as one
finding naming them, dated from the first.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
2026-10-14 16:50:31 -04:00
co-authored by Claude Opus 5.5
parent d0c563abd0
commit 1b82d8ad83
3 changed files with 64 additions and 15 deletions
+22 -4
View File
@@ -96,7 +96,7 @@ def test_service_findings_name_the_crash_loop_and_swarms_error():
{"name": "ok", "state": "healthy", "running": 1, "desired": 1, "crash_looping": False,
"failures_1h": 0, "failures_24h": 0, "last_error": None}]
got = {f["subject"]: f for f in t.service_findings(svcs)}
assert set(got) == {"db", "app", "web"}
assert set(got) == {"db", "app"} # web is back to all its replicas: history (#5722)
db = got["db"]
assert db["kind"] == "swarm_service_crash_looping" and db["severity"] == "critical"
assert "4 tasks failed in the last hour" in db["summary"]
@@ -104,9 +104,6 @@ def test_service_findings_name_the_crash_loop_and_swarms_error():
# Down without a loop still names why: here Swarm could not even start it.
assert got["app"]["kind"] == "swarm_service_down"
assert got["app"]["summary"].endswith("last task error: No such image: app:dev")
# Recovered, but a task died in the last day: worth knowing, not critical.
assert got["web"]["kind"] == "swarm_task_failed" and got["web"]["severity"] == "warning"
assert "had 1 failed task in the last 24h" in got["web"]["summary"]
def test_a_replaced_swarm_task_is_marked_and_judged_on_its_service():
@@ -178,3 +175,24 @@ def test_findings_carry_when_the_problem_began():
svc = {"name": "db", "state": "down", "running": 0, "desired": 1, "trouble_since": began}
(g,) = t.service_findings([svc])
assert g["since"] == "2026-10-06T11:20:00Z"
def test_services_that_failed_together_are_one_finding_and_recovered_ones_none():
"""#5722: nine services lost tasks within three minutes on one node's bad
moment, and stayed on the overview for a day after they were back. Those
back to full replicas are history; those still short and started together
are one finding naming them, dated from the first."""
at = NOW - timedelta(minutes=30)
svcs = [{"name": "auth", "state": "degraded", "running": 1, "desired": 2, "trouble_since": at},
{"name": "wiki", "state": "down", "running": 0, "desired": 1,
"trouble_since": at + timedelta(minutes=2)},
{"name": "later", "state": "degraded", "running": 1, "desired": 2,
"trouble_since": at + timedelta(minutes=20)},
{"name": "back", "state": "healthy", "running": 1, "desired": 1, "failures_24h": 3,
"trouble_since": at}]
together, alone = t.service_findings(svcs)
assert together["kind"] == "swarm_services_short" and together["severity"] == "critical"
assert together["subject"] == "2 swarm services"
assert "auth (1 of 2 replicas running), wiki (0 of 1 replicas running)" in together["summary"]
assert together["since"] == "2026-10-06T11:30:00Z"
assert alone["subject"] == "later" and alone["kind"] == "swarm_service_degraded"