feat(docker): Swarm task awareness — replaced tasks, service crash loops, last task error

Milestone 465 "Swarm task awareness". Fixes three defects from the live MCP review #4973: - #5008 "Swarm task history read as live state" - #5009 "Crash-loop detection is blind to Swarm services" - #5011 "Swarm task errors are not captured" Agent and ingest (agent 1.8.0): - A manager's agent now reports each service's newest tasks: slot, node, desired and current state, Swarm's Err/Message, exit code and time. - They are stored in docker_swarm_services.tasks_json (migration docker_010_swarm_tasks). Swarm view (pure): - A task container is "replaced" when Swarm set its task's desired state off `running`. Without the task on record, a newer or running task in the same slot decides. - Replaced containers are no longer replicas or crashes. - Each service gets, from the task records: - failures_1h and failures_24h (failed or rejected tasks); - crash_looping (Settings → Thresholds count); - last_error, in Swarm's own words. MCP: - New findings: swarm_service_crash_looping (critical) and swarm_task_failed (warning, on a healthy service with a failure in 24h). - Down and degraded findings name the last task error. - list_swarm_services carries the failure fields. - Container rows mark swarm_replaced and give a replaced task no problems. Web: - The containers page swarm cards and the Swarm topology page show crash loops, recent failures and the last task error. - The dashboard widget flags a crash loop, and its "failed (24h)" count no longer counts a rolling update's killed task. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-06 09:46:07 -04:00
parent 3474cdff46
commit bac2859455
17 changed files with 680 additions and 79 deletions
@@ -7,6 +7,7 @@ containers on different hosts no longer collide. Requires STEWARD_DATABASE_URL.
 from __future__ import annotations

 import asyncio
+import json
 import os
 import uuid
 from datetime import datetime, timezone
@@ -391,7 +392,11 @@ def test_swarm_topology_persisted(app):
    swarm = {
        "services": [{"service_name": "web", "mode": "replicated",
                      "desired": 3, "running": 2, "image": "nginx",
-                      "placement": [{"node_id": "n1", "running": 2}]}],
+                      "placement": [{"node_id": "n1", "running": 2}],
+                      "tasks": [{"task_id": "t1", "slot": 3, "node_id": "n1",
+                                 "desired_state": "shutdown", "state": "failed",
+                                 "message": "started", "error": "task: non-zero exit (1)",
+                                 "exit_code": 1, "at": "2026-10-06T13:19:51Z"}]}],
        "nodes": [{"node_id": "n1", "hostname": "mgr", "role": "manager",
                   "availability": "active", "status": "ready", "leader": True}],
    }
@@ -408,7 +413,7 @@ def test_swarm_topology_persisted(app):
                await persist(s, h, [], swarm)
                hid = h.id
            svc = (await s.execute(text(
-                "SELECT mode, desired, running, image, placement_json "
+                "SELECT mode, desired, running, image, placement_json, tasks_json "
                "FROM docker_swarm_services WHERE host_id = :h"), {"h": hid})).first()
            node = (await s.execute(text(
                "SELECT role, availability, status, leader "
@@ -418,6 +423,8 @@ def test_swarm_topology_persisted(app):
    svc, node = asyncio.run(_go())
    assert svc[0] == "replicated" and svc[1] == 3 and svc[2] == 2 and svc[3] == "nginx"
    assert "n1" in svc[4]  # placement JSON carries the node id
+    (task,) = json.loads(svc[5])  # docker_010: the manager's task records survive
+    assert task["error"] == "task: non-zero exit (1)" and task["slot"] == 3
    assert node[0] == "manager" and node[2] == "ready" and node[3] is True


@@ -9,6 +9,7 @@ from __future__ import annotations

 import asyncio
 import importlib
+import json
 import os
 import uuid
 from datetime import datetime, timedelta, timezone
@@ -57,14 +58,18 @@ def test_docker_and_agent_tools_over_seeded_rows(app):
    hid, hid2 = str(uuid.uuid4()), str(uuid.uuid4())
    host_name, quiet_host = f"dock-{tag}", f"quiet-{tag}"
    web, db_, job, svc = f"web-{tag}", f"db-{tag}", f"job-{tag}", f"svc-{tag}"
+    replaced = f"{svc}.1.old"
+
+    def _ago(minutes):
+        return (now - timedelta(minutes=minutes)).isoformat().replace("+00:00", "Z")

    container_sql = text("""
        INSERT INTO docker_containers
            (host_id, name, container_id, image, status, cpu_pct, mem_pct, mem_usage_bytes,
             mem_limit_bytes, restart_count, ports_json, started_at, scraped_at, health,
-             exit_code, oom_killed, compose_project, service_name)
+             exit_code, oom_killed, compose_project, service_name, task_id)
        VALUES (:h, :n, :cid, 'img:1', :st, 5.0, 10.0, 1000, 10000, :rc, '[]', :started, :now,
-                :health, :exit, false, :proj, :svc)
+                :health, :exit, false, :proj, :svc, :task)
    """)

    async def _seed():
@@ -73,20 +78,23 @@ def test_docker_and_agent_tools_over_seeded_rows(app):
                s.add(Host(id=hid, name=host_name, address="10.8.8.8"))
                s.add(Host(id=hid2, name=quiet_host, address="10.8.8.9"))
            async with s.begin():
-                for name, st, health, exit_code, proj, service in (
-                    (web, "running", "healthy", None, f"stack-{tag}", None),
-                    (db_, "running", "unhealthy", None, f"stack-{tag}", None),
-                    (job, "exited", None, 1, None, None),
-                    (f"{svc}.1.abc", "running", None, None, None, svc),
+                for name, st, health, exit_code, proj, service, task in (
+                    (web, "running", "healthy", None, f"stack-{tag}", None, None),
+                    (db_, "running", "unhealthy", None, f"stack-{tag}", None, None),
+                    (job, "exited", None, 1, None, None, None),
+                    (f"{svc}.1.abc", "running", None, None, None, svc, f"abc-{tag}"),
+                    # The task a rolling update replaced: killed (137), not crashed.
+                    (replaced, "exited", None, 137, None, svc, f"old-{tag}"),
                ):
                    await s.execute(container_sql, {
                        "h": hid, "n": name, "cid": uuid.uuid4().hex, "st": st, "rc": 0,
                        "started": now - timedelta(hours=2), "now": now, "health": health,
-                        "exit": exit_code, "proj": proj, "svc": service})
-                await s.execute(text(
-                    "INSERT INTO docker_events (id, host_id, container_name, event, detail, at) "
-                    "VALUES (:id, :h, :n, 'die', 'exit 1', :at)"),
-                    {"id": str(uuid.uuid4()), "h": hid, "n": job, "at": now - timedelta(minutes=10)})
+                        "exit": exit_code, "proj": proj, "svc": service, "task": task})
+                for name in (job, replaced):
+                    await s.execute(text(
+                        "INSERT INTO docker_events (id, host_id, container_name, event, detail, at) "
+                        "VALUES (:id, :h, :n, 'die', 'exit 1', :at)"),
+                        {"id": str(uuid.uuid4()), "h": hid, "n": name, "at": now - timedelta(minutes=10)})
                for i, (stream, line) in enumerate((
                        ("stdout", "starting up"), ("stdout", "connected to db"),
                        ("stderr", "ERROR: 100% of disk_used"), ("stdout", "retrying"),
@@ -104,8 +112,18 @@ def test_docker_and_agent_tools_over_seeded_rows(app):
                         "t": now - timedelta(minutes=30 - i * 10), "c": cpu})
                await s.execute(text(
                    "INSERT INTO docker_swarm_services (host_id, service_name, mode, desired, running, "
-                    "placement_json, updated_at) VALUES (:h, :n, 'replicated', 3, 1, '[]', :now)"),
-                    {"h": hid, "n": svc, "now": now})
+                    "placement_json, tasks_json, updated_at) "
+                    "VALUES (:h, :n, 'replicated', 3, 1, '[]', :tasks, :now)"),
+                    {"h": hid, "n": svc, "now": now, "tasks": json.dumps([
+                        {"task_id": f"abc-{tag}", "slot": 1, "node_id": f"node-{tag}",
+                         "desired_state": "running", "state": "running", "message": "started",
+                         "error": "", "exit_code": None, "at": _ago(9)},
+                        {"task_id": f"bad-{tag}", "slot": 2, "node_id": f"node-{tag}",
+                         "desired_state": "shutdown", "state": "failed", "message": "started",
+                         "error": "task: non-zero exit (1)", "exit_code": 1, "at": _ago(10)},
+                        {"task_id": f"old-{tag}", "slot": 1, "node_id": f"node-{tag}",
+                         "desired_state": "shutdown", "state": "shutdown", "message": "shutdown",
+                         "error": "", "exit_code": 137, "at": _ago(11)}])})
                await s.execute(text(
                    "INSERT INTO docker_swarm_nodes (host_id, node_id, hostname, role, availability, "
                    "status, leader, updated_at) VALUES (:h, :nid, :hn, 'manager', 'active', 'ready', "
@@ -154,7 +172,7 @@ def test_docker_and_agent_tools_over_seeded_rows(app):
        crashed = await dtools.get_container(job, host=host_name)
        assert [e["event"] for e in crashed["events"]] == ["die"]
        # A unique part of a long swarm task name is enough.
-        assert (await dtools.get_container(f"{svc}.1"))["swarm_service"] == svc
+        assert (await dtools.get_container(f"{svc}.1.ab"))["swarm_service"] == svc
        with pytest.raises(ValueError, match="list_containers"):
            await dtools.get_container(f"nope-{tag}")

@@ -180,6 +198,15 @@ def test_docker_and_agent_tools_over_seeded_rows(app):
        sw = await dtools.list_swarm_services(limit=500)
        s = next(x for x in sw["services"] if x["service_name"] == svc)
        assert (s["state"], s["running"], s["desired"]) == ("degraded", 1, 3)
+        # Swarm's task records: the replaced task is history, the failed one counts.
+        assert [r["container_name"] for r in s["replicas"]] == [f"{svc}.1.abc"]
+        assert s["previous_tasks"] == 1 and s["tasks_known"] is True
+        assert (s["failures_1h"], s["failures_24h"], s["crash_looping"]) == (1, 1, False)
+        assert s["last_error"]["error"] == "task: non-zero exit (1)"
+        assert s["last_error"]["node"] == host_name and s["last_error"]["age_s"] >= 600
+        old = (await dtools.list_containers(host=host_name, status="exited", limit=500))["containers"]
+        old = next(c for c in old if c["container_name"] == replaced)
+        assert old["swarm_replaced"] is True and old["problems"] == []
        assert any(n["node_id"] == f"node-{tag}" and n["leader"] for n in sw["nodes"])
        disk = (await dtools.get_docker_disk_usage(host=host_name))["hosts"]
        assert disk[0]["images_reclaimable_bytes"] == 3000
@@ -191,10 +218,23 @@ def test_docker_and_agent_tools_over_seeded_rows(app):
        assert ("container_unhealthy", db_) in found
        assert ("container_exited_nonzero", job) in found
        assert ("swarm_service_degraded", svc) in found
+        assert not any(subject == replaced for _, subject in found)
        assert not any(subject == web for _, subject in found)
        assert ("agent_never_reported", quiet_host) in agent_found
        assert not any(subject == host_name for _, subject in agent_found)

+        # ── the web pages say the same ─────────────────────────────────────
+        client = app.test_client()
+        async with client.session_transaction() as sess:
+            sess["user_id"], sess["user_role"], sess["username"] = "u", "viewer", "viewer"
+        topo = await (await client.get("/plugins/docker/swarm")).get_data(as_text=True)
+        assert svc in topo and "1 task failed in the last 24h" in topo
+        assert "task: non-zero exit (1)" in topo
+        cards = await (await client.get("/plugins/docker/rows")).get_data(as_text=True)
+        assert f"{svc}.1.abc" in cards and replaced not in cards
+        assert "task: non-zero exit (1)" in cards
+        assert (await client.get("/plugins/docker/widget")).status_code == 200
+
        # ── host agents ────────────────────────────────────────────────────
        agents = {a["host_name"]: a for a in (await atools.list_agents())["agents"]}
        assert agents[host_name]["state"] == "reporting" and agents[host_name]["agent_version"] == "1.5.0"