feat(docker): Swarm task awareness — replaced tasks, service crash loops, last task error

Milestone 465 "Swarm task awareness". Fixes three defects from the live MCP
review #4973:
- #5008 "Swarm task history read as live state"
- #5009 "Crash-loop detection is blind to Swarm services"
- #5011 "Swarm task errors are not captured"

Agent and ingest (agent 1.8.0):
- A manager's agent now reports each service's newest tasks: slot, node,
  desired and current state, Swarm's Err/Message, exit code and time.
- They are stored in docker_swarm_services.tasks_json (migration
  docker_010_swarm_tasks).

Swarm view (pure):
- A task container is "replaced" when Swarm set its task's desired state off
  `running`. Without the task on record, a newer or running task in the same
  slot decides.
- Replaced containers are no longer replicas or crashes.
- Each service gets, from the task records:
  - failures_1h and failures_24h (failed or rejected tasks);
  - crash_looping (Settings → Thresholds count);
  - last_error, in Swarm's own words.

MCP:
- New findings: swarm_service_crash_looping (critical) and swarm_task_failed
  (warning, on a healthy service with a failure in 24h).
- Down and degraded findings name the last task error.
- list_swarm_services carries the failure fields.
- Container rows mark swarm_replaced and give a replaced task no problems.

Web:
- The containers page swarm cards and the Swarm topology page show crash
  loops, recent failures and the last task error.
- The dashboard widget flags a crash loop, and its "failed (24h)" count no
  longer counts a rolling update's killed task.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

tests/integration/test_docker.py

@@ -7,6 +7,7 @@ containers on different hosts no longer collide. Requires STEWARD_DATABASE_URL.
 from __future__ import annotations
 
 import asyncio
+import json
 import os
 import uuid
 from datetime import datetime, timezone
@@ -391,7 +392,11 @@ def test_swarm_topology_persisted(app):
     swarm = {
         "services": [{"service_name": "web", "mode": "replicated",
                       "desired": 3, "running": 2, "image": "nginx",
-                      "placement": [{"node_id": "n1", "running": 2}]}],
+                      "placement": [{"node_id": "n1", "running": 2}],
+                      "tasks": [{"task_id": "t1", "slot": 3, "node_id": "n1",
+                                 "desired_state": "shutdown", "state": "failed",
+                                 "message": "started", "error": "task: non-zero exit (1)",
+                                 "exit_code": 1, "at": "2026-10-06T13:19:51Z"}]}],
         "nodes": [{"node_id": "n1", "hostname": "mgr", "role": "manager",
                    "availability": "active", "status": "ready", "leader": True}],
     }
@@ -408,7 +413,7 @@ def test_swarm_topology_persisted(app):
                 await persist(s, h, [], swarm)
                 hid = h.id
             svc = (await s.execute(text(
-                "SELECT mode, desired, running, image, placement_json "
+                "SELECT mode, desired, running, image, placement_json, tasks_json "
                 "FROM docker_swarm_services WHERE host_id = :h"), {"h": hid})).first()
             node = (await s.execute(text(
                 "SELECT role, availability, status, leader "
@@ -418,6 +423,8 @@ def test_swarm_topology_persisted(app):
     svc, node = asyncio.run(_go())
     assert svc[0] == "replicated" and svc[1] == 3 and svc[2] == 2 and svc[3] == "nginx"
     assert "n1" in svc[4]  # placement JSON carries the node id
+    (task,) = json.loads(svc[5])  # docker_010: the manager's task records survive
+    assert task["error"] == "task: non-zero exit (1)" and task["slot"] == 3
     assert node[0] == "manager" and node[2] == "ready" and node[3] is True
 
 

tests/integration/test_mcp_plugin_tools.py

@@ -9,6 +9,7 @@ from __future__ import annotations
 
 import asyncio
 import importlib
+import json
 import os
 import uuid
 from datetime import datetime, timedelta, timezone
@@ -57,14 +58,18 @@ def test_docker_and_agent_tools_over_seeded_rows(app):
     hid, hid2 = str(uuid.uuid4()), str(uuid.uuid4())
     host_name, quiet_host = f"dock-{tag}", f"quiet-{tag}"
     web, db_, job, svc = f"web-{tag}", f"db-{tag}", f"job-{tag}", f"svc-{tag}"
+    replaced = f"{svc}.1.old"
+
+    def _ago(minutes):
+        return (now - timedelta(minutes=minutes)).isoformat().replace("+00:00", "Z")
 
     container_sql = text("""
         INSERT INTO docker_containers
             (host_id, name, container_id, image, status, cpu_pct, mem_pct, mem_usage_bytes,
              mem_limit_bytes, restart_count, ports_json, started_at, scraped_at, health,
-             exit_code, oom_killed, compose_project, service_name)
+             exit_code, oom_killed, compose_project, service_name, task_id)
         VALUES (:h, :n, :cid, 'img:1', :st, 5.0, 10.0, 1000, 10000, :rc, '[]', :started, :now,
-                :health, :exit, false, :proj, :svc)
+                :health, :exit, false, :proj, :svc, :task)
     """)
 
     async def _seed():
@@ -73,20 +78,23 @@ def test_docker_and_agent_tools_over_seeded_rows(app):
                 s.add(Host(id=hid, name=host_name, address="10.8.8.8"))
                 s.add(Host(id=hid2, name=quiet_host, address="10.8.8.9"))
             async with s.begin():
-                for name, st, health, exit_code, proj, service in (
-                    (web, "running", "healthy", None, f"stack-{tag}", None),
-                    (db_, "running", "unhealthy", None, f"stack-{tag}", None),
-                    (job, "exited", None, 1, None, None),
-                    (f"{svc}.1.abc", "running", None, None, None, svc),
+                for name, st, health, exit_code, proj, service, task in (
+                    (web, "running", "healthy", None, f"stack-{tag}", None, None),
+                    (db_, "running", "unhealthy", None, f"stack-{tag}", None, None),
+                    (job, "exited", None, 1, None, None, None),
+                    (f"{svc}.1.abc", "running", None, None, None, svc, f"abc-{tag}"),
+                    # The task a rolling update replaced: killed (137), not crashed.
+                    (replaced, "exited", None, 137, None, svc, f"old-{tag}"),
                 ):
                     await s.execute(container_sql, {
                         "h": hid, "n": name, "cid": uuid.uuid4().hex, "st": st, "rc": 0,
                         "started": now - timedelta(hours=2), "now": now, "health": health,
-                        "exit": exit_code, "proj": proj, "svc": service})
-                await s.execute(text(
-                    "INSERT INTO docker_events (id, host_id, container_name, event, detail, at) "
-                    "VALUES (:id, :h, :n, 'die', 'exit 1', :at)"),
-                    {"id": str(uuid.uuid4()), "h": hid, "n": job, "at": now - timedelta(minutes=10)})
+                        "exit": exit_code, "proj": proj, "svc": service, "task": task})
+                for name in (job, replaced):
+                    await s.execute(text(
+                        "INSERT INTO docker_events (id, host_id, container_name, event, detail, at) "
+                        "VALUES (:id, :h, :n, 'die', 'exit 1', :at)"),
+                        {"id": str(uuid.uuid4()), "h": hid, "n": name, "at": now - timedelta(minutes=10)})
                 for i, (stream, line) in enumerate((
                         ("stdout", "starting up"), ("stdout", "connected to db"),
                         ("stderr", "ERROR: 100% of disk_used"), ("stdout", "retrying"),
@@ -104,8 +112,18 @@ def test_docker_and_agent_tools_over_seeded_rows(app):
                          "t": now - timedelta(minutes=30 - i * 10), "c": cpu})
                 await s.execute(text(
                     "INSERT INTO docker_swarm_services (host_id, service_name, mode, desired, running, "
-                    "placement_json, updated_at) VALUES (:h, :n, 'replicated', 3, 1, '[]', :now)"),
-                    {"h": hid, "n": svc, "now": now})
+                    "placement_json, tasks_json, updated_at) "
+                    "VALUES (:h, :n, 'replicated', 3, 1, '[]', :tasks, :now)"),
+                    {"h": hid, "n": svc, "now": now, "tasks": json.dumps([
+                        {"task_id": f"abc-{tag}", "slot": 1, "node_id": f"node-{tag}",
+                         "desired_state": "running", "state": "running", "message": "started",
+                         "error": "", "exit_code": None, "at": _ago(9)},
+                        {"task_id": f"bad-{tag}", "slot": 2, "node_id": f"node-{tag}",
+                         "desired_state": "shutdown", "state": "failed", "message": "started",
+                         "error": "task: non-zero exit (1)", "exit_code": 1, "at": _ago(10)},
+                        {"task_id": f"old-{tag}", "slot": 1, "node_id": f"node-{tag}",
+                         "desired_state": "shutdown", "state": "shutdown", "message": "shutdown",
+                         "error": "", "exit_code": 137, "at": _ago(11)}])})
                 await s.execute(text(
                     "INSERT INTO docker_swarm_nodes (host_id, node_id, hostname, role, availability, "
                     "status, leader, updated_at) VALUES (:h, :nid, :hn, 'manager', 'active', 'ready', "
@@ -154,7 +172,7 @@ def test_docker_and_agent_tools_over_seeded_rows(app):
         crashed = await dtools.get_container(job, host=host_name)
         assert [e["event"] for e in crashed["events"]] == ["die"]
         # A unique part of a long swarm task name is enough.
-        assert (await dtools.get_container(f"{svc}.1"))["swarm_service"] == svc
+        assert (await dtools.get_container(f"{svc}.1.ab"))["swarm_service"] == svc
         with pytest.raises(ValueError, match="list_containers"):
             await dtools.get_container(f"nope-{tag}")
 
@@ -180,6 +198,15 @@ def test_docker_and_agent_tools_over_seeded_rows(app):
         sw = await dtools.list_swarm_services(limit=500)
         s = next(x for x in sw["services"] if x["service_name"] == svc)
         assert (s["state"], s["running"], s["desired"]) == ("degraded", 1, 3)
+        # Swarm's task records: the replaced task is history, the failed one counts.
+        assert [r["container_name"] for r in s["replicas"]] == [f"{svc}.1.abc"]
+        assert s["previous_tasks"] == 1 and s["tasks_known"] is True
+        assert (s["failures_1h"], s["failures_24h"], s["crash_looping"]) == (1, 1, False)
+        assert s["last_error"]["error"] == "task: non-zero exit (1)"
+        assert s["last_error"]["node"] == host_name and s["last_error"]["age_s"] >= 600
+        old = (await dtools.list_containers(host=host_name, status="exited", limit=500))["containers"]
+        old = next(c for c in old if c["container_name"] == replaced)
+        assert old["swarm_replaced"] is True and old["problems"] == []
         assert any(n["node_id"] == f"node-{tag}" and n["leader"] for n in sw["nodes"])
         disk = (await dtools.get_docker_disk_usage(host=host_name))["hosts"]
         assert disk[0]["images_reclaimable_bytes"] == 3000
@@ -191,10 +218,23 @@ def test_docker_and_agent_tools_over_seeded_rows(app):
         assert ("container_unhealthy", db_) in found
         assert ("container_exited_nonzero", job) in found
         assert ("swarm_service_degraded", svc) in found
+        assert not any(subject == replaced for _, subject in found)
         assert not any(subject == web for _, subject in found)
         assert ("agent_never_reported", quiet_host) in agent_found
         assert not any(subject == host_name for _, subject in agent_found)
 
+        # ── the web pages say the same ─────────────────────────────────────
+        client = app.test_client()
+        async with client.session_transaction() as sess:
+            sess["user_id"], sess["user_role"], sess["username"] = "u", "viewer", "viewer"
+        topo = await (await client.get("/plugins/docker/swarm")).get_data(as_text=True)
+        assert svc in topo and "1 task failed in the last 24h" in topo
+        assert "task: non-zero exit (1)" in topo
+        cards = await (await client.get("/plugins/docker/rows")).get_data(as_text=True)
+        assert f"{svc}.1.abc" in cards and replaced not in cards
+        assert "task: non-zero exit (1)" in cards
+        assert (await client.get("/plugins/docker/widget")).status_code == 200
+
         # ── host agents ────────────────────────────────────────────────────
         agents = {a["host_name"]: a for a in (await atools.list_agents())["agents"]}
         assert agents[host_name]["state"] == "reporting" and agents[host_name]["agent_version"] == "1.5.0"

tests/plugins/docker/test_mcp_tools.py

@@ -73,6 +73,40 @@ def test_service_findings_by_state():
     assert got == {"b": "warning", "c": "critical"}
 
 
+def test_service_findings_name_the_crash_loop_and_swarms_error():
+    err = {"error": "task: non-zero exit (1)", "message": "started", "state": "failed",
+           "exit_code": 1, "node": "docker03"}
+    svcs = [{"name": "db", "state": "down", "running": 0, "desired": 1, "crash_looping": True,
+             "failures_1h": 4, "failures_24h": 5, "last_error": err},
+            {"name": "app", "state": "down", "running": 0, "desired": 1, "crash_looping": False,
+             "failures_1h": 0, "failures_24h": 0,
+             "last_error": {"error": "No such image: app:dev", "state": "rejected",
+                            "exit_code": None, "node": None, "message": None}},
+            {"name": "web", "state": "healthy", "running": 2, "desired": 2, "crash_looping": False,
+             "failures_1h": 0, "failures_24h": 1, "last_error": err},
+            {"name": "ok", "state": "healthy", "running": 1, "desired": 1, "crash_looping": False,
+             "failures_1h": 0, "failures_24h": 0, "last_error": None}]
+    got = {f["subject"]: f for f in t.service_findings(svcs)}
+    assert set(got) == {"db", "app", "web"}
+    db = got["db"]
+    assert db["kind"] == "swarm_service_crash_looping" and db["severity"] == "critical"
+    assert "4 tasks failed in the last hour" in db["summary"]
+    assert "last task error: task: non-zero exit (1), exit 1 on docker03" in db["summary"]
+    # Down without a loop still names why: here Swarm could not even start it.
+    assert got["app"]["kind"] == "swarm_service_down"
+    assert got["app"]["summary"].endswith("last task error: No such image: app:dev")
+    # Recovered, but a task died in the last day: worth knowing, not critical.
+    assert got["web"]["kind"] == "swarm_task_failed" and got["web"]["severity"] == "warning"
+    assert "had 1 failed task in the last 24h" in got["web"]["summary"]
+
+
+def test_a_replaced_swarm_task_is_marked_and_judged_on_its_service():
+    task = _c(name="app.1.old", status="exited", exit_code=137, service_name="app")
+    row = t.container_row(task, {}, [], NOW, replaced=True)
+    assert row["swarm_replaced"] is True and row["problems"] == []
+    assert t.container_row(_c(), {}, [], NOW)["swarm_replaced"] is None  # not a swarm task
+
+
 def test_plugin_exposes_the_mcp_hooks():
     reg = MagicMock()
     docker_plugin.register_mcp_tools(reg)

tests/plugins/docker/test_swarm_view.py

@@ -1,22 +1,40 @@
 """Unit tests for the swarm-aware container view builder (pure, no DB/ORM)."""
+import json
+from datetime import datetime, timedelta, timezone
 from types import SimpleNamespace
 
-from plugins.docker.swarm_view import build_swarm_services
+from plugins.docker.swarm_view import build_swarm_services, last_task_error, task_failures
 
 NODE_HOSTNAME = {"nA": "docker01", "nB": "docker02", "nC": "docker03"}
 HOST_NAME = {"h1": "docker01", "h2": "docker02"}  # agents run on docker01/02 only
 
 
-def _c(host_id, name, *, service=None, node=None, status="running", cpu=1.0):
+NOW = datetime(2026, 10, 6, 13, 30, tzinfo=timezone.utc)
+
+
+def _c(host_id, name, *, service=None, node=None, status="running", cpu=1.0,
+       task=None, started=None):
     return SimpleNamespace(host_id=host_id, name=name, service_name=service,
                            node_id=node, status=status, cpu_pct=cpu, mem_pct=2.0,
-                           health=None, restart_count=0, container_id=name)
+                           health=None, restart_count=0, container_id=name,
+                           task_id=task, started_at=started)
 
 
-def _svc(name, running, desired, placement, mode="replicated"):
-    import json
+def _svc(name, running, desired, placement, mode="replicated", tasks=None, updated=None):
     return SimpleNamespace(service_name=name, running=running, desired=desired,
-                           mode=mode, placement_json=json.dumps(placement))
+                           mode=mode, placement_json=json.dumps(placement),
+                           tasks_json=json.dumps(tasks or []), updated_at=updated)
+
+
+def _task(tid, *, state, desired="shutdown", ago_min=5, error="", exit_code=None, node="nA",
+          slot=1):
+    at = (NOW - timedelta(minutes=ago_min)).isoformat().replace("+00:00", "Z")
+    return {"task_id": tid, "slot": slot, "node_id": node, "desired_state": desired,
+            "state": state, "message": "", "error": error, "exit_code": exit_code, "at": at}
+
+
+def _view(containers, services):
+    return build_swarm_services(containers, services, NODE_HOSTNAME, HOST_NAME, now=NOW)
 
 
 def test_real_replicas_plus_agentless_ghost():
@@ -77,3 +95,80 @@ def test_down_service_state():
     services = [_svc("idle", 0, 2, [])]
     view = build_swarm_services([], services, NODE_HOSTNAME, HOST_NAME)
     assert view["services"][0]["state"] == "down"
+
+
+# ── task awareness (agent ≥ 1.8.0) ───────────────────────────────────────────
+
+def test_a_task_swarm_replaced_is_not_a_replica():
+    # A stop-first redeploy: the old task was killed (exit 137) and Swarm set its
+    # desired state to shutdown. It is history, not a dead replica.
+    containers = [_c("h1", "app.1.old", service="app", node="nA", status="exited", task="old"),
+                  _c("h2", "app.1.new", service="app", node="nB", task="new")]
+    tasks = [_task("new", state="running", desired="running", ago_min=1, node="nB"),
+             _task("old", state="shutdown", ago_min=2, exit_code=137)]
+    view = _view(containers, [_svc("app", 1, 1, [{"node_id": "nB", "running": 1}], tasks=tasks)])
+    (svc,) = view["services"]
+    assert [r["name"] for r in svc["replicas"]] == ["app.1.new"]
+    assert svc["previous_tasks"] == 1 and svc["state"] == "healthy"
+    assert view["superseded"] == {("h1", "app.1.old")}
+    # A deliberate shutdown is not a failure.
+    assert svc["failures_24h"] == 0 and svc["last_error"] is None and not svc["crash_looping"]
+
+
+def test_without_task_data_the_slot_decides():
+    # An older agent sends no tasks: a stopped container is replaced when its slot
+    # has a running task, or one that started later.
+    t0 = NOW - timedelta(hours=2)
+    containers = [_c("h1", "lib.1.a", service="lib", status="exited", started=t0),
+                  _c("h1", "lib.1.b", service="lib", status="exited", started=t0 + timedelta(minutes=5)),
+                  _c("h1", "lib.1.c", service="lib", started=t0 + timedelta(minutes=10)),
+                  _c("h1", "lib.2.d", service="lib", status="exited", started=t0)]
+    view = _view(containers, [_svc("lib", 1, 2, [])])
+    (svc,) = view["services"]
+    # Slot 2's only task is stopped with nothing newer: still the current task.
+    assert sorted(r["name"] for r in svc["replicas"]) == ["lib.1.c", "lib.2.d"]
+    assert svc["previous_tasks"] == 2
+    assert svc["tasks_known"] is False and svc["failures_1h"] is None
+    assert svc["crash_looping"] is False
+
+
+def test_a_crash_loop_is_counted_across_task_retries():
+    # Each retry is a new task and a new container that dies once; only the
+    # service sees the loop.
+    tasks = [_task("t5", state="starting", desired="running", ago_min=0)]
+    tasks += [_task(f"t{i}", state="failed", ago_min=10 * i, exit_code=1,
+                    error="task: non-zero exit (1)") for i in range(1, 5)]
+    tasks.append(_task("t9", state="failed", ago_min=180, exit_code=1))
+    view = build_swarm_services([], [_svc("db", 0, 1, [], tasks=tasks)], NODE_HOSTNAME, HOST_NAME,
+                                now=NOW, crash_loop_deaths=3)
+    (svc,) = view["services"]
+    assert svc["failures_1h"] == 4 and svc["failures_24h"] == 5
+    assert svc["crash_looping"] is True and svc["state"] == "down"
+    err = svc["last_error"]
+    assert err["error"] == "task: non-zero exit (1)" and err["exit_code"] == 1
+    assert err["node"] == "docker01" and err["state"] == "failed"
+    # The threshold comes from Settings → Thresholds.
+    lax = build_swarm_services([], [_svc("db", 0, 1, [], tasks=tasks)], NODE_HOSTNAME, HOST_NAME,
+                               now=NOW, crash_loop_deaths=5)
+    assert lax["services"][0]["crash_looping"] is False
+
+
+def test_a_rejected_or_stuck_task_explains_itself():
+    rejected = [_task("t1", state="rejected", desired="running", error="No such image: codex:dev")]
+    assert last_task_error(rejected)["error"] == "No such image: codex:dev"
+    # A task Swarm is still trying to place carries its reason too.
+    pending = [_task("t2", state="pending", desired="running", error="no suitable node")]
+    assert last_task_error(pending)["state"] == "pending"
+    assert last_task_error([_task("t3", state="running", desired="running")]) is None
+    assert task_failures(rejected, NOW - timedelta(hours=1)) == 1
+
+
+def test_the_newest_manager_report_wins():
+    # Two managers report the same service; the one updated last (here: the one
+    # with task data) is used.
+    old = _svc("web", 1, 1, [], updated=NOW - timedelta(minutes=5))
+    new = _svc("web", 1, 1, [], tasks=[_task("t1", state="failed", ago_min=3)],
+               updated=NOW - timedelta(seconds=30))
+    (svc,) = _view([], [old, new])["services"]
+    assert svc["tasks_known"] is True and svc["failures_1h"] == 1
+

tests/plugins/host_agent/test_agent_docker.py

@@ -226,6 +226,8 @@ def test_swarm_services_rolls_up_replicas():
          "Status": {"State": "running"}},
     ]
     out = {s["service_name"]: s for s in a._swarm_services(services, tasks)}
+    assert [t["state"] for t in out["web"].pop("tasks")] == ["running", "running", "failed"]
+    assert len(out["agent"].pop("tasks")) == 2
     assert out["web"] == {"service_name": "web", "mode": "replicated",
                           "desired": 3, "running": 2, "image": "nginx:1",
                           "placement": [{"node_id": "n1", "running": 2}]}
@@ -236,6 +238,33 @@ def test_swarm_services_rolls_up_replicas():
                                           {"node_id": "n2", "running": 1}]}
 
 
+def test_swarm_task_keeps_why_it_stopped():
+    t = a._swarm_task({
+        "ID": "t1", "Slot": 1, "NodeID": "n2", "DesiredState": "Shutdown",
+        "Status": {"State": "failed", "Timestamp": "2026-10-06T13:19:51.5Z",
+                   "Message": "started", "Err": "task: non-zero exit (1)" + "x" * 400,
+                   "ContainerStatus": {"ContainerID": "c1", "ExitCode": 1}}})
+    assert t["task_id"] == "t1" and t["slot"] == 1 and t["node_id"] == "n2"
+    assert t["desired_state"] == "shutdown" and t["state"] == "failed" and t["exit_code"] == 1
+    assert t["at"] == "2026-10-06T13:19:51.5Z" and t["message"] == "started"
+    assert t["error"].startswith("task: non-zero exit (1)") and len(t["error"]) == 300
+    # A task that never got a container has no exit code.
+    bare = a._swarm_task({"ID": "t2", "DesiredState": "running",
+                          "Status": {"State": "rejected", "Err": "No such image: x"}})
+    assert bare["exit_code"] is None and bare["slot"] is None and bare["error"] == "No such image: x"
+
+
+def test_swarm_services_keeps_the_newest_tasks():
+    services = [{"ID": "s", "Spec": {"Name": "db", "Mode": {"Replicated": {"Replicas": 1}}}}]
+    tasks = [{"ID": f"t{i}", "ServiceID": "s", "Slot": 1, "DesiredState": "shutdown",
+              "Status": {"State": "failed", "Timestamp": f"2026-10-06T13:{i:02d}:00Z"}}
+             for i in range(a.SWARM_TASKS_PER_SERVICE + 5)]
+    (svc,) = a._swarm_services(services, tasks)
+    ids = [t["task_id"] for t in svc["tasks"]]
+    newest = a.SWARM_TASKS_PER_SERVICE + 4
+    assert len(ids) == a.SWARM_TASKS_PER_SERVICE and ids[0] == f"t{newest}"
+
+
 def test_swarm_nodes_normalises_fields():
     nodes = [
         {"ID": "n1", "Description": {"Hostname": "mgr"},