feat(docker): Swarm task awareness — replaced tasks, service crash loops, last task error

Milestone 465 "Swarm task awareness". Fixes three defects from the live MCP
review #4973:
- #5008 "Swarm task history read as live state"
- #5009 "Crash-loop detection is blind to Swarm services"
- #5011 "Swarm task errors are not captured"

Agent and ingest (agent 1.8.0):
- A manager's agent now reports each service's newest tasks: slot, node,
  desired and current state, Swarm's Err/Message, exit code and time.
- They are stored in docker_swarm_services.tasks_json (migration
  docker_010_swarm_tasks).

Swarm view (pure):
- A task container is "replaced" when Swarm set its task's desired state off
  `running`. Without the task on record, a newer or running task in the same
  slot decides.
- Replaced containers are no longer replicas or crashes.
- Each service gets, from the task records:
  - failures_1h and failures_24h (failed or rejected tasks);
  - crash_looping (Settings → Thresholds count);
  - last_error, in Swarm's own words.

MCP:
- New findings: swarm_service_crash_looping (critical) and swarm_task_failed
  (warning, on a healthy service with a failure in 24h).
- Down and degraded findings name the last task error.
- list_swarm_services carries the failure fields.
- Container rows mark swarm_replaced and give a replaced task no problems.

Web:
- The containers page swarm cards and the Swarm topology page show crash
  loops, recent failures and the last task error.
- The dashboard widget flags a crash loop, and its "failed (24h)" count no
  longer counts a rolling update's killed task.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

tests/plugins/docker/test_mcp_tools.py

@@ -73,6 +73,40 @@ def test_service_findings_by_state():
     assert got == {"b": "warning", "c": "critical"}
 
 
+def test_service_findings_name_the_crash_loop_and_swarms_error():
+    err = {"error": "task: non-zero exit (1)", "message": "started", "state": "failed",
+           "exit_code": 1, "node": "docker03"}
+    svcs = [{"name": "db", "state": "down", "running": 0, "desired": 1, "crash_looping": True,
+             "failures_1h": 4, "failures_24h": 5, "last_error": err},
+            {"name": "app", "state": "down", "running": 0, "desired": 1, "crash_looping": False,
+             "failures_1h": 0, "failures_24h": 0,
+             "last_error": {"error": "No such image: app:dev", "state": "rejected",
+                            "exit_code": None, "node": None, "message": None}},
+            {"name": "web", "state": "healthy", "running": 2, "desired": 2, "crash_looping": False,
+             "failures_1h": 0, "failures_24h": 1, "last_error": err},
+            {"name": "ok", "state": "healthy", "running": 1, "desired": 1, "crash_looping": False,
+             "failures_1h": 0, "failures_24h": 0, "last_error": None}]
+    got = {f["subject"]: f for f in t.service_findings(svcs)}
+    assert set(got) == {"db", "app", "web"}
+    db = got["db"]
+    assert db["kind"] == "swarm_service_crash_looping" and db["severity"] == "critical"
+    assert "4 tasks failed in the last hour" in db["summary"]
+    assert "last task error: task: non-zero exit (1), exit 1 on docker03" in db["summary"]
+    # Down without a loop still names why: here Swarm could not even start it.
+    assert got["app"]["kind"] == "swarm_service_down"
+    assert got["app"]["summary"].endswith("last task error: No such image: app:dev")
+    # Recovered, but a task died in the last day: worth knowing, not critical.
+    assert got["web"]["kind"] == "swarm_task_failed" and got["web"]["severity"] == "warning"
+    assert "had 1 failed task in the last 24h" in got["web"]["summary"]
+
+
+def test_a_replaced_swarm_task_is_marked_and_judged_on_its_service():
+    task = _c(name="app.1.old", status="exited", exit_code=137, service_name="app")
+    row = t.container_row(task, {}, [], NOW, replaced=True)
+    assert row["swarm_replaced"] is True and row["problems"] == []
+    assert t.container_row(_c(), {}, [], NOW)["swarm_replaced"] is None  # not a swarm task
+
+
 def test_plugin_exposes_the_mcp_hooks():
     reg = MagicMock()
     docker_plugin.register_mcp_tools(reg)

tests/plugins/docker/test_swarm_view.py

@@ -1,22 +1,40 @@
 """Unit tests for the swarm-aware container view builder (pure, no DB/ORM)."""
+import json
+from datetime import datetime, timedelta, timezone
 from types import SimpleNamespace
 
-from plugins.docker.swarm_view import build_swarm_services
+from plugins.docker.swarm_view import build_swarm_services, last_task_error, task_failures
 
 NODE_HOSTNAME = {"nA": "docker01", "nB": "docker02", "nC": "docker03"}
 HOST_NAME = {"h1": "docker01", "h2": "docker02"}  # agents run on docker01/02 only
 
 
-def _c(host_id, name, *, service=None, node=None, status="running", cpu=1.0):
+NOW = datetime(2026, 10, 6, 13, 30, tzinfo=timezone.utc)
+
+
+def _c(host_id, name, *, service=None, node=None, status="running", cpu=1.0,
+       task=None, started=None):
     return SimpleNamespace(host_id=host_id, name=name, service_name=service,
                            node_id=node, status=status, cpu_pct=cpu, mem_pct=2.0,
-                           health=None, restart_count=0, container_id=name)
+                           health=None, restart_count=0, container_id=name,
+                           task_id=task, started_at=started)
 
 
-def _svc(name, running, desired, placement, mode="replicated"):
-    import json
+def _svc(name, running, desired, placement, mode="replicated", tasks=None, updated=None):
     return SimpleNamespace(service_name=name, running=running, desired=desired,
-                           mode=mode, placement_json=json.dumps(placement))
+                           mode=mode, placement_json=json.dumps(placement),
+                           tasks_json=json.dumps(tasks or []), updated_at=updated)
+
+
+def _task(tid, *, state, desired="shutdown", ago_min=5, error="", exit_code=None, node="nA",
+          slot=1):
+    at = (NOW - timedelta(minutes=ago_min)).isoformat().replace("+00:00", "Z")
+    return {"task_id": tid, "slot": slot, "node_id": node, "desired_state": desired,
+            "state": state, "message": "", "error": error, "exit_code": exit_code, "at": at}
+
+
+def _view(containers, services):
+    return build_swarm_services(containers, services, NODE_HOSTNAME, HOST_NAME, now=NOW)
 
 
 def test_real_replicas_plus_agentless_ghost():
@@ -77,3 +95,80 @@ def test_down_service_state():
     services = [_svc("idle", 0, 2, [])]
     view = build_swarm_services([], services, NODE_HOSTNAME, HOST_NAME)
     assert view["services"][0]["state"] == "down"
+
+
+# ── task awareness (agent ≥ 1.8.0) ───────────────────────────────────────────
+
+def test_a_task_swarm_replaced_is_not_a_replica():
+    # A stop-first redeploy: the old task was killed (exit 137) and Swarm set its
+    # desired state to shutdown. It is history, not a dead replica.
+    containers = [_c("h1", "app.1.old", service="app", node="nA", status="exited", task="old"),
+                  _c("h2", "app.1.new", service="app", node="nB", task="new")]
+    tasks = [_task("new", state="running", desired="running", ago_min=1, node="nB"),
+             _task("old", state="shutdown", ago_min=2, exit_code=137)]
+    view = _view(containers, [_svc("app", 1, 1, [{"node_id": "nB", "running": 1}], tasks=tasks)])
+    (svc,) = view["services"]
+    assert [r["name"] for r in svc["replicas"]] == ["app.1.new"]
+    assert svc["previous_tasks"] == 1 and svc["state"] == "healthy"
+    assert view["superseded"] == {("h1", "app.1.old")}
+    # A deliberate shutdown is not a failure.
+    assert svc["failures_24h"] == 0 and svc["last_error"] is None and not svc["crash_looping"]
+
+
+def test_without_task_data_the_slot_decides():
+    # An older agent sends no tasks: a stopped container is replaced when its slot
+    # has a running task, or one that started later.
+    t0 = NOW - timedelta(hours=2)
+    containers = [_c("h1", "lib.1.a", service="lib", status="exited", started=t0),
+                  _c("h1", "lib.1.b", service="lib", status="exited", started=t0 + timedelta(minutes=5)),
+                  _c("h1", "lib.1.c", service="lib", started=t0 + timedelta(minutes=10)),
+                  _c("h1", "lib.2.d", service="lib", status="exited", started=t0)]
+    view = _view(containers, [_svc("lib", 1, 2, [])])
+    (svc,) = view["services"]
+    # Slot 2's only task is stopped with nothing newer: still the current task.
+    assert sorted(r["name"] for r in svc["replicas"]) == ["lib.1.c", "lib.2.d"]
+    assert svc["previous_tasks"] == 2
+    assert svc["tasks_known"] is False and svc["failures_1h"] is None
+    assert svc["crash_looping"] is False
+
+
+def test_a_crash_loop_is_counted_across_task_retries():
+    # Each retry is a new task and a new container that dies once; only the
+    # service sees the loop.
+    tasks = [_task("t5", state="starting", desired="running", ago_min=0)]
+    tasks += [_task(f"t{i}", state="failed", ago_min=10 * i, exit_code=1,
+                    error="task: non-zero exit (1)") for i in range(1, 5)]
+    tasks.append(_task("t9", state="failed", ago_min=180, exit_code=1))
+    view = build_swarm_services([], [_svc("db", 0, 1, [], tasks=tasks)], NODE_HOSTNAME, HOST_NAME,
+                                now=NOW, crash_loop_deaths=3)
+    (svc,) = view["services"]
+    assert svc["failures_1h"] == 4 and svc["failures_24h"] == 5
+    assert svc["crash_looping"] is True and svc["state"] == "down"
+    err = svc["last_error"]
+    assert err["error"] == "task: non-zero exit (1)" and err["exit_code"] == 1
+    assert err["node"] == "docker01" and err["state"] == "failed"
+    # The threshold comes from Settings → Thresholds.
+    lax = build_swarm_services([], [_svc("db", 0, 1, [], tasks=tasks)], NODE_HOSTNAME, HOST_NAME,
+                               now=NOW, crash_loop_deaths=5)
+    assert lax["services"][0]["crash_looping"] is False
+
+
+def test_a_rejected_or_stuck_task_explains_itself():
+    rejected = [_task("t1", state="rejected", desired="running", error="No such image: codex:dev")]
+    assert last_task_error(rejected)["error"] == "No such image: codex:dev"
+    # A task Swarm is still trying to place carries its reason too.
+    pending = [_task("t2", state="pending", desired="running", error="no suitable node")]
+    assert last_task_error(pending)["state"] == "pending"
+    assert last_task_error([_task("t3", state="running", desired="running")]) is None
+    assert task_failures(rejected, NOW - timedelta(hours=1)) == 1
+
+
+def test_the_newest_manager_report_wins():
+    # Two managers report the same service; the one updated last (here: the one
+    # with task data) is used.
+    old = _svc("web", 1, 1, [], updated=NOW - timedelta(minutes=5))
+    new = _svc("web", 1, 1, [], tasks=[_task("t1", state="failed", ago_min=3)],
+               updated=NOW - timedelta(seconds=30))
+    (svc,) = _view([], [old, new])["services"]
+    assert svc["tasks_known"] is True and svc["failures_1h"] == 1
+

tests/plugins/host_agent/test_agent_docker.py

@@ -226,6 +226,8 @@ def test_swarm_services_rolls_up_replicas():
          "Status": {"State": "running"}},
     ]
     out = {s["service_name"]: s for s in a._swarm_services(services, tasks)}
+    assert [t["state"] for t in out["web"].pop("tasks")] == ["running", "running", "failed"]
+    assert len(out["agent"].pop("tasks")) == 2
     assert out["web"] == {"service_name": "web", "mode": "replicated",
                           "desired": 3, "running": 2, "image": "nginx:1",
                           "placement": [{"node_id": "n1", "running": 2}]}
@@ -236,6 +238,33 @@ def test_swarm_services_rolls_up_replicas():
                                           {"node_id": "n2", "running": 1}]}
 
 
+def test_swarm_task_keeps_why_it_stopped():
+    t = a._swarm_task({
+        "ID": "t1", "Slot": 1, "NodeID": "n2", "DesiredState": "Shutdown",
+        "Status": {"State": "failed", "Timestamp": "2026-10-06T13:19:51.5Z",
+                   "Message": "started", "Err": "task: non-zero exit (1)" + "x" * 400,
+                   "ContainerStatus": {"ContainerID": "c1", "ExitCode": 1}}})
+    assert t["task_id"] == "t1" and t["slot"] == 1 and t["node_id"] == "n2"
+    assert t["desired_state"] == "shutdown" and t["state"] == "failed" and t["exit_code"] == 1
+    assert t["at"] == "2026-10-06T13:19:51.5Z" and t["message"] == "started"
+    assert t["error"].startswith("task: non-zero exit (1)") and len(t["error"]) == 300
+    # A task that never got a container has no exit code.
+    bare = a._swarm_task({"ID": "t2", "DesiredState": "running",
+                          "Status": {"State": "rejected", "Err": "No such image: x"}})
+    assert bare["exit_code"] is None and bare["slot"] is None and bare["error"] == "No such image: x"
+
+
+def test_swarm_services_keeps_the_newest_tasks():
+    services = [{"ID": "s", "Spec": {"Name": "db", "Mode": {"Replicated": {"Replicas": 1}}}}]
+    tasks = [{"ID": f"t{i}", "ServiceID": "s", "Slot": 1, "DesiredState": "shutdown",
+              "Status": {"State": "failed", "Timestamp": f"2026-10-06T13:{i:02d}:00Z"}}
+             for i in range(a.SWARM_TASKS_PER_SERVICE + 5)]
+    (svc,) = a._swarm_services(services, tasks)
+    ids = [t["task_id"] for t in svc["tasks"]]
+    newest = a.SWARM_TASKS_PER_SERVICE + 4
+    assert len(ids) == a.SWARM_TASKS_PER_SERVICE and ids[0] == f"t{newest}"
+
+
 def test_swarm_nodes_normalises_fields():
     nodes = [
         {"ID": "n1", "Description": {"Hostname": "mgr"},