feat(docker): Swarm task awareness — replaced tasks, service crash loops, last task error

Milestone 465 "Swarm task awareness". Fixes three defects from the live MCP
review #4973:
- #5008 "Swarm task history read as live state"
- #5009 "Crash-loop detection is blind to Swarm services"
- #5011 "Swarm task errors are not captured"

Agent and ingest (agent 1.8.0):
- A manager's agent now reports each service's newest tasks: slot, node,
  desired and current state, Swarm's Err/Message, exit code and time.
- They are stored in docker_swarm_services.tasks_json (migration
  docker_010_swarm_tasks).

Swarm view (pure):
- A task container is "replaced" when Swarm set its task's desired state off
  `running`. Without the task on record, a newer or running task in the same
  slot decides.
- Replaced containers are no longer replicas or crashes.
- Each service gets, from the task records:
  - failures_1h and failures_24h (failed or rejected tasks);
  - crash_looping (Settings → Thresholds count);
  - last_error, in Swarm's own words.

MCP:
- New findings: swarm_service_crash_looping (critical) and swarm_task_failed
  (warning, on a healthy service with a failure in 24h).
- Down and degraded findings name the last task error.
- list_swarm_services carries the failure fields.
- Container rows mark swarm_replaced and give a replaced task no problems.

Web:
- The containers page swarm cards and the Swarm topology page show crash
  loops, recent failures and the last task error.
- The dashboard widget flags a crash loop, and its "failed (24h)" count no
  longer counts a rolling update's killed task.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

tests/integration/test_mcp_plugin_tools.py

@@ -9,6 +9,7 @@ from __future__ import annotations
 
 import asyncio
 import importlib
+import json
 import os
 import uuid
 from datetime import datetime, timedelta, timezone
@@ -57,14 +58,18 @@ def test_docker_and_agent_tools_over_seeded_rows(app):
     hid, hid2 = str(uuid.uuid4()), str(uuid.uuid4())
     host_name, quiet_host = f"dock-{tag}", f"quiet-{tag}"
     web, db_, job, svc = f"web-{tag}", f"db-{tag}", f"job-{tag}", f"svc-{tag}"
+    replaced = f"{svc}.1.old"
+
+    def _ago(minutes):
+        return (now - timedelta(minutes=minutes)).isoformat().replace("+00:00", "Z")
 
     container_sql = text("""
         INSERT INTO docker_containers
             (host_id, name, container_id, image, status, cpu_pct, mem_pct, mem_usage_bytes,
              mem_limit_bytes, restart_count, ports_json, started_at, scraped_at, health,
-             exit_code, oom_killed, compose_project, service_name)
+             exit_code, oom_killed, compose_project, service_name, task_id)
         VALUES (:h, :n, :cid, 'img:1', :st, 5.0, 10.0, 1000, 10000, :rc, '[]', :started, :now,
-                :health, :exit, false, :proj, :svc)
+                :health, :exit, false, :proj, :svc, :task)
     """)
 
     async def _seed():
@@ -73,20 +78,23 @@ def test_docker_and_agent_tools_over_seeded_rows(app):
                 s.add(Host(id=hid, name=host_name, address="10.8.8.8"))
                 s.add(Host(id=hid2, name=quiet_host, address="10.8.8.9"))
             async with s.begin():
-                for name, st, health, exit_code, proj, service in (
-                    (web, "running", "healthy", None, f"stack-{tag}", None),
-                    (db_, "running", "unhealthy", None, f"stack-{tag}", None),
-                    (job, "exited", None, 1, None, None),
-                    (f"{svc}.1.abc", "running", None, None, None, svc),
+                for name, st, health, exit_code, proj, service, task in (
+                    (web, "running", "healthy", None, f"stack-{tag}", None, None),
+                    (db_, "running", "unhealthy", None, f"stack-{tag}", None, None),
+                    (job, "exited", None, 1, None, None, None),
+                    (f"{svc}.1.abc", "running", None, None, None, svc, f"abc-{tag}"),
+                    # The task a rolling update replaced: killed (137), not crashed.
+                    (replaced, "exited", None, 137, None, svc, f"old-{tag}"),
                 ):
                     await s.execute(container_sql, {
                         "h": hid, "n": name, "cid": uuid.uuid4().hex, "st": st, "rc": 0,
                         "started": now - timedelta(hours=2), "now": now, "health": health,
-                        "exit": exit_code, "proj": proj, "svc": service})
-                await s.execute(text(
-                    "INSERT INTO docker_events (id, host_id, container_name, event, detail, at) "
-                    "VALUES (:id, :h, :n, 'die', 'exit 1', :at)"),
-                    {"id": str(uuid.uuid4()), "h": hid, "n": job, "at": now - timedelta(minutes=10)})
+                        "exit": exit_code, "proj": proj, "svc": service, "task": task})
+                for name in (job, replaced):
+                    await s.execute(text(
+                        "INSERT INTO docker_events (id, host_id, container_name, event, detail, at) "
+                        "VALUES (:id, :h, :n, 'die', 'exit 1', :at)"),
+                        {"id": str(uuid.uuid4()), "h": hid, "n": name, "at": now - timedelta(minutes=10)})
                 for i, (stream, line) in enumerate((
                         ("stdout", "starting up"), ("stdout", "connected to db"),
                         ("stderr", "ERROR: 100% of disk_used"), ("stdout", "retrying"),
@@ -104,8 +112,18 @@ def test_docker_and_agent_tools_over_seeded_rows(app):
                          "t": now - timedelta(minutes=30 - i * 10), "c": cpu})
                 await s.execute(text(
                     "INSERT INTO docker_swarm_services (host_id, service_name, mode, desired, running, "
-                    "placement_json, updated_at) VALUES (:h, :n, 'replicated', 3, 1, '[]', :now)"),
-                    {"h": hid, "n": svc, "now": now})
+                    "placement_json, tasks_json, updated_at) "
+                    "VALUES (:h, :n, 'replicated', 3, 1, '[]', :tasks, :now)"),
+                    {"h": hid, "n": svc, "now": now, "tasks": json.dumps([
+                        {"task_id": f"abc-{tag}", "slot": 1, "node_id": f"node-{tag}",
+                         "desired_state": "running", "state": "running", "message": "started",
+                         "error": "", "exit_code": None, "at": _ago(9)},
+                        {"task_id": f"bad-{tag}", "slot": 2, "node_id": f"node-{tag}",
+                         "desired_state": "shutdown", "state": "failed", "message": "started",
+                         "error": "task: non-zero exit (1)", "exit_code": 1, "at": _ago(10)},
+                        {"task_id": f"old-{tag}", "slot": 1, "node_id": f"node-{tag}",
+                         "desired_state": "shutdown", "state": "shutdown", "message": "shutdown",
+                         "error": "", "exit_code": 137, "at": _ago(11)}])})
                 await s.execute(text(
                     "INSERT INTO docker_swarm_nodes (host_id, node_id, hostname, role, availability, "
                     "status, leader, updated_at) VALUES (:h, :nid, :hn, 'manager', 'active', 'ready', "
@@ -154,7 +172,7 @@ def test_docker_and_agent_tools_over_seeded_rows(app):
         crashed = await dtools.get_container(job, host=host_name)
         assert [e["event"] for e in crashed["events"]] == ["die"]
         # A unique part of a long swarm task name is enough.
-        assert (await dtools.get_container(f"{svc}.1"))["swarm_service"] == svc
+        assert (await dtools.get_container(f"{svc}.1.ab"))["swarm_service"] == svc
         with pytest.raises(ValueError, match="list_containers"):
             await dtools.get_container(f"nope-{tag}")
 
@@ -180,6 +198,15 @@ def test_docker_and_agent_tools_over_seeded_rows(app):
         sw = await dtools.list_swarm_services(limit=500)
         s = next(x for x in sw["services"] if x["service_name"] == svc)
         assert (s["state"], s["running"], s["desired"]) == ("degraded", 1, 3)
+        # Swarm's task records: the replaced task is history, the failed one counts.
+        assert [r["container_name"] for r in s["replicas"]] == [f"{svc}.1.abc"]
+        assert s["previous_tasks"] == 1 and s["tasks_known"] is True
+        assert (s["failures_1h"], s["failures_24h"], s["crash_looping"]) == (1, 1, False)
+        assert s["last_error"]["error"] == "task: non-zero exit (1)"
+        assert s["last_error"]["node"] == host_name and s["last_error"]["age_s"] >= 600
+        old = (await dtools.list_containers(host=host_name, status="exited", limit=500))["containers"]
+        old = next(c for c in old if c["container_name"] == replaced)
+        assert old["swarm_replaced"] is True and old["problems"] == []
         assert any(n["node_id"] == f"node-{tag}" and n["leader"] for n in sw["nodes"])
         disk = (await dtools.get_docker_disk_usage(host=host_name))["hosts"]
         assert disk[0]["images_reclaimable_bytes"] == 3000
@@ -191,10 +218,23 @@ def test_docker_and_agent_tools_over_seeded_rows(app):
         assert ("container_unhealthy", db_) in found
         assert ("container_exited_nonzero", job) in found
         assert ("swarm_service_degraded", svc) in found
+        assert not any(subject == replaced for _, subject in found)
         assert not any(subject == web for _, subject in found)
         assert ("agent_never_reported", quiet_host) in agent_found
         assert not any(subject == host_name for _, subject in agent_found)
 
+        # ── the web pages say the same ─────────────────────────────────────
+        client = app.test_client()
+        async with client.session_transaction() as sess:
+            sess["user_id"], sess["user_role"], sess["username"] = "u", "viewer", "viewer"
+        topo = await (await client.get("/plugins/docker/swarm")).get_data(as_text=True)
+        assert svc in topo and "1 task failed in the last 24h" in topo
+        assert "task: non-zero exit (1)" in topo
+        cards = await (await client.get("/plugins/docker/rows")).get_data(as_text=True)
+        assert f"{svc}.1.abc" in cards and replaced not in cards
+        assert "task: non-zero exit (1)" in cards
+        assert (await client.get("/plugins/docker/widget")).status_code == 200
+
         # ── host agents ────────────────────────────────────────────────────
         agents = {a["host_name"]: a for a in (await atools.list_agents())["agents"]}
         assert agents[host_name]["state"] == "reporting" and agents[host_name]["agent_version"] == "1.5.0"