feat(docker): Swarm task awareness — replaced tasks, service crash loops, last task error

Milestone 465 "Swarm task awareness". Fixes three defects from the live MCP
review #4973:
- #5008 "Swarm task history read as live state"
- #5009 "Crash-loop detection is blind to Swarm services"
- #5011 "Swarm task errors are not captured"

Agent and ingest (agent 1.8.0):
- A manager's agent now reports each service's newest tasks: slot, node,
  desired and current state, Swarm's Err/Message, exit code and time.
- They are stored in docker_swarm_services.tasks_json (migration
  docker_010_swarm_tasks).

Swarm view (pure):
- A task container is "replaced" when Swarm set its task's desired state off
  `running`. Without the task on record, a newer or running task in the same
  slot decides.
- Replaced containers are no longer replicas or crashes.
- Each service gets, from the task records:
  - failures_1h and failures_24h (failed or rejected tasks);
  - crash_looping (Settings → Thresholds count);
  - last_error, in Swarm's own words.

MCP:
- New findings: swarm_service_crash_looping (critical) and swarm_task_failed
  (warning, on a healthy service with a failure in 24h).
- Down and degraded findings name the last task error.
- list_swarm_services carries the failure fields.
- Container rows mark swarm_replaced and give a replaced task no problems.

Web:
- The containers page swarm cards and the Swarm topology page show crash
  loops, recent failures and the last task error.
- The dashboard widget flags a crash loop, and its "failed (24h)" count no
  longer counts a rolling update's killed task.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

plugins/docker/mcp_tools.py

@@ -26,10 +26,10 @@ from steward.models.users import UserRole
 from . import service as docker_svc
 from .dedup import dedup_by_container_id
 from .models import DockerDiskUsage, DockerSwarmNode, DockerSwarmService
-from .swarm_view import build_swarm_services
+from .swarm_view import build_swarm_services, replaced_tasks, task_time
 
 PROBLEMS = ("restarting", "crash_looping", "unhealthy", "oom_killed", "exited_nonzero")
-CRASH_LOOP_DEATHS = 3        # default; Settings → Thresholds sets it
+CRASH_LOOP_DEATHS = docker_svc.CRASH_LOOP_DEATHS
 LOG_LINE_CHARS = 2000
 
 
@@ -61,7 +61,8 @@ def cut_middle(text: str, limit: int = LOG_LINE_CHARS) -> str:
     return f"{text[:keep]} [… {len(text) - 2 * keep} characters omitted …] {text[-keep:]}"
 
 
-def container_row(c, host_names: dict, problems: list[str], now: datetime) -> dict:
+def container_row(c, host_names: dict, problems: list[str], now: datetime,
+                  replaced: bool = False) -> dict:
     running = (c.status or "").lower() == "running"
     return {
         "container_name": c.name,
@@ -82,6 +83,8 @@ def container_row(c, host_names: dict, problems: list[str], now: datetime) -> di
         "uptime_s": age_s(c.started_at, now) if running and c.started_at else None,
         "compose_project": c.compose_project,
         "swarm_service": c.service_name,
+        # A swarm task Swarm has replaced: history, judged on its service.
+        "swarm_replaced": replaced if c.service_name else None,
         "reported_at": iso(c.scraped_at),
         "reported_age_s": age_s(c.scraped_at, now),
     }
@@ -110,15 +113,41 @@ def container_findings(rows: list[dict]) -> list[dict]:
     return out
 
 
+def task_error_text(err: dict | None) -> str:
+    """Swarm's own account of a task's end, as one clause."""
+    if not err:
+        return ""
+    why = err.get("error") or err.get("message") or err.get("state") or "unknown"
+    code = f", exit {err['exit_code']}" if err.get("exit_code") not in (None, 0) else ""
+    where = f" on {err['node']}" if err.get("node") else ""
+    return f"{why}{code}{where}"
+
+
 def service_findings(services: list[dict]) -> list[dict]:
+    """Swarm services that are failing: crash-looping, under-replicated, or that
+    lost a task to an error in the last 24h. Each names Swarm's last task error."""
     out = []
     for s in services:
-        if s["state"] == "healthy":
-            continue
-        out.append(finding(
-            "critical" if s["state"] == "down" else "warning", f"swarm_service_{s['state']}",
-            s["name"], f"swarm service {s['name']} has {s['running']} of {s['desired']} replicas running",
-            ids={"service_name": s["name"]}, next="list_swarm_services()"))
+        replicas = f"{s['running']} of {s['desired']} replicas running"
+        last = task_error_text(s.get("last_error"))
+        tail = f"; last task error: {last}" if last else ""
+        ids = {"service_name": s["name"]}
+        if s.get("crash_looping"):
+            out.append(finding(
+                "critical", "swarm_service_crash_looping", s["name"],
+                f"swarm service {s['name']} keeps failing: {s['failures_1h']} tasks failed in the "
+                f"last hour, {replicas}{tail}", ids=ids, next="list_swarm_services()"))
+        elif s["state"] != "healthy":
+            out.append(finding(
+                "critical" if s["state"] == "down" else "warning", f"swarm_service_{s['state']}",
+                s["name"], f"swarm service {s['name']} has {replicas}{tail}",
+                ids=ids, next="list_swarm_services()"))
+        elif s.get("failures_24h"):
+            n = s["failures_24h"]
+            out.append(finding(
+                "warning", "swarm_task_failed", s["name"],
+                f"swarm service {s['name']} had {n} failed task{'s' if n != 1 else ''} in the last "
+                f"24h and is now {replicas}{tail}", ids=ids, next="list_swarm_services()"))
     return out
 
 
@@ -129,38 +158,49 @@ async def _host_names(db) -> dict[str, str]:
 
 
 def _crash_loop_deaths() -> int:
-    th = (mcp_app().config.get("THRESHOLDS") or {}).get("crash_loop") or {}
-    return int(th.get("warn") or CRASH_LOOP_DEATHS)
+    return docker_svc.crash_loop_deaths(mcp_app().config)
+
+
+async def _replaced(db, containers) -> set[tuple]:
+    services = list((await db.execute(select(DockerSwarmService))).scalars())
+    return replaced_tasks(containers, services)
 
 
 async def _rows(db, now: datetime) -> list[dict]:
+    """Every container with its problems. A swarm task Swarm replaced has none:
+    its failure, if it was one, belongs to its service (service_findings)."""
     containers = dedup_by_container_id(await docker_svc.list_containers(db))
+    replaced = await _replaced(db, containers)
     failures = await docker_svc.recent_failures_by_container(db, now - timedelta(hours=24))
     hour_ago = now - timedelta(hours=1)
     host_names = await _host_names(db)
     deaths = _crash_loop_deaths()
     rows = []
     for c in containers:
+        gone = (c.host_id, c.name) in replaced
         events = failures.get((c.host_id, c.name), [])
         recent = sum(1 for e in events if e.at >= hour_ago)
-        rows.append(container_row(c, host_names, container_problems(c, recent, bool(events), deaths), now))
+        problems = [] if gone else container_problems(c, recent, bool(events), deaths)
+        rows.append(container_row(c, host_names, problems, now, replaced=gone))
     return rows
 
 
-async def _swarm(db):
+async def _swarm(db, now: datetime):
     containers = dedup_by_container_id(await docker_svc.list_containers(db))
     services = list((await db.execute(select(DockerSwarmService))).scalars())
     nodes = list((await db.execute(select(DockerSwarmNode))).scalars())
     host_names = await _host_names(db)
     node_hostname = {n.node_id: n.hostname for n in nodes if n.hostname}
-    built = build_swarm_services(containers, services, node_hostname, host_names)
+    built = build_swarm_services(containers, services, node_hostname, host_names,
+                                 now=now, crash_loop_deaths=_crash_loop_deaths())
     return built["services"], nodes, host_names
 
 
 async def mcp_overview(db) -> list[dict]:
     """get_overview findings: failing containers and under-replicated swarm services."""
-    rows = await _rows(db, now_utc())
-    services, _, _ = await _swarm(db)
+    now = now_utc()
+    rows = await _rows(db, now)
+    services, _, _ = await _swarm(db, now)
     return container_findings(rows) + service_findings(services)
 
 
@@ -184,6 +224,11 @@ async def list_containers(host: str = "", status: str = "", problem: str = "",
     Containers with problems come first. Each row's `problems` lists what is
     wrong, most severe first. `reported_age_s` is how old the agent's last
     report is, so a large value means the row may be stale.
+
+    A swarm task's container that Swarm has replaced (a rolling update, or a
+    retry after a failure) has `swarm_replaced: true` and no problems: Swarm
+    restarts a task as a new container, so failures are counted on the
+    service — list_swarm_services shows them.
     """
     if problem and problem != "any" and problem not in PROBLEMS:
         raise ValueError(f"problem {problem!r} is not 'any' or one of {', '.join(PROBLEMS)}.")
@@ -254,6 +299,7 @@ async def get_container(container: str, host: str = "", range: str = "24h", even
     now = now_utc()
     async with db_session() as db:
         c = await _pick_container(db, container, host)
+        gone = (c.host_id, c.name) in await _replaced(db, await docker_svc.list_containers(db))
         failures = (await docker_svc.recent_failures_by_container(
             db, now - timedelta(hours=24))).get((c.host_id, c.name), [])
         host_names = await _host_names(db)
@@ -265,9 +311,9 @@ async def get_container(container: str, host: str = "", range: str = "24h", even
         ports = json.loads(c.ports_json or "[]")
     except ValueError:
         ports = []
+    problems = [] if gone else container_problems(c, recent, bool(failures), _crash_loop_deaths())
     return {
-        **container_row(c, host_names,
-                        container_problems(c, recent, bool(failures), _crash_loop_deaths()), now),
+        **container_row(c, host_names, problems, now, replaced=gone),
         "container_id": c.container_id,
         "ports": ports,
         "net_rx_bytes": c.net_rx_bytes, "net_tx_bytes": c.net_tx_bytes,
@@ -285,15 +331,32 @@ async def list_swarm_services(limit: int = 100) -> dict:
     """Docker Swarm services with their replicas per node, and the swarm's nodes.
 
     `state` is "healthy" (all desired replicas running), "degraded" (some) or
-    "down" (none). Each replica names its host and container status; a replica
-    marked `ghost` runs on a node with no Steward agent, so Steward knows only
-    how many tasks are placed there. `nodes` lists every swarm node once, with
-    its role (manager/worker), availability and status.
+    "down" (none). `replicas` are the service's current tasks: each names its
+    host and container status, and one marked `ghost` runs on a node with no
+    Steward agent, so Steward knows only how many tasks are placed there.
+    Tasks Swarm replaced (rolling updates, retries) are left out and counted
+    in `previous_tasks`.
+
+    From Swarm's own task records (a manager's agent 1.8.0 or later;
+    `tasks_known: false` and nulls otherwise):
+    - `failures_1h` / `failures_24h`: tasks that failed or were rejected.
+      Swarm retries a failed task as a new one, so this, not any container's
+      restart count, is how a crash loop shows. Swarm keeps about 5 old tasks
+      per replica, which bounds the count.
+    - `crash_looping`: failures_1h reached the crash-loop count set under
+      Settings → Thresholds (default 3).
+    - `last_error`: the newest task that failed, was rejected, or carries an
+      error while Swarm retries it — `error` is Swarm's own text (e.g. "No
+      such image", "task: non-zero exit (1)"), with `exit_code`, `node`, `at`.
+
+    Services with a problem come first. `nodes` lists every swarm node once,
+    with its role (manager/worker), availability and status.
     limit: default 100, max 500.
     """
     limit = clamp_limit(limit, 100, 500)
+    now = now_utc()
     async with db_session() as db:
-        services, nodes, host_names = await _swarm(db)
+        services, nodes, host_names = await _swarm(db, now)
     seen, node_rows = set(), []
     for n in sorted(nodes, key=lambda n: n.updated_at, reverse=True):
         if n.node_id in seen:
@@ -304,10 +367,26 @@ async def list_swarm_services(limit: int = 100) -> dict:
                           "reported_by": host_names.get(n.host_id)})
     node_rows.sort(key=lambda n: n["hostname"].lower())
     order = {"down": 0, "degraded": 1, "healthy": 2}
+
+    def _rank(s):
+        return (not s["crash_looping"], order.get(s["state"], 3), not s["failures_24h"], s["name"])
+
+    def _error_row(err):
+        if err is None:
+            return None
+        at = task_time(err)
+        return {"error": err["error"], "message": err["message"], "state": err["state"],
+                "exit_code": err["exit_code"], "node": err["node"],
+                "at": iso(at), "age_s": age_s(at, now) if at else None}
+
     rows = [
         {
             "service_name": s["name"], "mode": s["mode"], "state": s["state"],
             "running": s["running"], "desired": s["desired"],
+            "crash_looping": s["crash_looping"],
+            "failures_1h": s["failures_1h"], "failures_24h": s["failures_24h"],
+            "last_error": _error_row(s["last_error"]),
+            "previous_tasks": s["previous_tasks"], "tasks_known": s["tasks_known"],
             "replicas": [
                 {"ghost": True, "host": r["host"], "count": r["count"]} if r["ghost"] else
                 {"ghost": False, "host": r["host"], "host_id": r["host_id"], "container_name": r["name"],
@@ -316,7 +395,7 @@ async def list_swarm_services(limit: int = 100) -> dict:
                 for r in s["replicas"]
             ],
         }
-        for s in sorted(services, key=lambda s: (order.get(s["state"], 3), s["name"]))
+        for s in sorted(services, key=_rank)
     ]
     return {"nodes": node_rows, **bounded(rows, limit, key="services")}