feat(docker): Swarm task awareness — replaced tasks, service crash loops, last task error

Milestone 465 "Swarm task awareness". Fixes three defects from the live MCP
review #4973:
- #5008 "Swarm task history read as live state"
- #5009 "Crash-loop detection is blind to Swarm services"
- #5011 "Swarm task errors are not captured"

Agent and ingest (agent 1.8.0):
- A manager's agent now reports each service's newest tasks: slot, node,
  desired and current state, Swarm's Err/Message, exit code and time.
- They are stored in docker_swarm_services.tasks_json (migration
  docker_010_swarm_tasks).

Swarm view (pure):
- A task container is "replaced" when Swarm set its task's desired state off
  `running`. Without the task on record, a newer or running task in the same
  slot decides.
- Replaced containers are no longer replicas or crashes.
- Each service gets, from the task records:
  - failures_1h and failures_24h (failed or rejected tasks);
  - crash_looping (Settings → Thresholds count);
  - last_error, in Swarm's own words.

MCP:
- New findings: swarm_service_crash_looping (critical) and swarm_task_failed
  (warning, on a healthy service with a failure in 24h).
- Down and degraded findings name the last task error.
- list_swarm_services carries the failure fields.
- Container rows mark swarm_replaced and give a replaced task no problems.

Web:
- The containers page swarm cards and the Swarm topology page show crash
  loops, recent failures and the last task error.
- The dashboard widget flags a crash loop, and its "failed (24h)" count no
  longer counts a rolling update's killed task.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

plugins/docker/ingest.py

@@ -169,6 +169,7 @@ async def _persist_swarm(session, host, swarm: dict) -> None:
         row.running = int(s.get("running") or 0)
         row.image = s.get("image")
         row.placement_json = json.dumps(s.get("placement") or [])
+        row.tasks_json = json.dumps(s.get("tasks") or [])
         row.updated_at = now
     stale_services = delete(DockerSwarmService).where(
         DockerSwarmService.host_id == host.id)

plugins/docker/mcp_tools.py

@@ -26,10 +26,10 @@ from steward.models.users import UserRole
 from . import service as docker_svc
 from .dedup import dedup_by_container_id
 from .models import DockerDiskUsage, DockerSwarmNode, DockerSwarmService
-from .swarm_view import build_swarm_services
+from .swarm_view import build_swarm_services, replaced_tasks, task_time
 
 PROBLEMS = ("restarting", "crash_looping", "unhealthy", "oom_killed", "exited_nonzero")
-CRASH_LOOP_DEATHS = 3        # default; Settings → Thresholds sets it
+CRASH_LOOP_DEATHS = docker_svc.CRASH_LOOP_DEATHS
 LOG_LINE_CHARS = 2000
 
 
@@ -61,7 +61,8 @@ def cut_middle(text: str, limit: int = LOG_LINE_CHARS) -> str:
     return f"{text[:keep]} [… {len(text) - 2 * keep} characters omitted …] {text[-keep:]}"
 
 
-def container_row(c, host_names: dict, problems: list[str], now: datetime) -> dict:
+def container_row(c, host_names: dict, problems: list[str], now: datetime,
+                  replaced: bool = False) -> dict:
     running = (c.status or "").lower() == "running"
     return {
         "container_name": c.name,
@@ -82,6 +83,8 @@ def container_row(c, host_names: dict, problems: list[str], now: datetime) -> di
         "uptime_s": age_s(c.started_at, now) if running and c.started_at else None,
         "compose_project": c.compose_project,
         "swarm_service": c.service_name,
+        # A swarm task Swarm has replaced: history, judged on its service.
+        "swarm_replaced": replaced if c.service_name else None,
         "reported_at": iso(c.scraped_at),
         "reported_age_s": age_s(c.scraped_at, now),
     }
@@ -110,15 +113,41 @@ def container_findings(rows: list[dict]) -> list[dict]:
     return out
 
 
+def task_error_text(err: dict | None) -> str:
+    """Swarm's own account of a task's end, as one clause."""
+    if not err:
+        return ""
+    why = err.get("error") or err.get("message") or err.get("state") or "unknown"
+    code = f", exit {err['exit_code']}" if err.get("exit_code") not in (None, 0) else ""
+    where = f" on {err['node']}" if err.get("node") else ""
+    return f"{why}{code}{where}"
+
+
 def service_findings(services: list[dict]) -> list[dict]:
+    """Swarm services that are failing: crash-looping, under-replicated, or that
+    lost a task to an error in the last 24h. Each names Swarm's last task error."""
     out = []
     for s in services:
-        if s["state"] == "healthy":
-            continue
-        out.append(finding(
-            "critical" if s["state"] == "down" else "warning", f"swarm_service_{s['state']}",
-            s["name"], f"swarm service {s['name']} has {s['running']} of {s['desired']} replicas running",
-            ids={"service_name": s["name"]}, next="list_swarm_services()"))
+        replicas = f"{s['running']} of {s['desired']} replicas running"
+        last = task_error_text(s.get("last_error"))
+        tail = f"; last task error: {last}" if last else ""
+        ids = {"service_name": s["name"]}
+        if s.get("crash_looping"):
+            out.append(finding(
+                "critical", "swarm_service_crash_looping", s["name"],
+                f"swarm service {s['name']} keeps failing: {s['failures_1h']} tasks failed in the "
+                f"last hour, {replicas}{tail}", ids=ids, next="list_swarm_services()"))
+        elif s["state"] != "healthy":
+            out.append(finding(
+                "critical" if s["state"] == "down" else "warning", f"swarm_service_{s['state']}",
+                s["name"], f"swarm service {s['name']} has {replicas}{tail}",
+                ids=ids, next="list_swarm_services()"))
+        elif s.get("failures_24h"):
+            n = s["failures_24h"]
+            out.append(finding(
+                "warning", "swarm_task_failed", s["name"],
+                f"swarm service {s['name']} had {n} failed task{'s' if n != 1 else ''} in the last "
+                f"24h and is now {replicas}{tail}", ids=ids, next="list_swarm_services()"))
     return out
 
 
@@ -129,38 +158,49 @@ async def _host_names(db) -> dict[str, str]:
 
 
 def _crash_loop_deaths() -> int:
-    th = (mcp_app().config.get("THRESHOLDS") or {}).get("crash_loop") or {}
-    return int(th.get("warn") or CRASH_LOOP_DEATHS)
+    return docker_svc.crash_loop_deaths(mcp_app().config)
+
+
+async def _replaced(db, containers) -> set[tuple]:
+    services = list((await db.execute(select(DockerSwarmService))).scalars())
+    return replaced_tasks(containers, services)
 
 
 async def _rows(db, now: datetime) -> list[dict]:
+    """Every container with its problems. A swarm task Swarm replaced has none:
+    its failure, if it was one, belongs to its service (service_findings)."""
     containers = dedup_by_container_id(await docker_svc.list_containers(db))
+    replaced = await _replaced(db, containers)
     failures = await docker_svc.recent_failures_by_container(db, now - timedelta(hours=24))
     hour_ago = now - timedelta(hours=1)
     host_names = await _host_names(db)
     deaths = _crash_loop_deaths()
     rows = []
     for c in containers:
+        gone = (c.host_id, c.name) in replaced
         events = failures.get((c.host_id, c.name), [])
         recent = sum(1 for e in events if e.at >= hour_ago)
-        rows.append(container_row(c, host_names, container_problems(c, recent, bool(events), deaths), now))
+        problems = [] if gone else container_problems(c, recent, bool(events), deaths)
+        rows.append(container_row(c, host_names, problems, now, replaced=gone))
     return rows
 
 
-async def _swarm(db):
+async def _swarm(db, now: datetime):
     containers = dedup_by_container_id(await docker_svc.list_containers(db))
     services = list((await db.execute(select(DockerSwarmService))).scalars())
     nodes = list((await db.execute(select(DockerSwarmNode))).scalars())
     host_names = await _host_names(db)
     node_hostname = {n.node_id: n.hostname for n in nodes if n.hostname}
-    built = build_swarm_services(containers, services, node_hostname, host_names)
+    built = build_swarm_services(containers, services, node_hostname, host_names,
+                                 now=now, crash_loop_deaths=_crash_loop_deaths())
     return built["services"], nodes, host_names
 
 
 async def mcp_overview(db) -> list[dict]:
     """get_overview findings: failing containers and under-replicated swarm services."""
-    rows = await _rows(db, now_utc())
-    services, _, _ = await _swarm(db)
+    now = now_utc()
+    rows = await _rows(db, now)
+    services, _, _ = await _swarm(db, now)
     return container_findings(rows) + service_findings(services)
 
 
@@ -184,6 +224,11 @@ async def list_containers(host: str = "", status: str = "", problem: str = "",
     Containers with problems come first. Each row's `problems` lists what is
     wrong, most severe first. `reported_age_s` is how old the agent's last
     report is, so a large value means the row may be stale.
+
+    A swarm task's container that Swarm has replaced (a rolling update, or a
+    retry after a failure) has `swarm_replaced: true` and no problems: Swarm
+    restarts a task as a new container, so failures are counted on the
+    service — list_swarm_services shows them.
     """
     if problem and problem != "any" and problem not in PROBLEMS:
         raise ValueError(f"problem {problem!r} is not 'any' or one of {', '.join(PROBLEMS)}.")
@@ -254,6 +299,7 @@ async def get_container(container: str, host: str = "", range: str = "24h", even
     now = now_utc()
     async with db_session() as db:
         c = await _pick_container(db, container, host)
+        gone = (c.host_id, c.name) in await _replaced(db, await docker_svc.list_containers(db))
         failures = (await docker_svc.recent_failures_by_container(
             db, now - timedelta(hours=24))).get((c.host_id, c.name), [])
         host_names = await _host_names(db)
@@ -265,9 +311,9 @@ async def get_container(container: str, host: str = "", range: str = "24h", even
         ports = json.loads(c.ports_json or "[]")
     except ValueError:
         ports = []
+    problems = [] if gone else container_problems(c, recent, bool(failures), _crash_loop_deaths())
     return {
-        **container_row(c, host_names,
-                        container_problems(c, recent, bool(failures), _crash_loop_deaths()), now),
+        **container_row(c, host_names, problems, now, replaced=gone),
         "container_id": c.container_id,
         "ports": ports,
         "net_rx_bytes": c.net_rx_bytes, "net_tx_bytes": c.net_tx_bytes,
@@ -285,15 +331,32 @@ async def list_swarm_services(limit: int = 100) -> dict:
     """Docker Swarm services with their replicas per node, and the swarm's nodes.
 
     `state` is "healthy" (all desired replicas running), "degraded" (some) or
-    "down" (none). Each replica names its host and container status; a replica
-    marked `ghost` runs on a node with no Steward agent, so Steward knows only
-    how many tasks are placed there. `nodes` lists every swarm node once, with
-    its role (manager/worker), availability and status.
+    "down" (none). `replicas` are the service's current tasks: each names its
+    host and container status, and one marked `ghost` runs on a node with no
+    Steward agent, so Steward knows only how many tasks are placed there.
+    Tasks Swarm replaced (rolling updates, retries) are left out and counted
+    in `previous_tasks`.
+
+    From Swarm's own task records (a manager's agent 1.8.0 or later;
+    `tasks_known: false` and nulls otherwise):
+    - `failures_1h` / `failures_24h`: tasks that failed or were rejected.
+      Swarm retries a failed task as a new one, so this, not any container's
+      restart count, is how a crash loop shows. Swarm keeps about 5 old tasks
+      per replica, which bounds the count.
+    - `crash_looping`: failures_1h reached the crash-loop count set under
+      Settings → Thresholds (default 3).
+    - `last_error`: the newest task that failed, was rejected, or carries an
+      error while Swarm retries it — `error` is Swarm's own text (e.g. "No
+      such image", "task: non-zero exit (1)"), with `exit_code`, `node`, `at`.
+
+    Services with a problem come first. `nodes` lists every swarm node once,
+    with its role (manager/worker), availability and status.
     limit: default 100, max 500.
     """
     limit = clamp_limit(limit, 100, 500)
+    now = now_utc()
     async with db_session() as db:
-        services, nodes, host_names = await _swarm(db)
+        services, nodes, host_names = await _swarm(db, now)
     seen, node_rows = set(), []
     for n in sorted(nodes, key=lambda n: n.updated_at, reverse=True):
         if n.node_id in seen:
@@ -304,10 +367,26 @@ async def list_swarm_services(limit: int = 100) -> dict:
                           "reported_by": host_names.get(n.host_id)})
     node_rows.sort(key=lambda n: n["hostname"].lower())
     order = {"down": 0, "degraded": 1, "healthy": 2}
+
+    def _rank(s):
+        return (not s["crash_looping"], order.get(s["state"], 3), not s["failures_24h"], s["name"])
+
+    def _error_row(err):
+        if err is None:
+            return None
+        at = task_time(err)
+        return {"error": err["error"], "message": err["message"], "state": err["state"],
+                "exit_code": err["exit_code"], "node": err["node"],
+                "at": iso(at), "age_s": age_s(at, now) if at else None}
+
     rows = [
         {
             "service_name": s["name"], "mode": s["mode"], "state": s["state"],
             "running": s["running"], "desired": s["desired"],
+            "crash_looping": s["crash_looping"],
+            "failures_1h": s["failures_1h"], "failures_24h": s["failures_24h"],
+            "last_error": _error_row(s["last_error"]),
+            "previous_tasks": s["previous_tasks"], "tasks_known": s["tasks_known"],
             "replicas": [
                 {"ghost": True, "host": r["host"], "count": r["count"]} if r["ghost"] else
                 {"ghost": False, "host": r["host"], "host_id": r["host_id"], "container_name": r["name"],
@@ -316,7 +395,7 @@ async def list_swarm_services(limit: int = 100) -> dict:
                 for r in s["replicas"]
             ],
         }
-        for s in sorted(services, key=lambda s: (order.get(s["state"], 3), s["name"]))
+        for s in sorted(services, key=_rank)
     ]
     return {"nodes": node_rows, **bounded(rows, limit, key="services")}
 

plugins/docker/migrations/versions/docker_010_swarm_tasks.py

@@ -0,0 +1,30 @@
+"""Docker swarm service tasks column
+
+Adds docker_swarm_services.tasks_json — each service's newest tasks as a
+manager's agent (≥ 1.8.0) reports them: slot, node, desired and current state,
+Swarm's error and message, exit code and time. Lets Steward tell a task Swarm
+replaced from one that crashed, count a service's failures across task
+retries, and say why its last task failed. Additive column; no DROP+recreate.
+
+Revision ID: docker_010_swarm_tasks
+Revises: docker_009_container_logs
+Create Date: 2026-10-06
+"""
+from typing import Sequence, Union
+from alembic import op
+import sqlalchemy as sa
+
+revision: str = "docker_010_swarm_tasks"
+down_revision: Union[str, None] = "docker_009_container_logs"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    op.add_column("docker_swarm_services",
+                  sa.Column("tasks_json", sa.Text, nullable=False,
+                            server_default="[]"))
+
+
+def downgrade() -> None:
+    op.drop_column("docker_swarm_services", "tasks_json")

plugins/docker/models.py

@@ -223,6 +223,11 @@ class DockerSwarmService(Base):
     # A manager sees every task, so this captures cross-node placement that the
     # local docker_containers rows (this host only) can't.
     placement_json: Mapped[str] = mapped_column(Text, nullable=False, default="[]")
+    # The service's newest tasks, newest first, as the manager's agent shaped
+    # them: [{"task_id", "slot", "node_id", "desired_state", "state", "message",
+    # "error", "exit_code", "at"}], JSON. Swarm's own record of which tasks it
+    # replaced and why each one stopped; "[]" from agents older than 1.8.0.
+    tasks_json: Mapped[str] = mapped_column(Text, nullable=False, default="[]")
     updated_at: Mapped[datetime] = mapped_column(
         DateTime(timezone=True), nullable=False,
         default=lambda: datetime.now(timezone.utc),

plugins/docker/routes.py

@@ -15,7 +15,7 @@ from steward.models.hosts import Host
 from steward.core.time_range import parse_range, DEFAULT_RANGE
 from . import service as docker_svc
 from .dedup import dedup_by_container_id
-from .swarm_view import build_swarm_services
+from .swarm_view import build_swarm_services, parse_tasks, task_summary
 from .models import (
     DockerContainer, DockerDiskUsage, DockerEvent, DockerImage,
     DockerSwarmNode, DockerSwarmService,
@@ -128,7 +128,9 @@ async def rows():
         # from the managers' placement). Non-swarm containers keep the host view.
         node_hostname = {n.node_id: n.hostname for n in nodes if n.hostname}
         host_name = {h.id: h.name for h in hosts.values()}
-        swarm_view = build_swarm_services(containers, services, node_hostname, host_name)
+        swarm_view = build_swarm_services(
+            containers, services, node_hostname, host_name,
+            crash_loop_deaths=docker_svc.crash_loop_deaths(current_app.config))
 
         standalone = [c for c in containers if not c.service_name]
         groups: dict[str, dict] = {}
@@ -204,13 +206,13 @@ async def widget():
         )).scalars()))
         services = list((await db.execute(select(DockerSwarmService))).scalars())
         nodes = list((await db.execute(select(DockerSwarmNode))).scalars())
-        # Recent failures are more actionable than a static "stopped" tally. Count
+        # Recent failures are more actionable than a static "stopped" tally. Read
         # DISTINCT container names so the two managers' duplicate die/oom events
-        # don't double it.
-        failed_24h = (await db.execute(
-            select(func.count(func.distinct(DockerEvent.container_name)))
+        # don't double it; swarm tasks are counted on their service below.
+        dead_names = list((await db.execute(
+            select(func.distinct(DockerEvent.container_name))
             .where(DockerEvent.event.in_(("die", "oom")), DockerEvent.at >= since_24h)
-        )).scalar() or 0
+        )).scalars())
         hosts = await docker_svc.host_map(
             db,
             {c.host_id for c in all_containers}
@@ -221,7 +223,10 @@ async def widget():
     # host); non-swarm containers stay grouped by host.
     node_hostname = {n.node_id: n.hostname for n in nodes if n.hostname}
     host_name = {h.id: h.name for h in hosts.values()}
-    swarm_services = build_swarm_services(all_containers, services, node_hostname, host_name)["services"]
+    swarm_services = build_swarm_services(
+        all_containers, services, node_hostname, host_name,
+        crash_loop_deaths=docker_svc.crash_loop_deaths(current_app.config))["services"]
+    failed_24h = docker_svc.failed_24h(dead_names, swarm_services)
 
     standalone = [c for c in all_containers if not c.service_name]
     running = [c for c in standalone if c.status == "running"]
@@ -413,6 +418,8 @@ async def swarm():
                 node_by_id[n.node_id] = n
 
         node_names = {nid: (nrow.hostname or nid) for nid, nrow in node_by_id.items()}
+        now = datetime.now(timezone.utc)
+        deaths = docker_svc.crash_loop_deaths(current_app.config)
         svc_dicts = []
         for s in sorted(svc_by_name.values(), key=lambda s: s.service_name):
             placement = []
@@ -428,7 +435,10 @@ async def swarm():
                 "healthy": s.running >= s.desired and s.desired > 0,
                 "degraded": 0 < s.running < s.desired,
                 "down": s.running == 0 and s.desired > 0,
+                "state": ("healthy" if s.running >= s.desired and s.desired > 0
+                          else "degraded" if s.running > 0 else "down"),
                 "placement": placement,
+                **task_summary(parse_tasks(s), now, deaths, node_names),
             })
         node_rows = sorted(
             node_by_id.values(), key=lambda n: (n.role, (n.hostname or n.node_id).lower())

plugins/docker/service.py

@@ -14,6 +14,37 @@ from steward.models.hosts import Host
 from .models import DockerContainer, DockerEvent, DockerLog, DockerMetric, DockerMetricHourly
 
 
+CRASH_LOOP_DEATHS = 3  # default; Settings → Thresholds sets it
+
+
+def crash_loop_deaths(config) -> int:
+    """Deaths within an hour that make a container or a swarm service a crash
+    loop, from Settings → Thresholds (app.config["THRESHOLDS"])."""
+    th = (config.get("THRESHOLDS") or {}).get("crash_loop") or {}
+    return int(th.get("warn") or CRASH_LOOP_DEATHS)
+
+
+def failed_24h(dead_names, swarm_services) -> int:
+    """Failures in the last 24h for the dashboard: distinct plain containers
+    that died or were OOM-killed, plus each swarm service's failed tasks.
+
+    A swarm task's container dies whenever Swarm replaces it (a rolling update
+    kills it with 137), so its die event is not a failure. Services with
+    Swarm's task records count their failed tasks instead; a service without
+    them (an agent older than 1.8.0) falls back to its tasks' die events.
+    """
+    by_prefix = {s["name"] + ".": s for s in swarm_services}
+    plain, fallback = 0, set()
+    for name in dead_names:
+        svc = next((s for p, s in by_prefix.items() if name.startswith(p)), None)
+        if svc is None:
+            plain += 1
+        elif not svc["tasks_known"]:
+            fallback.add(name)
+    tasks = sum(s["failures_24h"] or 0 for s in swarm_services if s["tasks_known"])
+    return plain + len(fallback) + tasks
+
+
 def bucket(values: list[float], target: int = 40) -> list[float]:
     """Bucket-average a series down to ~target points (DB-agnostic, in Python).
 

plugins/docker/swarm_view.py

@@ -11,12 +11,28 @@ For every swarm service we list the real replicas we have detail for, then add
 "ghost" replicas for the remaining placement count on each node (agent-less
 nodes). Non-swarm containers pass through grouped by host, unchanged.
 
+A third source, DockerSwarmService.tasks_json (agent ≥ 1.8.0), is Swarm's own
+record of each service's newest tasks. It decides which task containers are
+*superseded* — replaced by Swarm, so neither a replica nor a crash — and gives
+each service its failures across task retries and the error of its last
+failed task. Swarm restarts a task by creating a new one (a new container), so
+failures can only be counted per service, never per container.
+
 Kept model-free (no ORM imports) so it's unit-testable and safe to import in
 tests without the plugin-loader "table already defined" gotcha.
 """
 from __future__ import annotations
 
 import json
+from datetime import datetime, timedelta, timezone
+
+# Task states that mean the task ended in error: `failed` (its container exited
+# non-zero or never started) and `rejected` (Swarm could not start it at all,
+# e.g. no such image, a bad mount).
+FAILED_TASK_STATES = ("failed", "rejected")
+# Desired states of a task Swarm still means to run. Anything else (shutdown,
+# remove, complete, orphaned) is a task Swarm has finished with.
+_CURRENT_DESIRED = ("running", "ready", "")
 
 
 def _service_state(running: int, desired: int) -> str:
@@ -36,30 +52,179 @@ def _placement(service) -> list[dict]:
     return [p for p in data if isinstance(p, dict)]
 
 
-def build_swarm_services(containers, services, node_hostname: dict, host_name: dict) -> dict:
-    """Return {"services": [...], "standalone": [...]}.
+def parse_tasks(service) -> list[dict]:
+    """A service row's task records, newest first; [] when its agent sent none."""
+    raw = getattr(service, "tasks_json", None) or "[]"
+    try:
+        data = json.loads(raw)
+    except (ValueError, TypeError):
+        return []
+    return [t for t in data if isinstance(t, dict)]
 
-    services[i] = {name, mode, desired, running, state, replicas:[...]}
-      real replica  = {ghost:False, host, host_id, name, status, cpu_pct, mem_pct,
-                       health, restart_count}
-      ghost replica = {ghost:True, host, count}   # placement with no local row
-    standalone[i] = {host_id, host, containers:[...]}  # non-swarm, by host
+
+def task_time(task: dict) -> datetime | None:
+    """A task's status timestamp. Docker writes nanoseconds; Python ≥ 3.11 keeps
+    the first six digits."""
+    raw = task.get("at") or ""
+    try:
+        at = datetime.fromisoformat(raw.replace("Z", "+00:00"))
+    except ValueError:
+        return None
+    return at if at.tzinfo else at.replace(tzinfo=timezone.utc)
+
+
+def task_failures(tasks: list[dict], since: datetime) -> int:
+    """How many tasks ended in error at or after `since`."""
+    n = 0
+    for t in tasks:
+        at = task_time(t)
+        if t.get("state") in FAILED_TASK_STATES and at is not None and at >= since:
+            n += 1
+    return n
+
+
+def last_task_error(tasks: list[dict]) -> dict | None:
+    """Why the service's newest troubled task is in trouble, from Swarm itself.
+
+    The newest task that failed or was rejected, or that Swarm attached an error
+    to while still trying (a pending task with "no suitable node", say). None
+    when no retained task has a problem.
     """
+    for t in tasks:  # newest first
+        if t.get("state") in FAILED_TASK_STATES or t.get("error"):
+            return {"state": t.get("state"), "error": t.get("error") or None,
+                    "message": t.get("message") or None, "exit_code": t.get("exit_code"),
+                    "node_id": t.get("node_id") or None, "at": t.get("at") or None}
+    return None
+
+
+def task_summary(tasks: list[dict], now: datetime, crash_loop_deaths: int,
+                 node_hostname: dict) -> dict:
+    """What a service's task records say: tasks_known, failures_1h,
+    failures_24h (None without records), crash_looping, and last_error with
+    the node's hostname as `node`."""
+    known = bool(tasks)
+    failures_1h = task_failures(tasks, now - timedelta(hours=1)) if known else None
+    last_error = last_task_error(tasks)
+    if last_error is not None:
+        nid = last_error["node_id"]
+        last_error["node"] = node_hostname.get(nid) or (nid[:12] if nid else None)
+        at = task_time(last_error)
+        last_error["age_s"] = max(0, int((now - at).total_seconds())) if at else None
+    return {
+        "tasks_known": known,
+        "failures_1h": failures_1h,
+        "failures_24h": task_failures(tasks, now - timedelta(hours=24)) if known else None,
+        "crash_looping": bool(known and failures_1h >= crash_loop_deaths),
+        "last_error": last_error,
+    }
+
+
+def _slot_key(c) -> str:
+    """A task container's slot within its service: "<service>.<slot>.<task>" for
+    replicated services, "<service>.<node id>.<task>" for global ones."""
+    svc, name = c.service_name, c.name or ""
+    rest = name[len(svc) + 1:] if name.startswith(svc + ".") else name
+    return rest.split(".", 1)[0]
+
+
+def superseded_containers(swarm_cs, tasks_by_id: dict[str, dict]) -> set[tuple]:
+    """(host_id, name) of task containers Swarm has replaced.
+
+    With the task on record, its desired state decides: Swarm sets a replaced
+    or failed task's desired state to shutdown. Without it (an older agent, or a
+    task older than the retained history), a container that is not running is
+    superseded when another task in the same slot is running or started later.
+    """
+    out: set[tuple] = set()
+    by_slot: dict[tuple, list] = {}
+    for c in swarm_cs:
+        by_slot.setdefault((c.service_name, _slot_key(c)), []).append(c)
+    for c in swarm_cs:
+        task = tasks_by_id.get(getattr(c, "task_id", None) or "")
+        if task is not None:
+            if task.get("desired_state") not in _CURRENT_DESIRED:
+                out.add((c.host_id, c.name))
+            continue
+        if (getattr(c, "status", "") or "").lower() == "running":
+            continue
+        mine = getattr(c, "started_at", None)
+        for other in by_slot[(c.service_name, _slot_key(c))]:
+            if other is c:
+                continue
+            theirs = getattr(other, "started_at", None)
+            if ((getattr(other, "status", "") or "").lower() == "running"
+                    or (theirs is not None and (mine is None or theirs > mine))):
+                out.add((c.host_id, c.name))
+                break
+    return out
+
+
+def _newest_per_name(services) -> dict[str, object]:
+    """Every manager reports the same cluster-global service set; keep the most
+    recently updated row per name, so a manager on an older agent (no tasks)
+    doesn't shadow one that reports them."""
+    by_name: dict[str, object] = {}
+    for s in services:
+        name = getattr(s, "service_name", None)
+        if not name:
+            continue
+        kept = by_name.get(name)
+        if kept is None:
+            by_name[name] = s
+            continue
+        a, b = getattr(s, "updated_at", None), getattr(kept, "updated_at", None)
+        if a is not None and (b is None or a > b):
+            by_name[name] = s
+    return by_name
+
+
+def replaced_tasks(containers, services) -> set[tuple]:
+    """(host_id, name) of the task containers Swarm has replaced, from the
+    container rows and the managers' service rows. What the container views use
+    to leave history out of their problem lists."""
+    swarm_cs = [c for c in containers if getattr(c, "service_name", None)]
+    tasks_by_id = {t["task_id"]: t for s in _newest_per_name(services).values()
+                   for t in parse_tasks(s) if t.get("task_id")}
+    return superseded_containers(swarm_cs, tasks_by_id)
+
+
+def build_swarm_services(containers, services, node_hostname: dict, host_name: dict,
+                         *, now: datetime | None = None, crash_loop_deaths: int = 3) -> dict:
+    """Return {"services": [...], "standalone": [...], "superseded": {(host_id, name)}}.
+
+    services[i] = {name, mode, desired, running, state, replicas:[...],
+                   previous_tasks, tasks_known, failures_1h, failures_24h,
+                   crash_looping, last_error}
+      real replica  = {ghost:False, host, host_id, name, status, cpu_pct, mem_pct,
+                       health, restart_count}   # current tasks only
+      ghost replica = {ghost:True, host, count}   # placement with no local row
+      previous_tasks = task containers Swarm replaced, left out of replicas
+      failures_*    = tasks that failed or were rejected in the window (None
+                      without task data); crash_looping = failures_1h ≥ the
+                      Settings → Thresholds crash-loop count
+      last_error    = last_task_error(), plus `node` (the node's hostname)
+    standalone[i] = {host_id, host, containers:[...]}  # non-swarm, by host
+    superseded    = the task containers Swarm replaced, so callers can skip them
+    """
+    now = now or datetime.now(timezone.utc)
     containers = list(containers)
     swarm_cs = [c for c in containers if getattr(c, "service_name", None)]
     standalone_cs = [c for c in containers if not getattr(c, "service_name", None)]
 
-    # Dedup service rows by name (every manager reports the same cluster-global
-    # set); keep the first — placement is identical across managers.
-    svc_by_name: dict[str, object] = {}
-    for s in services:
-        name = getattr(s, "service_name", None)
-        if name and name not in svc_by_name:
-            svc_by_name[name] = s
+    svc_by_name = _newest_per_name(services)
+    tasks_by_svc = {name: parse_tasks(s) for name, s in svc_by_name.items()}
+    tasks_by_id = {t["task_id"]: t for ts in tasks_by_svc.values() for t in ts
+                   if t.get("task_id")}
+    superseded = superseded_containers(swarm_cs, tasks_by_id)
 
-    # Real containers grouped by service name.
+    # Current task containers grouped by service name; replaced ones are counted.
     cs_by_service: dict[str, list] = {}
+    previous: dict[str, int] = {}
     for c in swarm_cs:
+        if (c.host_id, c.name) in superseded:
+            previous[c.service_name] = previous.get(c.service_name, 0) + 1
+            continue
         cs_by_service.setdefault(c.service_name, []).append(c)
 
     def _host_of(c):
@@ -124,6 +289,8 @@ def build_swarm_services(containers, services, node_hostname: dict, host_name: d
             "desired": desired,
             "state": _service_state(int(running or 0), int(desired or 0)),
             "replicas": replicas,
+            "previous_tasks": previous.get(name, 0),
+            **task_summary(tasks_by_svc.get(name, []), now, crash_loop_deaths, node_hostname),
         })
 
     # Non-swarm containers grouped by host, running-first order preserved.
@@ -137,4 +304,4 @@ def build_swarm_services(containers, services, node_hostname: dict, host_name: d
         g["containers"].append(c)
     standalone = sorted(groups.values(), key=lambda g: g["host"].lower())
 
-    return {"services": out_services, "standalone": standalone}
+    return {"services": out_services, "standalone": standalone, "superseded": superseded}

plugins/docker/templates/docker/_swarm_failures.html

@@ -0,0 +1,31 @@
+{#- One line on a swarm service's task failures, from Swarm's own task records
+    (agent ≥ 1.8.0): a crash loop or the failures of the last 24h, and how the
+    newest troubled task ended, in Swarm's words. Expects `s` with state,
+    crash_looping, failures_1h, failures_24h and last_error; `style` is
+    optional extra CSS for the wrapper. Renders nothing for a healthy service
+    with no recent failures. -#}
+{%- macro ago(sec) -%}
+  {%- if sec < 90 -%}{{ sec }}s ago
+  {%- elif sec < 5400 -%}{{ sec // 60 }}m ago
+  {%- elif sec < 172800 -%}{{ sec // 3600 }}h ago
+  {%- else -%}{{ sec // 86400 }}d ago{%- endif -%}
+{%- endmacro -%}
+{%- set e = s.last_error -%}
+{%- if s.crash_looping or s.failures_24h or (e and s.state != 'healthy') %}
+<div style="display:flex;flex-wrap:wrap;align-items:baseline;gap:0.2rem 0.6rem;font-size:0.78rem;color:var(--text-muted);{{ style or '' }}">
+  {% if s.crash_looping %}
+  <span style="color:var(--red);font-weight:600;">Crash-looping · {{ s.failures_1h }} tasks failed in the last hour</span>
+  {% elif s.failures_24h %}
+  <span style="color:var(--orange);">{{ s.failures_24h }} task{{ 's' if s.failures_24h != 1 }} failed in the last 24h</span>
+  {% endif %}
+  {% if e %}
+  <span title="How Swarm recorded the end of this service's newest failed task">
+    Last task {{ e.state }}:
+    <span style="color:var(--text);font-family:ui-monospace,monospace;overflow-wrap:anywhere;">{{ e.error or e.message or e.state }}</span>
+    {%- if e.exit_code %} · exit {{ e.exit_code }}{% endif %}
+    {%- if e.node %} · {{ e.node }}{% endif %}
+    {%- if e.age_s is not none %} · {{ ago(e.age_s) }}{% endif %}
+  </span>
+  {% endif %}
+</div>
+{%- endif %}

plugins/docker/templates/docker/rows.html

@@ -26,9 +26,10 @@
   </div>
 </div>
 
-{# ── Swarm services: collapsed per-service, each replica labelled with its host.
-   Ghost replicas (tasks on nodes with no Steward agent) come from the managers'
-   placement data so the picture is cluster-complete. ──────────────────────── #}
+{# ── Swarm services: collapsed per-service, each current replica labelled with
+   its host (tasks Swarm replaced are left out). Ghost replicas (tasks on nodes
+   with no Steward agent) come from the managers' placement data so the picture
+   is cluster-complete. A failing service names Swarm's last task error. ───── #}
 {% if swarm_services %}
 <div class="card-flush" style="margin-bottom:1.5rem;">
   <div style="display:flex;align-items:baseline;gap:0.6rem;padding:0.5rem 0.75rem;border-bottom:1px solid var(--border);">
@@ -46,6 +47,7 @@
           {{ s.running }}/{{ s.desired }} running
         </span>
       </div>
+      {% with style="padding-left:1.1rem;margin-bottom:0.35rem;" %}{% include "docker/_swarm_failures.html" %}{% endwith %}
       <div style="display:grid;grid-template-columns:repeat(auto-fill,minmax(210px,1fr));gap:0.4rem;padding-left:1.1rem;">
         {% for r in s.replicas %}
         {% if r.ghost %}

plugins/docker/templates/docker/swarm.html

@@ -22,7 +22,7 @@
     <table class="table">
       <thead>
         <tr>
-          <th>Service</th><th>Mode</th><th>Replicas</th><th>Image</th><th>Placement</th>
+          <th>Service</th><th>Mode</th><th>Replicas</th><th>Image</th><th>Placement</th><th>Task failures</th>
         </tr>
       </thead>
       <tbody>
@@ -39,9 +39,14 @@
           <td style="font-size:0.78rem;color:var(--text-muted);">
             {% for p in s.placement %}{{ p.node }} ({{ p.running }}){% if not loop.last %}, {% endif %}{% else %}—{% endfor %}
           </td>
+          <td style="max-width:360px;">
+            {% include "docker/_swarm_failures.html" %}
+            {%- if not s.tasks_known %}<span style="font-size:0.78rem;color:var(--text-dim);" title="The reporting manager's agent predates task reporting (1.8.0); update it to see task failures.">—</span>
+            {%- elif not (s.crash_looping or s.failures_24h or (s.last_error and s.state != 'healthy')) %}<span style="font-size:0.78rem;color:var(--text-dim);">none in 24h</span>{% endif %}
+          </td>
         </tr>
         {% else %}
-        <tr><td colspan="5" style="color:var(--text-muted);font-size:0.85rem;text-align:center;padding:1.5rem;">No services in this swarm.</td></tr>
+        <tr><td colspan="6" style="color:var(--text-muted);font-size:0.85rem;text-align:center;padding:1.5rem;">No services in this swarm.</td></tr>
         {% endfor %}
       </tbody>
     </table>

plugins/docker/templates/docker/widget.html

@@ -8,7 +8,7 @@
     <span style="font-size:1.4rem;font-weight:700;line-height:1;color:var(--green);">{{ running_count }}</span>
     <span style="font-size:0.78rem;color:var(--text-muted);margin-left:0.25rem;">running</span>
   </div>
-  <div title="Distinct containers with a die or OOM event in the last 24h">
+  <div title="Containers that died or were OOM-killed in the last 24h, plus swarm tasks that failed (a task Swarm replaced is not a failure)">
     <span style="font-size:1.4rem;font-weight:700;line-height:1;color:{% if failed_24h %}var(--red){% else %}var(--text-muted){% endif %};">{{ failed_24h }}</span>
     <span style="font-size:0.78rem;color:var(--text-muted);margin-left:0.25rem;">failed (24h)</span>
   </div>
@@ -24,6 +24,7 @@
     <div style="display:flex;align-items:center;gap:0.4rem;font-size:0.82rem;">
       <span class="dot {% if s.state == 'healthy' %}dot-up{% elif s.state == 'degraded' %}dot-warn{% else %}dot-down{% endif %}"></span>
       <span style="font-weight:500;min-width:0;white-space:nowrap;overflow:hidden;text-overflow:ellipsis;">{{ s.name }}</span>
+      {% if s.crash_looping %}<span style="color:var(--red);font-size:0.7rem;font-weight:600;flex-shrink:0;" title="{{ s.failures_1h }} tasks failed in the last hour{% if s.last_error %}: {{ s.last_error.error or s.last_error.message or s.last_error.state }}{% endif %}">crash-looping</span>{% endif %}
       <span style="color:var(--text-muted);font-family:ui-monospace,monospace;font-size:0.74rem;margin-left:auto;flex-shrink:0;">{{ s.running }}/{{ s.desired }}</span>
     </div>
     <div style="display:flex;flex-wrap:wrap;gap:0.25rem;margin:0.2rem 0 0 1rem;">

plugins/host_agent/agent.py

@@ -20,7 +20,7 @@ from collections import deque
 from concurrent.futures import ThreadPoolExecutor
 from datetime import datetime, timezone
 
-AGENT_VERSION = "1.7.0"
+AGENT_VERSION = "1.8.0"
 
 # Default path to the local Docker Engine socket. Overridable via the
 # `docker_socket` config key; collection is silently skipped if it's absent or
@@ -874,14 +874,46 @@ def _swarm_is_manager(info: dict) -> bool:
     return bool((info or {}).get("Swarm", {}).get("ControlAvailable"))
 
 
+# Newest tasks kept per service. Swarm retains about 5 per slot by default, so
+# this covers a replicated service's recent history and a crash loop's deaths
+# without shipping a global service's whole fleet every sample.
+SWARM_TASKS_PER_SERVICE = 10
+_TASK_TEXT_CHARS = 300
+
+
+def _swarm_task(t: dict) -> dict:
+    """One task as Steward stores it: where it ran, what Swarm wants of it,
+    what it is doing, and why it stopped (Err/Message, exit code)."""
+    status = t.get("Status") or {}
+    cstatus = status.get("ContainerStatus") or {}
+    exit_code = cstatus.get("ExitCode")
+    return {
+        "task_id": t.get("ID", "") or "",
+        "slot": t.get("Slot"),
+        "node_id": t.get("NodeID", "") or "",
+        "desired_state": (t.get("DesiredState") or "").lower(),
+        "state": (status.get("State") or "").lower(),
+        "message": (status.get("Message") or "")[:_TASK_TEXT_CHARS],
+        "error": (status.get("Err") or "")[:_TASK_TEXT_CHARS],
+        "exit_code": exit_code if isinstance(exit_code, int) else None,
+        "at": status.get("Timestamp") or t.get("UpdatedAt") or "",
+    }
+
+
 def _swarm_services(services: list, tasks: list) -> list:
     """Roll desired-vs-running replicas per service up from the task list.
 
     Replica health isn't on the service object — it's derived by counting tasks:
     `running` = tasks currently in state running; `desired` = the service's
     configured replica count (replicated) or the count of tasks the scheduler
-    wants running (global, which has no fixed number).
+    wants running (global, which has no fixed number). Each service also
+    carries its newest tasks (`tasks`), so Steward can tell a task Swarm
+    replaced from one that crashed, and say why the last one failed.
     """
+    tasks_by_svc: dict = {}
+    for t in (tasks or []):
+        if t.get("ServiceID"):
+            tasks_by_svc.setdefault(t["ServiceID"], []).append(_swarm_task(t))
     running_by_svc: dict = {}
     desired_by_svc: dict = {}  # only used for global-mode services
     placement: dict = {}       # svc_id -> {node_id: running_count}
@@ -921,6 +953,8 @@ def _swarm_services(services: list, tasks: list) -> list:
             # sees every task, even ones whose containers live on other hosts).
             "placement": [{"node_id": n, "running": cnt}
                           for n, cnt in sorted(placement.get(sid, {}).items())],
+            "tasks": sorted(tasks_by_svc.get(sid, []), key=lambda t: t["at"],
+                            reverse=True)[:SWARM_TASKS_PER_SERVICE],
         })
     return out