feat(docker): Swarm task awareness — replaced tasks, service crash loops, last task error
Milestone 465 "Swarm task awareness". Fixes three defects from the live MCP review #4973: - #5008 "Swarm task history read as live state" - #5009 "Crash-loop detection is blind to Swarm services" - #5011 "Swarm task errors are not captured" Agent and ingest (agent 1.8.0): - A manager's agent now reports each service's newest tasks: slot, node, desired and current state, Swarm's Err/Message, exit code and time. - They are stored in docker_swarm_services.tasks_json (migration docker_010_swarm_tasks). Swarm view (pure): - A task container is "replaced" when Swarm set its task's desired state off `running`. Without the task on record, a newer or running task in the same slot decides. - Replaced containers are no longer replicas or crashes. - Each service gets, from the task records: - failures_1h and failures_24h (failed or rejected tasks); - crash_looping (Settings → Thresholds count); - last_error, in Swarm's own words. MCP: - New findings: swarm_service_crash_looping (critical) and swarm_task_failed (warning, on a healthy service with a failure in 24h). - Down and degraded findings name the last task error. - list_swarm_services carries the failure fields. - Container rows mark swarm_replaced and give a replaced task no problems. Web: - The containers page swarm cards and the Swarm topology page show crash loops, recent failures and the last task error. - The dashboard widget flags a crash loop, and its "failed (24h)" count no longer counts a rolling update's killed task. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -15,7 +15,7 @@ from steward.models.hosts import Host
|
||||
from steward.core.time_range import parse_range, DEFAULT_RANGE
|
||||
from . import service as docker_svc
|
||||
from .dedup import dedup_by_container_id
|
||||
from .swarm_view import build_swarm_services
|
||||
from .swarm_view import build_swarm_services, parse_tasks, task_summary
|
||||
from .models import (
|
||||
DockerContainer, DockerDiskUsage, DockerEvent, DockerImage,
|
||||
DockerSwarmNode, DockerSwarmService,
|
||||
@@ -128,7 +128,9 @@ async def rows():
|
||||
# from the managers' placement). Non-swarm containers keep the host view.
|
||||
node_hostname = {n.node_id: n.hostname for n in nodes if n.hostname}
|
||||
host_name = {h.id: h.name for h in hosts.values()}
|
||||
swarm_view = build_swarm_services(containers, services, node_hostname, host_name)
|
||||
swarm_view = build_swarm_services(
|
||||
containers, services, node_hostname, host_name,
|
||||
crash_loop_deaths=docker_svc.crash_loop_deaths(current_app.config))
|
||||
|
||||
standalone = [c for c in containers if not c.service_name]
|
||||
groups: dict[str, dict] = {}
|
||||
@@ -204,13 +206,13 @@ async def widget():
|
||||
)).scalars()))
|
||||
services = list((await db.execute(select(DockerSwarmService))).scalars())
|
||||
nodes = list((await db.execute(select(DockerSwarmNode))).scalars())
|
||||
# Recent failures are more actionable than a static "stopped" tally. Count
|
||||
# Recent failures are more actionable than a static "stopped" tally. Read
|
||||
# DISTINCT container names so the two managers' duplicate die/oom events
|
||||
# don't double it.
|
||||
failed_24h = (await db.execute(
|
||||
select(func.count(func.distinct(DockerEvent.container_name)))
|
||||
# don't double it; swarm tasks are counted on their service below.
|
||||
dead_names = list((await db.execute(
|
||||
select(func.distinct(DockerEvent.container_name))
|
||||
.where(DockerEvent.event.in_(("die", "oom")), DockerEvent.at >= since_24h)
|
||||
)).scalar() or 0
|
||||
)).scalars())
|
||||
hosts = await docker_svc.host_map(
|
||||
db,
|
||||
{c.host_id for c in all_containers}
|
||||
@@ -221,7 +223,10 @@ async def widget():
|
||||
# host); non-swarm containers stay grouped by host.
|
||||
node_hostname = {n.node_id: n.hostname for n in nodes if n.hostname}
|
||||
host_name = {h.id: h.name for h in hosts.values()}
|
||||
swarm_services = build_swarm_services(all_containers, services, node_hostname, host_name)["services"]
|
||||
swarm_services = build_swarm_services(
|
||||
all_containers, services, node_hostname, host_name,
|
||||
crash_loop_deaths=docker_svc.crash_loop_deaths(current_app.config))["services"]
|
||||
failed_24h = docker_svc.failed_24h(dead_names, swarm_services)
|
||||
|
||||
standalone = [c for c in all_containers if not c.service_name]
|
||||
running = [c for c in standalone if c.status == "running"]
|
||||
@@ -413,6 +418,8 @@ async def swarm():
|
||||
node_by_id[n.node_id] = n
|
||||
|
||||
node_names = {nid: (nrow.hostname or nid) for nid, nrow in node_by_id.items()}
|
||||
now = datetime.now(timezone.utc)
|
||||
deaths = docker_svc.crash_loop_deaths(current_app.config)
|
||||
svc_dicts = []
|
||||
for s in sorted(svc_by_name.values(), key=lambda s: s.service_name):
|
||||
placement = []
|
||||
@@ -428,7 +435,10 @@ async def swarm():
|
||||
"healthy": s.running >= s.desired and s.desired > 0,
|
||||
"degraded": 0 < s.running < s.desired,
|
||||
"down": s.running == 0 and s.desired > 0,
|
||||
"state": ("healthy" if s.running >= s.desired and s.desired > 0
|
||||
else "degraded" if s.running > 0 else "down"),
|
||||
"placement": placement,
|
||||
**task_summary(parse_tasks(s), now, deaths, node_names),
|
||||
})
|
||||
node_rows = sorted(
|
||||
node_by_id.values(), key=lambda n: (n.role, (n.hostname or n.node_id).lower())
|
||||
|
||||
Reference in New Issue
Block a user