bvandeusen/FabledSteward
1
0
Fork 0
Code Issues 1 Pull Requests 1 Actions Packages Projects Releases Wiki Activity

feat(docker): overview findings say when the problem began

Browse Source 
#5013 "Overview findings for containers and Swarm services carry since=null"

- Container problems are dated from their own evidence: a crash or restart
  loop from its first death in the hour, an OOM kill or error exit from the
  death that ended it, a failing healthcheck from its last turn to unhealthy.
  list_containers and get_container carry problem_since / problem_age_s.
- Swarm services carry trouble_since: a crash loop's first failure in the
  hour, a short service's oldest troubled task in each failing slot, a
  recovered service's first failure in 24h. list_swarm_services shows it.
- get_overview findings from the docker plugin now have since and age_s,
  so they rank by how long they have stood.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
bvandeusen
2026-10-06 10:24:10 -04:00
co-authored by Claude Opus 5.5
parent bac2859455
commit e9341c5639
6 changed files with 209 additions and 15 deletions
Download Patch File Download Diff File Expand all files Collapse all files
plugins/docker/mcp_tools.py
+60 -11
View File
@@ -53,6 +53,29 @@ def container_problems(c, failures_1h: int, failed_24h: bool,
return out
def problem_since(problems: list[str], deaths: list, hour_ago: datetime,
unhealthy_at: datetime | None = None) -> datetime | None:
"""When a container's worst problem began; None when the retained events
cannot say. `deaths` are its die/oom events of the last 24h, newest first.
A restart or crash loop dates from its first death in the last hour, an
OOM kill or error exit from the death that ended it, and a failing
healthcheck from when its health last turned unhealthy.
"""
if not problems:
return None
worst = problems[0]
if worst in ("restarting", "crash_looping"):
in_hour = [e.at for e in deaths if e.at >= hour_ago]
return min(in_hour) if in_hour else (deaths[0].at if deaths else None)
if worst == "unhealthy":
return unhealthy_at
if worst == "oom_killed":
ooms = [e.at for e in deaths if e.event == "oom"]
return ooms[0] if ooms else (deaths[0].at if deaths else None)
return deaths[0].at if deaths else None
def cut_middle(text: str, limit: int = LOG_LINE_CHARS) -> str:
"""Keep both ends of an over-long line and say how much of the middle went."""
if text is None or len(text) <= limit:
@@ -62,7 +85,7 @@ def cut_middle(text: str, limit: int = LOG_LINE_CHARS) -> str:
def container_row(c, host_names: dict, problems: list[str], now: datetime,
replaced: bool = False) -> dict:
replaced: bool = False, since: datetime | None = None) -> dict:
running = (c.status or "").lower() == "running"
return {
"container_name": c.name,
@@ -72,6 +95,9 @@ def container_row(c, host_names: dict, problems: list[str], now: datetime,
"status": c.status,
"health": c.health,
"problems": problems,
# When the worst problem began, from the container's retained events.
"problem_since": iso(since),
"problem_age_s": age_s(since, now),
"exit_code": c.exit_code,
"oom_killed": bool(c.oom_killed),
"restart_count": c.restart_count,
@@ -97,6 +123,10 @@ _SAYS = {"restarting": "is restarting", "crash_looping": "keeps dying",
"exited_nonzero": "exited with an error"}
def _from_iso(text: str | None) -> datetime | None:
return datetime.fromisoformat(text.replace("Z", "+00:00")) if text else None
def container_findings(rows: list[dict]) -> list[dict]:
out = []
for r in rows:
@@ -108,6 +138,7 @@ def container_findings(rows: list[dict]) -> list[dict]:
out.append(finding(
_SEVERITY[worst], f"container_{worst}", r["container_name"],
f"container {r['container_name']}{where} {' and '.join(_SAYS[p] for p in r['problems'])}{detail}",
since=_from_iso(r["problem_since"]),
ids={"host_id": r["host_id"], "container_name": r["container_name"]},
next=f'get_container(container="{r["container_name"]}", host="{r["host_id"]}")'))
return out
@@ -132,22 +163,25 @@ def service_findings(services: list[dict]) -> list[dict]:
last = task_error_text(s.get("last_error"))
tail = f"; last task error: {last}" if last else ""
ids = {"service_name": s["name"]}
since = s.get("trouble_since")
if s.get("crash_looping"):
out.append(finding(
"critical", "swarm_service_crash_looping", s["name"],
f"swarm service {s['name']} keeps failing: {s['failures_1h']} tasks failed in the "
f"last hour, {replicas}{tail}", ids=ids, next="list_swarm_services()"))
f"last hour, {replicas}{tail}", since=since, ids=ids,
next="list_swarm_services()"))
elif s["state"] != "healthy":
out.append(finding(
"critical" if s["state"] == "down" else "warning", f"swarm_service_{s['state']}",
s["name"], f"swarm service {s['name']} has {replicas}{tail}",
ids=ids, next="list_swarm_services()"))
since=since, ids=ids, next="list_swarm_services()"))
elif s.get("failures_24h"):
n = s["failures_24h"]
out.append(finding(
"warning", "swarm_task_failed", s["name"],
f"swarm service {s['name']} had {n} failed task{'s' if n != 1 else ''} in the last "
f"24h and is now {replicas}{tail}", ids=ids, next="list_swarm_services()"))
f"24h and is now {replicas}{tail}", since=since, ids=ids,
next="list_swarm_services()"))
return out
@@ -175,13 +209,17 @@ async def _rows(db, now: datetime) -> list[dict]:
hour_ago = now - timedelta(hours=1)
host_names = await _host_names(db)
deaths = _crash_loop_deaths()
sick = await docker_svc.unhealthy_since(
db, [(c.host_id, c.name) for c in containers if (c.health or "") == "unhealthy"])
rows = []
for c in containers:
gone = (c.host_id, c.name) in replaced
events = failures.get((c.host_id, c.name), [])
key = (c.host_id, c.name)
gone = key in replaced
events = failures.get(key, [])
recent = sum(1 for e in events if e.at >= hour_ago)
problems = [] if gone else container_problems(c, recent, bool(events), deaths)
rows.append(container_row(c, host_names, problems, now, replaced=gone))
since = problem_since(problems, events, hour_ago, sick.get(key))
rows.append(container_row(c, host_names, problems, now, replaced=gone, since=since))
return rows
@@ -222,8 +260,10 @@ async def list_containers(host: str = "", status: str = "", problem: str = "",
limit: default 100, max 500.
Containers with problems come first. Each row's `problems` lists what is
wrong, most severe first. `reported_age_s` is how old the agent's last
report is, so a large value means the row may be stale.
wrong, most severe first, and `problem_since` / `problem_age_s` say when
the worst of them began (null when Steward's retained events cannot say).
`reported_age_s` is how old the agent's last report is, so a large value
means the row may be stale.
A swarm task's container that Swarm has replaced (a rolling update, or a
retry after a failure) has `swarm_replaced: true` and no problems: Swarm
@@ -306,14 +346,17 @@ async def get_container(container: str, host: str = "", range: str = "24h", even
cpu = await docker_svc.container_points(db, c.host_id, c.name, "cpu_pct", since)
mem = await docker_svc.container_points(db, c.host_id, c.name, "mem_pct", since)
evs = await docker_svc.container_events(db, c.host_id, c.name, events)
recent = sum(1 for e in failures if e.at >= now - timedelta(hours=1))
sick = await docker_svc.unhealthy_since(db, [(c.host_id, c.name)])
hour_ago = now - timedelta(hours=1)
recent = sum(1 for e in failures if e.at >= hour_ago)
try:
ports = json.loads(c.ports_json or "[]")
except ValueError:
ports = []
problems = [] if gone else container_problems(c, recent, bool(failures), _crash_loop_deaths())
began = problem_since(problems, failures, hour_ago, sick.get((c.host_id, c.name)))
return {
**container_row(c, host_names, problems, now, replaced=gone),
**container_row(c, host_names, problems, now, replaced=gone, since=began),
"container_id": c.container_id,
"ports": ports,
"net_rx_bytes": c.net_rx_bytes, "net_tx_bytes": c.net_tx_bytes,
@@ -348,6 +391,10 @@ async def list_swarm_services(limit: int = 100) -> dict:
- `last_error`: the newest task that failed, was rejected, or carries an
error while Swarm retries it — `error` is Swarm's own text (e.g. "No
such image", "task: non-zero exit (1)"), with `exit_code`, `node`, `at`.
- `trouble_since` / `trouble_age_s`: when the current trouble began — a
crash loop's first failure in the hour, a short service's first failed
task in its failing slots, a recovered service's first failure in 24h.
Swarm keeps few old tasks, so a long outage reads as a lower bound.
Services with a problem come first. `nodes` lists every swarm node once,
with its role (manager/worker), availability and status.
@@ -386,6 +433,8 @@ async def list_swarm_services(limit: int = 100) -> dict:
"crash_looping": s["crash_looping"],
"failures_1h": s["failures_1h"], "failures_24h": s["failures_24h"],
"last_error": _error_row(s["last_error"]),
"trouble_since": iso(s["trouble_since"]),
"trouble_age_s": age_s(s["trouble_since"], now),
"previous_tasks": s["previous_tasks"], "tasks_known": s["tasks_known"],
"replicas": [
{"ghost": True, "host": r["host"], "count": r["count"]} if r["ghost"] else