feat(docker): overview findings say when the problem began

#5013 "Overview findings for containers and Swarm services carry since=null"

- Container problems are dated from their own evidence: a crash or restart
  loop from its first death in the hour, an OOM kill or error exit from the
  death that ended it, a failing healthcheck from its last turn to unhealthy.
  list_containers and get_container carry problem_since / problem_age_s.
- Swarm services carry trouble_since: a crash loop's first failure in the
  hour, a short service's oldest troubled task in each failing slot, a
  recovered service's first failure in 24h. list_swarm_services shows it.
- get_overview findings from the docker plugin now have since and age_s,
  so they rank by how long they have stood.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
2026-10-06 10:24:10 -04:00
co-authored by Claude Opus 5.5
parent bac2859455
commit e9341c5639
6 changed files with 209 additions and 15 deletions
+47 -2
View File
@@ -120,6 +120,46 @@ def task_summary(tasks: list[dict], now: datetime, crash_loop_deaths: int,
}
def _oldest_failure(tasks: list[dict], since: datetime) -> datetime | None:
times = [at for t in tasks if t.get("state") in FAILED_TASK_STATES
and (at := task_time(t)) is not None and at >= since]
return min(times, default=None)
def trouble_since(tasks: list[dict], state: str, crash_looping: bool,
now: datetime) -> datetime | None:
"""When a service's current trouble began, as far as its retained tasks
tell; None when it has none, or the tasks cannot say.
A crash loop dates from its first failure in the last hour, and a healthy
service that lost tasks from its first failure in the last 24h. A service
short of replicas dates from its troubled slots (replicated: the slot;
global: the node): in each slot whose newest task is not running, the
oldest of its newest troubled tasks (failed, rejected, or carrying an
error), walking back until a running task interrupts them. Swarm keeps few
old tasks, so a long outage reads as the oldest one retained: a lower
bound.
"""
if crash_looping:
return _oldest_failure(tasks, now - timedelta(hours=1))
if state == "healthy":
return _oldest_failure(tasks, now - timedelta(hours=24))
by_slot: dict = {}
for t in tasks: # newest first, so each slot's list is too
by_slot.setdefault(t.get("slot") or t.get("node_id"), []).append(t)
began = []
for slot_tasks in by_slot.values():
oldest = None
for t in slot_tasks:
if t.get("state") == "running":
break
if t.get("state") in FAILED_TASK_STATES or t.get("error"):
oldest = task_time(t) or oldest
if oldest is not None:
began.append(oldest)
return min(began, default=None)
def _slot_key(c) -> str:
"""A task container's slot within its service: "<service>.<slot>.<task>" for
replicated services, "<service>.<node id>.<task>" for global ones."""
@@ -204,6 +244,7 @@ def build_swarm_services(containers, services, node_hostname: dict, host_name: d
without task data); crash_looping = failures_1h ≥ the
Settings → Thresholds crash-loop count
last_error = last_task_error(), plus `node` (the node's hostname)
trouble_since = trouble_since(): when the current trouble began, or None
standalone[i] = {host_id, host, containers:[...]} # non-swarm, by host
superseded = the task containers Swarm replaced, so callers can skip them
"""
@@ -282,15 +323,19 @@ def build_swarm_services(containers, services, node_hostname: dict, host_name: d
if desired is None:
desired = len(replicas)
tasks = tasks_by_svc.get(name, [])
state = _service_state(int(running or 0), int(desired or 0))
summary = task_summary(tasks, now, crash_loop_deaths, node_hostname)
out_services.append({
"name": name,
"mode": getattr(svc, "mode", "replicated") if svc is not None else "replicated",
"running": running,
"desired": desired,
"state": _service_state(int(running or 0), int(desired or 0)),
"state": state,
"replicas": replicas,
"previous_tasks": previous.get(name, 0),
**task_summary(tasks_by_svc.get(name, []), now, crash_loop_deaths, node_hostname),
**summary,
"trouble_since": trouble_since(tasks, state, summary["crash_looping"], now),
})
# Non-swarm containers grouped by host, running-first order preserved.