feat(docker): overview findings say when the problem began
#5013 "Overview findings for containers and Swarm services carry since=null" - Container problems are dated from their own evidence: a crash or restart loop from its first death in the hour, an OOM kill or error exit from the death that ended it, a failing healthcheck from its last turn to unhealthy. list_containers and get_container carry problem_since / problem_age_s. - Swarm services carry trouble_since: a crash loop's first failure in the hour, a short service's oldest troubled task in each failing slot, a recovered service's first failure in 24h. list_swarm_services shows it. - get_overview findings from the docker plugin now have since and age_s, so they rank by how long they have stood. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -120,6 +120,46 @@ def task_summary(tasks: list[dict], now: datetime, crash_loop_deaths: int,
|
||||
}
|
||||
|
||||
|
||||
def _oldest_failure(tasks: list[dict], since: datetime) -> datetime | None:
|
||||
times = [at for t in tasks if t.get("state") in FAILED_TASK_STATES
|
||||
and (at := task_time(t)) is not None and at >= since]
|
||||
return min(times, default=None)
|
||||
|
||||
|
||||
def trouble_since(tasks: list[dict], state: str, crash_looping: bool,
|
||||
now: datetime) -> datetime | None:
|
||||
"""When a service's current trouble began, as far as its retained tasks
|
||||
tell; None when it has none, or the tasks cannot say.
|
||||
|
||||
A crash loop dates from its first failure in the last hour, and a healthy
|
||||
service that lost tasks from its first failure in the last 24h. A service
|
||||
short of replicas dates from its troubled slots (replicated: the slot;
|
||||
global: the node): in each slot whose newest task is not running, the
|
||||
oldest of its newest troubled tasks (failed, rejected, or carrying an
|
||||
error), walking back until a running task interrupts them. Swarm keeps few
|
||||
old tasks, so a long outage reads as the oldest one retained: a lower
|
||||
bound.
|
||||
"""
|
||||
if crash_looping:
|
||||
return _oldest_failure(tasks, now - timedelta(hours=1))
|
||||
if state == "healthy":
|
||||
return _oldest_failure(tasks, now - timedelta(hours=24))
|
||||
by_slot: dict = {}
|
||||
for t in tasks: # newest first, so each slot's list is too
|
||||
by_slot.setdefault(t.get("slot") or t.get("node_id"), []).append(t)
|
||||
began = []
|
||||
for slot_tasks in by_slot.values():
|
||||
oldest = None
|
||||
for t in slot_tasks:
|
||||
if t.get("state") == "running":
|
||||
break
|
||||
if t.get("state") in FAILED_TASK_STATES or t.get("error"):
|
||||
oldest = task_time(t) or oldest
|
||||
if oldest is not None:
|
||||
began.append(oldest)
|
||||
return min(began, default=None)
|
||||
|
||||
|
||||
def _slot_key(c) -> str:
|
||||
"""A task container's slot within its service: "<service>.<slot>.<task>" for
|
||||
replicated services, "<service>.<node id>.<task>" for global ones."""
|
||||
@@ -204,6 +244,7 @@ def build_swarm_services(containers, services, node_hostname: dict, host_name: d
|
||||
without task data); crash_looping = failures_1h ≥ the
|
||||
Settings → Thresholds crash-loop count
|
||||
last_error = last_task_error(), plus `node` (the node's hostname)
|
||||
trouble_since = trouble_since(): when the current trouble began, or None
|
||||
standalone[i] = {host_id, host, containers:[...]} # non-swarm, by host
|
||||
superseded = the task containers Swarm replaced, so callers can skip them
|
||||
"""
|
||||
@@ -282,15 +323,19 @@ def build_swarm_services(containers, services, node_hostname: dict, host_name: d
|
||||
if desired is None:
|
||||
desired = len(replicas)
|
||||
|
||||
tasks = tasks_by_svc.get(name, [])
|
||||
state = _service_state(int(running or 0), int(desired or 0))
|
||||
summary = task_summary(tasks, now, crash_loop_deaths, node_hostname)
|
||||
out_services.append({
|
||||
"name": name,
|
||||
"mode": getattr(svc, "mode", "replicated") if svc is not None else "replicated",
|
||||
"running": running,
|
||||
"desired": desired,
|
||||
"state": _service_state(int(running or 0), int(desired or 0)),
|
||||
"state": state,
|
||||
"replicas": replicas,
|
||||
"previous_tasks": previous.get(name, 0),
|
||||
**task_summary(tasks_by_svc.get(name, []), now, crash_loop_deaths, node_hostname),
|
||||
**summary,
|
||||
"trouble_since": trouble_since(tasks, state, summary["crash_looping"], now),
|
||||
})
|
||||
|
||||
# Non-swarm containers grouped by host, running-first order preserved.
|
||||
|
||||
Reference in New Issue
Block a user