feat(docker): overview findings say when the problem began

#5013 "Overview findings for containers and Swarm services carry since=null"

- Container problems are dated from their own evidence: a crash or restart
  loop from its first death in the hour, an OOM kill or error exit from the
  death that ended it, a failing healthcheck from its last turn to unhealthy.
  list_containers and get_container carry problem_since / problem_age_s.
- Swarm services carry trouble_since: a crash loop's first failure in the
  hour, a short service's oldest troubled task in each failing slot, a
  recovered service's first failure in 24h. list_swarm_services shows it.
- get_overview findings from the docker plugin now have since and age_s,
  so they rank by how long they have stood.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
2026-10-06 10:24:10 -04:00
co-authored by Claude Opus 5.5
parent bac2859455
commit e9341c5639
6 changed files with 209 additions and 15 deletions
+32
View File
@@ -137,3 +137,35 @@ async def test_prune_docker_refuses_an_unknown_target_before_reading_anything(mo
with pytest.raises(ValueError, match="'volumes' is not one of containers, images, system"):
await t.prune_docker("nas", "volumes")
assert inspect.signature(t.prune_docker).parameters["check"].default is True
def _death(event, minutes_ago):
return NS(event=event, at=NOW - timedelta(minutes=minutes_ago))
def test_problem_since_dates_each_problem_from_its_own_evidence():
hour_ago = NOW - timedelta(hours=1)
deaths = [_death("die", 5), _death("die", 20), _death("die", 50), _death("die", 300)]
# A crash loop began with its first death inside the hour, not the older one.
assert t.problem_since(["crash_looping"], deaths, hour_ago) == NOW - timedelta(minutes=50)
# An error exit dates from the death that ended it.
assert t.problem_since(["exited_nonzero"], deaths, hour_ago) == NOW - timedelta(minutes=5)
oom = [_death("die", 2), _death("oom", 3)]
assert t.problem_since(["oom_killed", "exited_nonzero"], oom, hour_ago) == NOW - timedelta(minutes=3)
turned = NOW - timedelta(hours=2)
assert t.problem_since(["unhealthy"], [], hour_ago, turned) == turned
# No retained evidence: unknown, not now.
assert t.problem_since(["unhealthy"], [], hour_ago) is None
assert t.problem_since([], deaths, hour_ago) is None
def test_findings_carry_when_the_problem_began():
began = NOW - timedelta(minutes=40)
row = t.container_row(_c(name="db", status="exited", exit_code=1), {}, ["exited_nonzero"],
NOW, since=began)
assert row["problem_since"] == "2026-10-06T11:20:00Z" and row["problem_age_s"] == 2400
(f,) = t.container_findings([row])
assert f["since"] == "2026-10-06T11:20:00Z" and f["age_s"] is not None
svc = {"name": "db", "state": "down", "running": 0, "desired": 1, "trouble_since": began}
(g,) = t.service_findings([svc])
assert g["since"] == "2026-10-06T11:20:00Z"
+40 -1
View File
@@ -3,7 +3,9 @@ import json
from datetime import datetime, timedelta, timezone
from types import SimpleNamespace
from plugins.docker.swarm_view import build_swarm_services, last_task_error, task_failures
from plugins.docker.swarm_view import (
build_swarm_services, last_task_error, task_failures, trouble_since,
)
NODE_HOSTNAME = {"nA": "docker01", "nB": "docker02", "nC": "docker03"}
HOST_NAME = {"h1": "docker01", "h2": "docker02"} # agents run on docker01/02 only
@@ -172,3 +174,40 @@ def test_the_newest_manager_report_wins():
(svc,) = _view([], [old, new])["services"]
assert svc["tasks_known"] is True and svc["failures_1h"] == 1
def _ago(minutes):
return NOW - timedelta(minutes=minutes)
def test_trouble_since_for_each_kind_of_trouble():
loop = [_task("t1", state="failed", ago_min=2), _task("t2", state="failed", ago_min=20),
_task("t3", state="failed", ago_min=45), _task("t4", state="failed", ago_min=90)]
# A crash loop: its first failure inside the hour.
assert trouble_since(loop, "down", True, NOW) == _ago(45)
# Down: the oldest of the newest troubled tasks, back to a running one.
down = [_task("t1", state="pending", ago_min=1, error="no suitable node"),
_task("t2", state="rejected", ago_min=30, error="No such image"),
_task("t3", state="running", desired="running", ago_min=600),
_task("t4", state="failed", ago_min=900)]
assert trouble_since(down, "degraded", False, NOW) == _ago(30)
# Another slot's newer, healthy task does not hide the failing one.
two = [_task("a2", state="running", desired="running", ago_min=5, slot=1),
_task("b1", state="failed", ago_min=8, slot=2),
_task("b0", state="failed", ago_min=40, slot=2),
_task("a1", state="failed", ago_min=300, slot=1)]
assert trouble_since(two, "degraded", False, NOW) == _ago(40)
# Healthy again, but a task failed today: dated from that failure.
recovered = [_task("t1", state="running", desired="running", ago_min=10),
_task("t2", state="failed", ago_min=60 * 5),
_task("t3", state="failed", ago_min=60 * 30)]
assert trouble_since(recovered, "healthy", False, NOW) == _ago(60 * 5)
# No task records: unknown.
assert trouble_since([], "down", False, NOW) is None
def test_the_view_dates_a_down_service():
tasks = [_task("t1", state="failed", ago_min=3, exit_code=1),
_task("t2", state="failed", ago_min=70, exit_code=1)]
(s,) = _view([], [_svc("db", 0, 1, [], tasks=tasks)])["services"]
assert s["state"] == "down" and s["trouble_since"] == _ago(70)