feat(docker): overview findings say when the problem began
#5013 "Overview findings for containers and Swarm services carry since=null" - Container problems are dated from their own evidence: a crash or restart loop from its first death in the hour, an OOM kill or error exit from the death that ended it, a failing healthcheck from its last turn to unhealthy. list_containers and get_container carry problem_since / problem_age_s. - Swarm services carry trouble_since: a crash loop's first failure in the hour, a short service's oldest troubled task in each failing slot, a recovered service's first failure in 24h. list_swarm_services shows it. - get_overview findings from the docker plugin now have since and age_s, so they rank by how long they have stood. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -137,3 +137,35 @@ async def test_prune_docker_refuses_an_unknown_target_before_reading_anything(mo
|
||||
with pytest.raises(ValueError, match="'volumes' is not one of containers, images, system"):
|
||||
await t.prune_docker("nas", "volumes")
|
||||
assert inspect.signature(t.prune_docker).parameters["check"].default is True
|
||||
|
||||
|
||||
def _death(event, minutes_ago):
|
||||
return NS(event=event, at=NOW - timedelta(minutes=minutes_ago))
|
||||
|
||||
|
||||
def test_problem_since_dates_each_problem_from_its_own_evidence():
|
||||
hour_ago = NOW - timedelta(hours=1)
|
||||
deaths = [_death("die", 5), _death("die", 20), _death("die", 50), _death("die", 300)]
|
||||
# A crash loop began with its first death inside the hour, not the older one.
|
||||
assert t.problem_since(["crash_looping"], deaths, hour_ago) == NOW - timedelta(minutes=50)
|
||||
# An error exit dates from the death that ended it.
|
||||
assert t.problem_since(["exited_nonzero"], deaths, hour_ago) == NOW - timedelta(minutes=5)
|
||||
oom = [_death("die", 2), _death("oom", 3)]
|
||||
assert t.problem_since(["oom_killed", "exited_nonzero"], oom, hour_ago) == NOW - timedelta(minutes=3)
|
||||
turned = NOW - timedelta(hours=2)
|
||||
assert t.problem_since(["unhealthy"], [], hour_ago, turned) == turned
|
||||
# No retained evidence: unknown, not now.
|
||||
assert t.problem_since(["unhealthy"], [], hour_ago) is None
|
||||
assert t.problem_since([], deaths, hour_ago) is None
|
||||
|
||||
|
||||
def test_findings_carry_when_the_problem_began():
|
||||
began = NOW - timedelta(minutes=40)
|
||||
row = t.container_row(_c(name="db", status="exited", exit_code=1), {}, ["exited_nonzero"],
|
||||
NOW, since=began)
|
||||
assert row["problem_since"] == "2026-10-06T11:20:00Z" and row["problem_age_s"] == 2400
|
||||
(f,) = t.container_findings([row])
|
||||
assert f["since"] == "2026-10-06T11:20:00Z" and f["age_s"] is not None
|
||||
svc = {"name": "db", "state": "down", "running": 0, "desired": 1, "trouble_since": began}
|
||||
(g,) = t.service_findings([svc])
|
||||
assert g["since"] == "2026-10-06T11:20:00Z"
|
||||
|
||||
@@ -3,7 +3,9 @@ import json
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from types import SimpleNamespace
|
||||
|
||||
from plugins.docker.swarm_view import build_swarm_services, last_task_error, task_failures
|
||||
from plugins.docker.swarm_view import (
|
||||
build_swarm_services, last_task_error, task_failures, trouble_since,
|
||||
)
|
||||
|
||||
NODE_HOSTNAME = {"nA": "docker01", "nB": "docker02", "nC": "docker03"}
|
||||
HOST_NAME = {"h1": "docker01", "h2": "docker02"} # agents run on docker01/02 only
|
||||
@@ -172,3 +174,40 @@ def test_the_newest_manager_report_wins():
|
||||
(svc,) = _view([], [old, new])["services"]
|
||||
assert svc["tasks_known"] is True and svc["failures_1h"] == 1
|
||||
|
||||
|
||||
|
||||
def _ago(minutes):
|
||||
return NOW - timedelta(minutes=minutes)
|
||||
|
||||
|
||||
def test_trouble_since_for_each_kind_of_trouble():
|
||||
loop = [_task("t1", state="failed", ago_min=2), _task("t2", state="failed", ago_min=20),
|
||||
_task("t3", state="failed", ago_min=45), _task("t4", state="failed", ago_min=90)]
|
||||
# A crash loop: its first failure inside the hour.
|
||||
assert trouble_since(loop, "down", True, NOW) == _ago(45)
|
||||
# Down: the oldest of the newest troubled tasks, back to a running one.
|
||||
down = [_task("t1", state="pending", ago_min=1, error="no suitable node"),
|
||||
_task("t2", state="rejected", ago_min=30, error="No such image"),
|
||||
_task("t3", state="running", desired="running", ago_min=600),
|
||||
_task("t4", state="failed", ago_min=900)]
|
||||
assert trouble_since(down, "degraded", False, NOW) == _ago(30)
|
||||
# Another slot's newer, healthy task does not hide the failing one.
|
||||
two = [_task("a2", state="running", desired="running", ago_min=5, slot=1),
|
||||
_task("b1", state="failed", ago_min=8, slot=2),
|
||||
_task("b0", state="failed", ago_min=40, slot=2),
|
||||
_task("a1", state="failed", ago_min=300, slot=1)]
|
||||
assert trouble_since(two, "degraded", False, NOW) == _ago(40)
|
||||
# Healthy again, but a task failed today: dated from that failure.
|
||||
recovered = [_task("t1", state="running", desired="running", ago_min=10),
|
||||
_task("t2", state="failed", ago_min=60 * 5),
|
||||
_task("t3", state="failed", ago_min=60 * 30)]
|
||||
assert trouble_since(recovered, "healthy", False, NOW) == _ago(60 * 5)
|
||||
# No task records: unknown.
|
||||
assert trouble_since([], "down", False, NOW) is None
|
||||
|
||||
|
||||
def test_the_view_dates_a_down_service():
|
||||
tasks = [_task("t1", state="failed", ago_min=3, exit_code=1),
|
||||
_task("t2", state="failed", ago_min=70, exit_code=1)]
|
||||
(s,) = _view([], [_svc("db", 0, 1, [], tasks=tasks)])["services"]
|
||||
assert s["state"] == "down" and s["trouble_since"] == _ago(70)
|
||||
|
||||
Reference in New Issue
Block a user