feat(docker): overview findings say when the problem began

#5013 "Overview findings for containers and Swarm services carry since=null"

- Container problems are dated from their own evidence: a crash or restart
  loop from its first death in the hour, an OOM kill or error exit from the
  death that ended it, a failing healthcheck from its last turn to unhealthy.
  list_containers and get_container carry problem_since / problem_age_s.
- Swarm services carry trouble_since: a crash loop's first failure in the
  hour, a short service's oldest troubled task in each failing slot, a
  recovered service's first failure in 24h. list_swarm_services shows it.
- get_overview findings from the docker plugin now have since and age_s,
  so they rank by how long they have stood.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
2026-10-06 10:24:10 -04:00
co-authored by Claude Opus 5.5
parent bac2859455
commit e9341c5639
6 changed files with 209 additions and 15 deletions
+40 -1
View File
@@ -3,7 +3,9 @@ import json
from datetime import datetime, timedelta, timezone
from types import SimpleNamespace
from plugins.docker.swarm_view import build_swarm_services, last_task_error, task_failures
from plugins.docker.swarm_view import (
build_swarm_services, last_task_error, task_failures, trouble_since,
)
NODE_HOSTNAME = {"nA": "docker01", "nB": "docker02", "nC": "docker03"}
HOST_NAME = {"h1": "docker01", "h2": "docker02"} # agents run on docker01/02 only
@@ -172,3 +174,40 @@ def test_the_newest_manager_report_wins():
(svc,) = _view([], [old, new])["services"]
assert svc["tasks_known"] is True and svc["failures_1h"] == 1
def _ago(minutes):
return NOW - timedelta(minutes=minutes)
def test_trouble_since_for_each_kind_of_trouble():
loop = [_task("t1", state="failed", ago_min=2), _task("t2", state="failed", ago_min=20),
_task("t3", state="failed", ago_min=45), _task("t4", state="failed", ago_min=90)]
# A crash loop: its first failure inside the hour.
assert trouble_since(loop, "down", True, NOW) == _ago(45)
# Down: the oldest of the newest troubled tasks, back to a running one.
down = [_task("t1", state="pending", ago_min=1, error="no suitable node"),
_task("t2", state="rejected", ago_min=30, error="No such image"),
_task("t3", state="running", desired="running", ago_min=600),
_task("t4", state="failed", ago_min=900)]
assert trouble_since(down, "degraded", False, NOW) == _ago(30)
# Another slot's newer, healthy task does not hide the failing one.
two = [_task("a2", state="running", desired="running", ago_min=5, slot=1),
_task("b1", state="failed", ago_min=8, slot=2),
_task("b0", state="failed", ago_min=40, slot=2),
_task("a1", state="failed", ago_min=300, slot=1)]
assert trouble_since(two, "degraded", False, NOW) == _ago(40)
# Healthy again, but a task failed today: dated from that failure.
recovered = [_task("t1", state="running", desired="running", ago_min=10),
_task("t2", state="failed", ago_min=60 * 5),
_task("t3", state="failed", ago_min=60 * 30)]
assert trouble_since(recovered, "healthy", False, NOW) == _ago(60 * 5)
# No task records: unknown.
assert trouble_since([], "down", False, NOW) is None
def test_the_view_dates_a_down_service():
tasks = [_task("t1", state="failed", ago_min=3, exit_code=1),
_task("t2", state="failed", ago_min=70, exit_code=1)]
(s,) = _view([], [_svc("db", 0, 1, [], tasks=tasks)])["services"]
assert s["state"] == "down" and s["trouble_since"] == _ago(70)