feat(docker): overview findings say when the problem began
#5013 "Overview findings for containers and Swarm services carry since=null" - Container problems are dated from their own evidence: a crash or restart loop from its first death in the hour, an OOM kill or error exit from the death that ended it, a failing healthcheck from its last turn to unhealthy. list_containers and get_container carry problem_since / problem_age_s. - Swarm services carry trouble_since: a crash loop's first failure in the hour, a short service's oldest troubled task in each failing slot, a recovered service's first failure in 24h. list_swarm_services shows it. - get_overview findings from the docker plugin now have since and age_s, so they rank by how long they have stood. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -3,7 +3,9 @@ import json
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from types import SimpleNamespace
|
||||
|
||||
from plugins.docker.swarm_view import build_swarm_services, last_task_error, task_failures
|
||||
from plugins.docker.swarm_view import (
|
||||
build_swarm_services, last_task_error, task_failures, trouble_since,
|
||||
)
|
||||
|
||||
NODE_HOSTNAME = {"nA": "docker01", "nB": "docker02", "nC": "docker03"}
|
||||
HOST_NAME = {"h1": "docker01", "h2": "docker02"} # agents run on docker01/02 only
|
||||
@@ -172,3 +174,40 @@ def test_the_newest_manager_report_wins():
|
||||
(svc,) = _view([], [old, new])["services"]
|
||||
assert svc["tasks_known"] is True and svc["failures_1h"] == 1
|
||||
|
||||
|
||||
|
||||
def _ago(minutes):
|
||||
return NOW - timedelta(minutes=minutes)
|
||||
|
||||
|
||||
def test_trouble_since_for_each_kind_of_trouble():
|
||||
loop = [_task("t1", state="failed", ago_min=2), _task("t2", state="failed", ago_min=20),
|
||||
_task("t3", state="failed", ago_min=45), _task("t4", state="failed", ago_min=90)]
|
||||
# A crash loop: its first failure inside the hour.
|
||||
assert trouble_since(loop, "down", True, NOW) == _ago(45)
|
||||
# Down: the oldest of the newest troubled tasks, back to a running one.
|
||||
down = [_task("t1", state="pending", ago_min=1, error="no suitable node"),
|
||||
_task("t2", state="rejected", ago_min=30, error="No such image"),
|
||||
_task("t3", state="running", desired="running", ago_min=600),
|
||||
_task("t4", state="failed", ago_min=900)]
|
||||
assert trouble_since(down, "degraded", False, NOW) == _ago(30)
|
||||
# Another slot's newer, healthy task does not hide the failing one.
|
||||
two = [_task("a2", state="running", desired="running", ago_min=5, slot=1),
|
||||
_task("b1", state="failed", ago_min=8, slot=2),
|
||||
_task("b0", state="failed", ago_min=40, slot=2),
|
||||
_task("a1", state="failed", ago_min=300, slot=1)]
|
||||
assert trouble_since(two, "degraded", False, NOW) == _ago(40)
|
||||
# Healthy again, but a task failed today: dated from that failure.
|
||||
recovered = [_task("t1", state="running", desired="running", ago_min=10),
|
||||
_task("t2", state="failed", ago_min=60 * 5),
|
||||
_task("t3", state="failed", ago_min=60 * 30)]
|
||||
assert trouble_since(recovered, "healthy", False, NOW) == _ago(60 * 5)
|
||||
# No task records: unknown.
|
||||
assert trouble_since([], "down", False, NOW) is None
|
||||
|
||||
|
||||
def test_the_view_dates_a_down_service():
|
||||
tasks = [_task("t1", state="failed", ago_min=3, exit_code=1),
|
||||
_task("t2", state="failed", ago_min=70, exit_code=1)]
|
||||
(s,) = _view([], [_svc("db", 0, 1, [], tasks=tasks)])["services"]
|
||||
assert s["state"] == "down" and s["trouble_since"] == _ago(70)
|
||||
|
||||
Reference in New Issue
Block a user