bvandeusenandClaude Opus 5.5
213a28c23f
A verdict line (services healthy · other containers running · failed in 24h), then "Needs a look": troubled services with why (crash-looping, N of M running), since when and the last error, and troubled containers (unhealthy, restarting, exited with an error, out of memory). Then "Failed in the last 24h" by name, with how often, when and where. Then the healthy rest one line each (name · hosts · replicas) in as many columns as the widget is wide, collapsing after 40 to "+N more healthy". Per-container CPU is gone from this widget: Busiest Containers owns it. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
211 lines
13 KiB
Python
211 lines
13 KiB
Python
"""The Docker page leads with what needs attention, ranked the way the MCP
|
|
tools rank it, and marks a container whose report is old as stale (#5123)."""
|
|
import re
|
|
from datetime import datetime, timedelta, timezone
|
|
from types import SimpleNamespace as NS
|
|
|
|
|
|
from plugins.docker import service as svc
|
|
from plugins.docker.routes import _is_stale
|
|
from tests._templates import render_plugin_template
|
|
from tests.test_style_tokens import hex_colours
|
|
|
|
NOW = datetime.now(timezone.utc)
|
|
|
|
|
|
def _c(name, **kw):
|
|
base = dict(name=name, host_id="h1", image="nginx:1", status="running", health=None, exit_code=None,
|
|
oom_killed=False, restart_count=0, cpu_pct=5.0, mem_pct=10.0, compose_project=None,
|
|
service_name=None, ports_json=None, started_at=NOW - timedelta(hours=2),
|
|
scraped_at=NOW - timedelta(seconds=20))
|
|
base.update(kw)
|
|
return NS(**base)
|
|
|
|
|
|
def test_needs_attention_ranks_the_worst_problem_first():
|
|
ranked = [{"container": _c(n), "problems": p, "since": None, "replaced": False} for n, p in [
|
|
("fine", []), ("oom", ["oom_killed"]), ("sick", ["unhealthy"]),
|
|
("loop", ["crash_looping"]), ("exit", ["exited_nonzero"]), ("bounce", ["restarting", "crash_looping"])]]
|
|
assert [r["container"].name for r in svc.needs_attention(ranked)] == ["bounce", "loop", "sick", "oom", "exit"]
|
|
|
|
|
|
def test_stale_is_older_than_the_agents_window():
|
|
assert not _is_stale(_c("a"), NOW, 180)
|
|
assert _is_stale(_c("a", scraped_at=NOW - timedelta(minutes=10)), NOW, 180)
|
|
assert _is_stale(_c("a", scraped_at=None), NOW, 180)
|
|
# A fresh row on a host whose Docker stopped answering is stale too (#5152).
|
|
assert _is_stale(_c("a"), NOW, 180, {"h1": object()})
|
|
assert not _is_stale(_c("a"), NOW, 180, {"h2": object()})
|
|
assert svc.stale_after_seconds({"PLUGINS": {"host_agent": {"stale_after_seconds": 60}}}) == 60
|
|
assert svc.stale_after_seconds({}) == 180
|
|
|
|
|
|
async def test_a_crash_loop_comes_first_and_a_stale_row_says_so(app):
|
|
loop = _c("api", status="restarting", exit_code=1, restart_count=7)
|
|
old = _c("cache", scraped_at=NOW - timedelta(minutes=30), cpu_pct=95.0)
|
|
attention = [{"container": loop, "label": "api", "problems": ["restarting", "crash_looping"], "replaced": False,
|
|
"since": datetime.now(timezone.utc) - timedelta(minutes=8), "stale": False}]
|
|
groups = [{"host": None, "host_id": "h1", "host_name": "nas", "running": 1, "stopped": 1, "grouped": False,
|
|
"subgroups": [{"label": None, "containers": [
|
|
{"container": loop, "ports": [], "uptime": None, "stale": False,
|
|
"usage": svc.usage(loop, 1, None), "exit_failed": True},
|
|
{"container": old, "ports": [], "uptime": "2h", "stale": True,
|
|
"usage": svc.usage(old, 1, None), "exit_failed": False}]}]}]
|
|
async with app.test_request_context("/plugins/docker/rows"):
|
|
page = await render_plugin_template(
|
|
app, "docker", "docker/rows.html", attention=attention, says=svc.PROBLEM_SAYS, severity=svc.PROBLEM_SEVERITY,
|
|
unhealthy=0, restarting=1, stale=1, host_groups=groups, swarm_services=[],
|
|
running=1, stopped=1, docker_hosts=1, failures=[], failing_services=[])
|
|
page = re.sub(r">\s+<", "><", " ".join(page.split()))
|
|
|
|
card = page[page.index("Needs attention"):page.index("</section>")]
|
|
assert '<span class="badge badge-crit">Restarting</span>' in card
|
|
assert 'href="/plugins/docker/container/h1/api" title="api">api</a>' in card
|
|
assert "is restarting and keeps dying · exit 1 · since <time" in card and ">8m ago</time>" in card
|
|
assert "cache" not in card
|
|
assert page.index("Needs attention") < page.index("<table")
|
|
|
|
assert '<span class="stat-tile-label">Restarting</span><span class="stat-tile-value">1</span>' in page
|
|
assert '<span class="stat-tile-label">Stale</span><span class="stat-tile-value">1</span>' in page
|
|
assert 'class="badge badge-warn" title="Last report' in page # the stale row
|
|
# CPU in cores, toned by its share of the host's CPUs through Settings → Thresholds.
|
|
assert ('<span class="tone-crit" title="Docker counts CPU per core: 95% of one core, '
|
|
'95.0% of this host\'s 1 core">0.95 cores</span>') in page
|
|
assert "history" not in page.lower() and "<svg" not in page # no sparklines on the overview
|
|
assert not hex_colours(page), hex_colours(page)
|
|
|
|
|
|
async def test_a_healthy_board_has_no_attention_card(app):
|
|
async with app.test_request_context("/plugins/docker/rows"):
|
|
page = await render_plugin_template(
|
|
app, "docker", "docker/rows.html", attention=[], says=svc.PROBLEM_SAYS, severity=svc.PROBLEM_SEVERITY,
|
|
unhealthy=0, restarting=0, stale=0, host_groups=[], swarm_services=[], running=0, stopped=0,
|
|
docker_hosts=0, failures=[], failing_services=[])
|
|
assert "Needs attention</h2>" not in page and "No containers reported yet" in page
|
|
|
|
|
|
async def test_a_host_whose_docker_stopped_answering_says_so(app):
|
|
now = datetime.now(timezone.utc) # not the module's NOW: |ago reads the clock
|
|
down = [{"host_id": "h1", "host_name": "nas", "status": "unavailable",
|
|
"detail": "no Docker socket at /var/run/docker.sock", "since": now - timedelta(minutes=5)},
|
|
{"host_id": "h2", "host_name": "web-1", "status": "error",
|
|
"detail": "permission denied on /var/run/docker.sock", "since": now - timedelta(minutes=1)}]
|
|
async with app.test_request_context("/plugins/docker/rows"):
|
|
page = await render_plugin_template(
|
|
app, "docker", "docker/rows.html", docker_down=down, attention=[], says=svc.PROBLEM_SAYS,
|
|
severity=svc.PROBLEM_SEVERITY, unhealthy=0, restarting=0, stale=0, host_groups=[],
|
|
swarm_services=[], running=0, stopped=0, docker_hosts=0, failures=[], failing_services=[])
|
|
page = re.sub(r">\s+<", "><", " ".join(page.split()))
|
|
card = page[page.index("Docker not answering"):page.index("</section>")]
|
|
assert '<span class="badge badge-warn">No Docker</span><a href="/hosts/h1">nas</a>' in card
|
|
assert "no Docker socket at /var/run/docker.sock · since <time" in card and ">5m " in card
|
|
assert '<span class="badge badge-crit">Error</span><a href="/hosts/h2">web-1</a>' in card
|
|
|
|
|
|
async def test_no_card_while_every_docker_answers(app):
|
|
async with app.test_request_context("/plugins/docker/rows"):
|
|
page = await render_plugin_template(
|
|
app, "docker", "docker/rows.html", docker_down=[], attention=[], says=svc.PROBLEM_SAYS,
|
|
severity=svc.PROBLEM_SEVERITY, unhealthy=0, restarting=0, stale=0, host_groups=[],
|
|
swarm_services=[], running=0, stopped=0, docker_hosts=0, failures=[], failing_services=[])
|
|
assert "Docker not answering" not in page
|
|
|
|
|
|
async def test_a_failing_service_leads_and_the_failures_are_listed(app):
|
|
now = datetime.now(timezone.utc)
|
|
sick = {"name": "codex_db", "mode": "replicated", "running": 0, "desired": 1, "state": "down",
|
|
"crash_looping": True, "failures_1h": 7, "failures_24h": 9, "tasks_known": True,
|
|
"trouble_since": now - timedelta(minutes=20), "replicas": [], "previous_tasks": 7,
|
|
"last_error": {"state": "failed", "error": "task: non-zero exit (1)", "exit_code": 1,
|
|
"node": "nas", "at": None, "age_s": 120}}
|
|
failures = [{"kind": "service", "name": "codex_db", "host_id": None, "host": "nas", "count": 9,
|
|
"at": now - timedelta(minutes=2), "detail": "task: non-zero exit (1)"},
|
|
{"kind": "container", "name": "backup", "host_id": "h1", "host": "nas", "count": 1,
|
|
"at": now - timedelta(hours=3), "detail": "exit 2"}]
|
|
async with app.test_request_context("/plugins/docker/rows"):
|
|
page = await render_plugin_template(
|
|
app, "docker", "docker/rows.html", docker_down=[], attention=[], says=svc.PROBLEM_SAYS,
|
|
severity=svc.PROBLEM_SEVERITY, unhealthy=0, restarting=0, stale=0, host_groups=[],
|
|
swarm_services=[sick], services_healthy=0, failing_services=[sick], failures=failures,
|
|
running=0, stopped=0, docker_hosts=1)
|
|
page = re.sub(r">\s+<", "><", " ".join(page.split()))
|
|
card = page[page.index("Needs attention"):page.index("</section>")]
|
|
assert '<span class="badge badge-crit">Crash-looping</span><a href="/plugins/docker/swarm">codex_db</a>' in card
|
|
assert "0 of 1 running · 7 tasks failed in the last hour · last task failed: task: non-zero exit (1)" in card
|
|
assert "since <time" in card
|
|
assert '<span class="stat-tile-label">Swarm services</span><span class="stat-tile-value">0/1</span>' in page
|
|
listed = page[page.index('id="failures"'):]
|
|
listed = listed[:listed.index("</section>")]
|
|
assert listed.index("codex_db") < listed.index("backup") # newest first
|
|
assert "9 failures · last: task: non-zero exit (1) · nas" in listed
|
|
assert 'href="/plugins/docker/container/h1/backup">backup</a>' in listed
|
|
|
|
|
|
async def test_the_disk_page_explains_its_terms_and_names_untagged_images(app):
|
|
"""#5212: no unexplained jargon, no "<none>" rows, and a capped list says so."""
|
|
summary = {"images_total": 312, "images_active": 40, "images_size": "40 GiB",
|
|
"images_reclaimable": "12 GiB", "layers_size": "30 GiB", "containers_count": 9,
|
|
"containers_size": "1.2 GiB", "volumes_count": 4, "volumes_size": "3 GiB",
|
|
"build_cache_size": "2 GiB"}
|
|
images = [{"repo_tag": "nginx:1", "image_id": "aaaaaaaaaaaa", "size": "180 MiB", "shared": "70 MiB",
|
|
"containers": 2, "reclaimable": False},
|
|
{"repo_tag": None, "image_id": "bbbbbbbbbbbb", "size": "700 MiB", "shared": "0 B",
|
|
"containers": 0, "reclaimable": True}]
|
|
group = {"host_id": "h1", "host_name": "nas", "as_of": NOW - timedelta(minutes=2), "summary": summary,
|
|
"stopped": 1, "images": images, "has_target": True, "images_capped": True}
|
|
async with app.test_request_context("/plugins/docker/disk"):
|
|
page = await render_plugin_template(app, "docker", "docker/disk.html",
|
|
host_groups=[group], ansible_available=True)
|
|
assert "untagged</span> bbbbbbbbbbbb" in page and "<none>" not in page
|
|
assert "The 2 largest of 312 images" in page and "as of <time" in page
|
|
assert "Writable layers" not in page and "Container files" in page
|
|
assert "held by images no container uses" in page and "safe to clear" in page
|
|
assert "Shared with other images" in page and "2 containers" in page and ">nothing<" in page
|
|
assert "dangling" not in page
|
|
|
|
|
|
def _widget_svc(name, state="healthy", running=1, desired=1, hosts=("VDNT-Docker02",), **kw):
|
|
base = dict(name=name, state=state, running=running, desired=desired, crash_looping=False,
|
|
failures_1h=0, last_error=None, trouble_since=None,
|
|
replicas=[{"host": h, "ghost": False} for h in hosts])
|
|
base.update(kw)
|
|
return base
|
|
|
|
|
|
async def _services_widget(app, **ctx):
|
|
base = dict(bad_services=[], bad_containers=[], healthy=[], healthy_more=0, services_total=0,
|
|
services_healthy=0, running_count=0, failures=[], host_name={"h1": "VDNT-Docker02"},
|
|
total_count=1, widget_id="1")
|
|
base.update(ctx)
|
|
async with app.test_request_context("/plugins/docker/widget"):
|
|
return await render_plugin_template(app, "docker", "docker/widget.html", **base)
|
|
|
|
|
|
async def test_the_services_widget_lists_what_is_wrong_then_packs_the_healthy(app):
|
|
"""#5252: a verdict, the troubled service with why and since when, the
|
|
failures by name, then healthy services one line each in a packed grid."""
|
|
bad = _widget_svc("codex_db", state="down", running=0, crash_looping=True, failures_1h=7,
|
|
trouble_since=datetime.now(timezone.utc) - timedelta(minutes=9))
|
|
page = await _services_widget(
|
|
app, bad_services=[bad], services_total=3, services_healthy=2, running_count=4,
|
|
failures=[{"name": "codex_db", "host": "VDNT-Docker03", "count": 7, "at": NOW - timedelta(minutes=1),
|
|
"detail": "exit 1"}],
|
|
healthy=[{"kind": "service", "s": _widget_svc("curator_app")},
|
|
{"kind": "service", "s": _widget_svc("web", running=2, desired=2, hosts=("a", "b", "a"))},
|
|
{"kind": "container", "c": _c("portainer_agent")}],
|
|
healthy_more=12)
|
|
assert "2/3 services healthy" in page and "4 other containers running" in page and "1 failed in 24h" in page
|
|
assert page.index("Needs a look") < page.index("Failed in the last 24h") < page.index("Healthy")
|
|
assert "crash-looping: 7 failed in the last hour" in page and "9m ago" in page
|
|
assert "7 times, last" in page and "exit 1" in page
|
|
assert 'class="w-grid"' in page and " · a, b</span>" in page and ">2/2<" in page
|
|
assert "+12 more healthy" in page
|
|
assert not hex_colours(page), hex_colours(page)
|
|
|
|
|
|
async def test_the_services_widget_says_when_all_is_well(app):
|
|
page = await _services_widget(app, services_total=2, services_healthy=2, running_count=1,
|
|
healthy=[{"kind": "service", "s": _widget_svc("a")}])
|
|
assert "2/2 services healthy" in page and "nothing failed in 24h" in page and "tone-ok" in page
|
|
assert "Needs a look" not in page and "Failed in the last 24h" not in page
|