bvandeusenandClaude Opus 5.5
05eaca3f2f
The fill bars were a share of the host's capacity, a sliver on hosts with 80 cores. Each figure now sits beside 24 bars of the last 6 hours (1h/24h per widget), each row scaled to its own peak but never below 1 core or 1 GiB (configurable per widget), so a 0.05-core container reads flat and a bar's height means real load. One grouped date_bin query feeds them. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
270 lines
17 KiB
Python
270 lines
17 KiB
Python
"""The Docker page leads with what needs attention, ranked the way the MCP
|
|
tools rank it, and marks a container whose report is old as stale (#5123)."""
|
|
import re
|
|
from datetime import datetime, timedelta, timezone
|
|
from types import SimpleNamespace as NS
|
|
|
|
|
|
from plugins.docker import service as svc
|
|
from plugins.docker.routes import _is_stale
|
|
from tests._templates import render_plugin_template
|
|
from tests.test_style_tokens import hex_colours
|
|
|
|
NOW = datetime.now(timezone.utc)
|
|
|
|
|
|
def _c(name, **kw):
|
|
base = dict(name=name, host_id="h1", image="nginx:1", status="running", health=None, exit_code=None,
|
|
oom_killed=False, restart_count=0, cpu_pct=5.0, mem_pct=10.0, compose_project=None,
|
|
service_name=None, ports_json=None, started_at=NOW - timedelta(hours=2),
|
|
scraped_at=NOW - timedelta(seconds=20))
|
|
base.update(kw)
|
|
return NS(**base)
|
|
|
|
|
|
def test_needs_attention_ranks_the_worst_problem_first():
|
|
ranked = [{"container": _c(n), "problems": p, "since": None, "replaced": False} for n, p in [
|
|
("fine", []), ("oom", ["oom_killed"]), ("sick", ["unhealthy"]),
|
|
("loop", ["crash_looping"]), ("exit", ["exited_nonzero"]), ("bounce", ["restarting", "crash_looping"])]]
|
|
assert [r["container"].name for r in svc.needs_attention(ranked)] == ["bounce", "loop", "sick", "oom", "exit"]
|
|
|
|
|
|
def test_stale_is_older_than_the_agents_window():
|
|
assert not _is_stale(_c("a"), NOW, 180)
|
|
assert _is_stale(_c("a", scraped_at=NOW - timedelta(minutes=10)), NOW, 180)
|
|
assert _is_stale(_c("a", scraped_at=None), NOW, 180)
|
|
# A fresh row on a host whose Docker stopped answering is stale too (#5152).
|
|
assert _is_stale(_c("a"), NOW, 180, {"h1": object()})
|
|
assert not _is_stale(_c("a"), NOW, 180, {"h2": object()})
|
|
assert svc.stale_after_seconds({"PLUGINS": {"host_agent": {"stale_after_seconds": 60}}}) == 60
|
|
assert svc.stale_after_seconds({}) == 180
|
|
|
|
|
|
async def test_a_crash_loop_comes_first_and_a_stale_row_says_so(app):
|
|
loop = _c("api", status="restarting", exit_code=1, restart_count=7)
|
|
old = _c("cache", scraped_at=NOW - timedelta(minutes=30), cpu_pct=95.0)
|
|
attention = [{"container": loop, "label": "api", "problems": ["restarting", "crash_looping"], "replaced": False,
|
|
"since": datetime.now(timezone.utc) - timedelta(minutes=8), "stale": False}]
|
|
groups = [{"host": None, "host_id": "h1", "host_name": "nas", "running": 1, "stopped": 1, "grouped": False,
|
|
"subgroups": [{"label": None, "containers": [
|
|
{"container": loop, "ports": [], "uptime": None, "stale": False,
|
|
"usage": svc.usage(loop, 1, None), "exit_failed": True},
|
|
{"container": old, "ports": [], "uptime": "2h", "stale": True,
|
|
"usage": svc.usage(old, 1, None), "exit_failed": False}]}]}]
|
|
async with app.test_request_context("/plugins/docker/rows"):
|
|
page = await render_plugin_template(
|
|
app, "docker", "docker/rows.html", attention=attention, says=svc.PROBLEM_SAYS, severity=svc.PROBLEM_SEVERITY,
|
|
unhealthy=0, restarting=1, stale=1, host_groups=groups, swarm_services=[],
|
|
running=1, stopped=1, docker_hosts=1, failures=[], failing_services=[])
|
|
page = re.sub(r">\s+<", "><", " ".join(page.split()))
|
|
|
|
card = page[page.index("Needs attention"):page.index("</section>")]
|
|
assert '<span class="badge badge-crit">Restarting</span>' in card
|
|
assert 'href="/plugins/docker/container/h1/api" title="api">api</a>' in card
|
|
assert "is restarting and keeps dying · exit 1 · since <time" in card and ">8m ago</time>" in card
|
|
assert "cache" not in card
|
|
assert page.index("Needs attention") < page.index("<table")
|
|
|
|
assert '<span class="stat-tile-label">Restarting</span><span class="stat-tile-value">1</span>' in page
|
|
assert '<span class="stat-tile-label">Stale</span><span class="stat-tile-value">1</span>' in page
|
|
assert 'class="badge badge-warn" title="Last report' in page # the stale row
|
|
# CPU in cores, toned by its share of the host's CPUs through Settings → Thresholds.
|
|
assert ('<span class="tone-crit" title="Docker counts CPU per core: 95% of one core, '
|
|
'95.0% of this host\'s 1 core">0.95 cores</span>') in page
|
|
assert "history" not in page.lower() and "<svg" not in page # no sparklines on the overview
|
|
assert not hex_colours(page), hex_colours(page)
|
|
|
|
|
|
async def test_a_healthy_board_has_no_attention_card(app):
|
|
async with app.test_request_context("/plugins/docker/rows"):
|
|
page = await render_plugin_template(
|
|
app, "docker", "docker/rows.html", attention=[], says=svc.PROBLEM_SAYS, severity=svc.PROBLEM_SEVERITY,
|
|
unhealthy=0, restarting=0, stale=0, host_groups=[], swarm_services=[], running=0, stopped=0,
|
|
docker_hosts=0, failures=[], failing_services=[])
|
|
assert "Needs attention</h2>" not in page and "No containers reported yet" in page
|
|
|
|
|
|
async def test_a_host_whose_docker_stopped_answering_says_so(app):
|
|
now = datetime.now(timezone.utc) # not the module's NOW: |ago reads the clock
|
|
down = [{"host_id": "h1", "host_name": "nas", "status": "unavailable",
|
|
"detail": "no Docker socket at /var/run/docker.sock", "since": now - timedelta(minutes=5)},
|
|
{"host_id": "h2", "host_name": "web-1", "status": "error",
|
|
"detail": "permission denied on /var/run/docker.sock", "since": now - timedelta(minutes=1)}]
|
|
async with app.test_request_context("/plugins/docker/rows"):
|
|
page = await render_plugin_template(
|
|
app, "docker", "docker/rows.html", docker_down=down, attention=[], says=svc.PROBLEM_SAYS,
|
|
severity=svc.PROBLEM_SEVERITY, unhealthy=0, restarting=0, stale=0, host_groups=[],
|
|
swarm_services=[], running=0, stopped=0, docker_hosts=0, failures=[], failing_services=[])
|
|
page = re.sub(r">\s+<", "><", " ".join(page.split()))
|
|
card = page[page.index("Docker not answering"):page.index("</section>")]
|
|
assert '<span class="badge badge-warn">No Docker</span><a href="/hosts/h1">nas</a>' in card
|
|
assert "no Docker socket at /var/run/docker.sock · since <time" in card and ">5m " in card
|
|
assert '<span class="badge badge-crit">Error</span><a href="/hosts/h2">web-1</a>' in card
|
|
|
|
|
|
async def test_no_card_while_every_docker_answers(app):
|
|
async with app.test_request_context("/plugins/docker/rows"):
|
|
page = await render_plugin_template(
|
|
app, "docker", "docker/rows.html", docker_down=[], attention=[], says=svc.PROBLEM_SAYS,
|
|
severity=svc.PROBLEM_SEVERITY, unhealthy=0, restarting=0, stale=0, host_groups=[],
|
|
swarm_services=[], running=0, stopped=0, docker_hosts=0, failures=[], failing_services=[])
|
|
assert "Docker not answering" not in page
|
|
|
|
|
|
async def test_a_failing_service_leads_and_the_failures_are_listed(app):
|
|
now = datetime.now(timezone.utc)
|
|
sick = {"name": "codex_db", "mode": "replicated", "running": 0, "desired": 1, "state": "down",
|
|
"crash_looping": True, "failures_1h": 7, "failures_24h": 9, "tasks_known": True,
|
|
"trouble_since": now - timedelta(minutes=20), "replicas": [], "previous_tasks": 7,
|
|
"last_error": {"state": "failed", "error": "task: non-zero exit (1)", "exit_code": 1,
|
|
"node": "nas", "at": None, "age_s": 120}}
|
|
failures = [{"kind": "service", "name": "codex_db", "host_id": None, "host": "nas", "count": 9,
|
|
"at": now - timedelta(minutes=2), "detail": "task: non-zero exit (1)"},
|
|
{"kind": "container", "name": "backup", "host_id": "h1", "host": "nas", "count": 1,
|
|
"at": now - timedelta(hours=3), "detail": "exit 2"}]
|
|
async with app.test_request_context("/plugins/docker/rows"):
|
|
page = await render_plugin_template(
|
|
app, "docker", "docker/rows.html", docker_down=[], attention=[], says=svc.PROBLEM_SAYS,
|
|
severity=svc.PROBLEM_SEVERITY, unhealthy=0, restarting=0, stale=0, host_groups=[],
|
|
swarm_services=[sick], services_healthy=0, failing_services=[sick], failures=failures,
|
|
running=0, stopped=0, docker_hosts=1)
|
|
page = re.sub(r">\s+<", "><", " ".join(page.split()))
|
|
card = page[page.index("Needs attention"):page.index("</section>")]
|
|
assert '<span class="badge badge-crit">Crash-looping</span><a href="/plugins/docker/swarm">codex_db</a>' in card
|
|
assert "0 of 1 running · 7 tasks failed in the last hour · last task failed: task: non-zero exit (1)" in card
|
|
assert "since <time" in card
|
|
assert '<span class="stat-tile-label">Swarm services</span><span class="stat-tile-value">0/1</span>' in page
|
|
listed = page[page.index('id="failures"'):]
|
|
listed = listed[:listed.index("</section>")]
|
|
assert listed.index("codex_db") < listed.index("backup") # newest first
|
|
assert "9 failures · last: task: non-zero exit (1) · nas" in listed
|
|
assert 'href="/plugins/docker/container/h1/backup">backup</a>' in listed
|
|
|
|
|
|
async def test_the_disk_page_explains_its_terms_and_names_untagged_images(app):
|
|
"""#5212: no unexplained jargon, no "<none>" rows, and a capped list says so."""
|
|
summary = {"images_total": 312, "images_active": 40, "images_size": "40 GiB",
|
|
"images_reclaimable": "12 GiB", "layers_size": "30 GiB", "containers_count": 9,
|
|
"containers_size": "1.2 GiB", "volumes_count": 4, "volumes_size": "3 GiB",
|
|
"build_cache_size": "2 GiB"}
|
|
images = [{"repo_tag": "nginx:1", "image_id": "aaaaaaaaaaaa", "size": "180 MiB", "shared": "70 MiB",
|
|
"containers": 2, "reclaimable": False},
|
|
{"repo_tag": None, "image_id": "bbbbbbbbbbbb", "size": "700 MiB", "shared": "0 B",
|
|
"containers": 0, "reclaimable": True}]
|
|
group = {"host_id": "h1", "host_name": "nas", "as_of": NOW - timedelta(minutes=2), "summary": summary,
|
|
"stopped": 1, "images": images, "has_target": True, "images_capped": True}
|
|
async with app.test_request_context("/plugins/docker/disk"):
|
|
page = await render_plugin_template(app, "docker", "docker/disk.html",
|
|
host_groups=[group], ansible_available=True)
|
|
assert "untagged</span> bbbbbbbbbbbb" in page and "<none>" not in page
|
|
assert "The 2 largest of 312 images" in page and "as of <time" in page
|
|
assert "Writable layers" not in page and "Container files" in page
|
|
assert "held by images no container uses" in page and "safe to clear" in page
|
|
assert "Shared with other images" in page and "2 containers" in page and ">nothing<" in page
|
|
assert "dangling" not in page
|
|
|
|
|
|
def _widget_svc(name, state="healthy", running=1, desired=1, hosts=("VDNT-Docker02",), **kw):
|
|
base = dict(name=name, state=state, running=running, desired=desired, crash_looping=False,
|
|
failures_1h=0, last_error=None, trouble_since=None,
|
|
replicas=[{"host": h, "ghost": False} for h in hosts])
|
|
base.update(kw)
|
|
return base
|
|
|
|
|
|
async def _services_widget(app, **ctx):
|
|
base = dict(bad_services=[], bad_containers=[], healthy=[], healthy_more=0, services_total=0,
|
|
services_healthy=0, running_count=0, failures=[], host_name={"h1": "VDNT-Docker02"},
|
|
total_count=1, widget_id="1")
|
|
base.update(ctx)
|
|
async with app.test_request_context("/plugins/docker/widget"):
|
|
return await render_plugin_template(app, "docker", "docker/widget.html", **base)
|
|
|
|
|
|
async def test_the_services_widget_lists_what_is_wrong_then_packs_the_healthy(app):
|
|
"""#5252: a verdict, the troubled service with why and since when, the
|
|
failures by name, then healthy services one line each in a packed grid."""
|
|
bad = _widget_svc("codex_db", state="down", running=0, crash_looping=True, failures_1h=7,
|
|
trouble_since=datetime.now(timezone.utc) - timedelta(minutes=9))
|
|
page = await _services_widget(
|
|
app, bad_services=[bad], services_total=3, services_healthy=2, running_count=4,
|
|
failures=[{"name": "codex_db", "host": "VDNT-Docker03", "count": 7, "at": NOW - timedelta(minutes=1),
|
|
"detail": "exit 1"}],
|
|
healthy=[{"kind": "service", "s": _widget_svc("curator_app")},
|
|
{"kind": "service", "s": _widget_svc("web", running=2, desired=2, hosts=("a", "b", "a"))},
|
|
{"kind": "container", "c": _c("portainer_agent")}],
|
|
healthy_more=12)
|
|
assert "2/3 services healthy" in page and "4 other containers running" in page and "1 failed in 24h" in page
|
|
assert page.index("Needs a look") < page.index("Failed in the last 24h") < page.index("Healthy")
|
|
assert "crash-looping: 7 failed in the last hour" in page and "9m ago" in page
|
|
assert "7 times, last" in page and "exit 1" in page
|
|
assert 'class="w-grid"' in page and " · a, b</span>" in page and ">2/2<" in page
|
|
assert "+12 more healthy" in page
|
|
assert not hex_colours(page), hex_colours(page)
|
|
|
|
|
|
async def test_the_services_widget_says_when_all_is_well(app):
|
|
page = await _services_widget(app, services_total=2, services_healthy=2, running_count=1,
|
|
healthy=[{"kind": "service", "s": _widget_svc("a")}])
|
|
assert "2/2 services healthy" in page and "nothing failed in 24h" in page and "tone-ok" in page
|
|
assert "Needs a look" not in page and "Failed in the last 24h" not in page
|
|
|
|
|
|
def test_a_container_reads_by_its_service_or_ci_job():
|
|
"""#5253: no task id, no slot number, no forty-character CI container name."""
|
|
assert svc.display_name(_c("scribe_app.1.x7q", service_name="scribe_app")) == "scribe_app"
|
|
ci = "GITEA-ACTIONS-TASK-32792-WORKFLOW-Desktop-Tauri-JOB-verify-Web-d718b95dad0b02ad3cc1"
|
|
assert svc.display_name(_c(ci)) == "CI: Desktop-Tauri · verify-Web"
|
|
assert svc.display_name(_c("portainer_agent")) == "portainer_agent"
|
|
|
|
|
|
def _bars(values, floor):
|
|
return svc.scaled_bars(values, floor)
|
|
|
|
|
|
async def test_busiest_containers_show_the_number_beside_its_recent_bars(app):
|
|
"""#5253, #5214: every figure has its unit; the bars beside it are the last
|
|
hours, scaled to the row's own peak; the widget names the busiest."""
|
|
rows = [
|
|
{"c": _c("GITEA-ACTIONS-TASK-1-WORKFLOW-Android-JOB-build-0123456789abcdef0"),
|
|
"label": "CI: Android · build", "host_name": "VDNT-Docker03",
|
|
"usage": {"cores": 1.62, "cpus": 8, "cpu_share": 20.25, "mem_bytes": 820 * 2**20,
|
|
"mem_pct": 5.0, "mem_of": "the host's memory", "mem_limit": None},
|
|
"cpu_bars": _bars([None, 0.8, 3.2, 1.6], 1.0), "mem_bars": _bars([None] * 4, 2**30)},
|
|
{"c": _c("tdarr"), "label": "tdarr", "host_name": "VDNT-Docker03",
|
|
"usage": {"cores": 0.25, "cpus": 8, "cpu_share": 3.1, "mem_bytes": None,
|
|
"mem_pct": None, "mem_of": None, "mem_limit": None},
|
|
"cpu_bars": _bars([0.25] * 4, 1.0), "mem_bars": _bars([None] * 4, 2**30)},
|
|
]
|
|
async with app.test_request_context("/plugins/docker/widget/resources"):
|
|
page = await render_plugin_template(app, "docker", "docker/widget_resources.html",
|
|
rows=rows, minutes=5, hours=6, cpu_floor=1.0,
|
|
mem_floor=2**30, widget_id="1")
|
|
assert "Busiest: <strong>CI: Android · build</strong> on VDNT-Docker03" in page and "1.6 of 8 cores" in page
|
|
assert ">1.6 cores<" in page and ">0.25 cores<" in page and ">820.0 MiB<" in page and ">—<" in page
|
|
# Android peaks at 3.2 cores, so 1.6 is half height; tdarr's 0.25 sits under the 1-core floor.
|
|
assert 'class="gap"' in page and "height:50.0%" in page and "height:25.0%" in page
|
|
assert "Peak 3.2 cores in the last 6h; bars reach the top at 3.2 cores" in page
|
|
assert "Peak 0.25 cores in the last 6h; bars reach the top at 1.0 cores" in page
|
|
assert "No samples in the last 6h" in page
|
|
assert len(re.findall(r"<th[ >]", page)) == 4
|
|
assert ">CI: Android · build<" in page # the long CI name only as the hover title
|
|
assert not hex_colours(page), hex_colours(page)
|
|
|
|
|
|
def test_bars_reach_the_peak_but_never_below_the_floor():
|
|
"""#5214: a 0.05 → 0.10 wobble stays near the floor; a 1.5 → 3 spike fills."""
|
|
idle = svc.scaled_bars([0.05, 0.10, None], 1.0)
|
|
assert idle == {"bars": [5.0, 10.0, None], "peak": 0.10, "roof": 1.0}
|
|
busy = svc.scaled_bars([1.5, 3.0], 1.0)
|
|
assert busy["bars"] == [50.0, 100.0] and busy["roof"] == 3.0
|
|
assert svc.scaled_bars([None, None], 1.0) == {"bars": [None, None], "peak": None, "roof": 1.0}
|
|
|
|
|
|
def test_bar_slots_end_at_the_bucket_now_falls_in():
|
|
now = datetime(2026, 10, 7, 12, 7, 30, tzinfo=timezone.utc)
|
|
slots, width = svc.bar_slots(now, 6)
|
|
assert width == 900 and len(slots) == 24
|
|
assert slots[-1] == datetime(2026, 10, 7, 12, 0, tzinfo=timezone.utc)
|
|
assert slots[0] == datetime(2026, 10, 7, 6, 15, tzinfo=timezone.utc)
|