bvandeusenandClaude Opus 5.5
8566c8f78e
- #5304 "Docker disk totals recorded over time per host, with a growth chart and alertable metrics" - #5305 "Disk page: images grouped by repository, versions folded under it" - #5306 "Docker overview: one disk line per host linking to the Disk page" Each `docker system df` report is now also recorded as metrics, with the host as the resource: disk_total_bytes, its parts, and disk_reclaimable_bytes. An alert rule can watch them. The Disk page starts with the hosts side by side, showing how much each grew over the selected range with a line of it, then gives each host its own section. An image's versions fold under its repository, which is charged only for the layers its versions don't share. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
333 lines
20 KiB
Python
333 lines
20 KiB
Python
"""The Docker page's tables (#5299, #5301) and a service's page (#5302): the
|
|
rows, their search, filters and sort, and the helpers that find a service's
|
|
tasks, logs and summed history."""
|
|
import re
|
|
from datetime import datetime, timedelta, timezone
|
|
from types import SimpleNamespace as NS
|
|
|
|
from quart import session
|
|
|
|
from plugins.docker import service as svc
|
|
from plugins.docker.overview import (
|
|
NO_STACK, container_row, filter_options, filter_rows, service_row, sort_rows, stack_of,
|
|
)
|
|
from plugins.docker.routes import _filters, _page_url, _sort_url, _task_rows
|
|
from tests._templates import render_plugin_template
|
|
|
|
NOW = datetime(2026, 10, 7, 12, 0, tzinfo=timezone.utc)
|
|
TASK = "x7q2m9k4p1r8s5t3v6w0y2z4a" # 25 characters, like Swarm's
|
|
|
|
|
|
def _replica(host, cpu=10.0, mem=100, started=2, status="running", **kw):
|
|
return dict(ghost=False, host=host, host_id=f"id-{host}", name=f"r.{host}", status=status,
|
|
cpu_pct=cpu, mem_bytes=mem, started_at=NOW - timedelta(hours=started), image="img:1",
|
|
health=kw.get("health"))
|
|
|
|
|
|
def _service(name, state="healthy", running=1, desired=1, replicas=None, **kw):
|
|
s = dict(name=name, mode="replicated", running=running, desired=desired, state=state,
|
|
crash_looping=False, failures_24h=0, last_error=None, trouble_since=None,
|
|
replicas=replicas if replicas is not None else [_replica("a")])
|
|
s.update(kw)
|
|
return s
|
|
|
|
|
|
def _container(name, **kw):
|
|
base = dict(name=name, host_id="h1", image="nginx:1", status="running", health=None, exit_code=None,
|
|
oom_killed=False, restart_count=0, compose_project=None, started_at=NOW - timedelta(hours=1),
|
|
scraped_at=NOW)
|
|
base.update(kw)
|
|
c = NS(**base)
|
|
usage = {"cores": 0.5, "mem_bytes": 2048}
|
|
return container_row(c, label=name, host="nas", usage=usage, ports=[], stale=False,
|
|
exit_failed=bool(kw.get("exit_code")), now=NOW)
|
|
|
|
|
|
def test_a_service_reads_its_stack_from_its_name():
|
|
assert stack_of("arr_radarr") == "arr" and stack_of("steward_app_db") == "steward"
|
|
assert stack_of("portainer") is None and stack_of("_x") is None and stack_of("x_") is None
|
|
|
|
|
|
def test_a_service_row_adds_up_its_running_replicas():
|
|
s = _service("arr_radarr", running=2, desired=3, state="degraded", replicas=[
|
|
_replica("b", cpu=50.0, mem=300, started=5), _replica("a", cpu=25.0, mem=100, started=1),
|
|
_replica("c", cpu=99.0, mem=999, status="exited"), {"ghost": True, "host": "edge", "count": 1}])
|
|
row = service_row(s, NOW)
|
|
assert row["stack"] == "arr" and row["trouble"] == 2
|
|
assert row["cpu_cores"] == 0.75 and row["mem_bytes"] == 400 # the exited one is not counted
|
|
assert row["started_at"] == NOW - timedelta(hours=1) and row["up_s"] == 3600
|
|
assert row["nodes"] == ["a", "b", "c"] and row["ghost_nodes"] == ["edge"]
|
|
assert row["image"] == "img:1"
|
|
empty = service_row(_service("solo", state="down", running=0, replicas=[]), NOW)
|
|
assert empty["cpu_cores"] is None and empty["mem_bytes"] is None and empty["up_s"] is None
|
|
|
|
|
|
def test_rows_filter_by_search_stack_host_and_status():
|
|
rows = [service_row(_service("arr_radarr"), NOW),
|
|
service_row(_service("arr_sonarr", state="down", running=0, replicas=[]), NOW),
|
|
service_row(_service("portainer", replicas=[_replica("b")]), NOW),
|
|
_container("traefik", compose_project="edge"),
|
|
_container("backup", status="exited", exit_code=0)]
|
|
|
|
def names(**kw):
|
|
return [r["name"] for r in filter_rows(rows, **kw)]
|
|
|
|
assert names(q="SONARR") == ["arr_sonarr"] and names(q="nginx") == ["traefik", "backup"]
|
|
assert names(stack="arr") == ["arr_radarr", "arr_sonarr"] and names(q="edge") == ["traefik"]
|
|
assert names(stack=NO_STACK) == ["portainer", "backup"]
|
|
assert names(host="b") == ["portainer"] and names(host="nas") == ["traefik", "backup"]
|
|
assert names(state="problem") == ["arr_sonarr"]
|
|
assert names(state="stopped") == ["arr_sonarr", "backup"]
|
|
assert names(state="healthy") == ["arr_radarr", "portainer", "traefik"]
|
|
assert filter_options(rows[:3], rows[3:]) == {"stacks": ["arr", "edge"], "hosts": ["a", "b", "nas"]}
|
|
|
|
|
|
def test_rows_sort_by_column_with_missing_values_last():
|
|
busy = service_row(_service("busy", replicas=[_replica("a", cpu=300.0)]), NOW)
|
|
idle = service_row(_service("idle", replicas=[_replica("a", cpu=5.0)]), NOW)
|
|
down = service_row(_service("down", state="down", running=0, replicas=[]), NOW)
|
|
rows = [idle, down, busy]
|
|
assert [r["name"] for r in sort_rows(rows, "cpu", True)] == ["busy", "idle", "down"]
|
|
assert [r["name"] for r in sort_rows(rows, "cpu", False)] == ["idle", "busy", "down"]
|
|
assert [r["name"] for r in sort_rows(rows, "status", False)] == ["down", "busy", "idle"]
|
|
assert [r["name"] for r in sort_rows(rows, "name", True)] == ["idle", "down", "busy"]
|
|
|
|
|
|
def test_the_filters_live_in_the_url_without_their_defaults():
|
|
f = _filters({"q": " web ", "stack": "arr", "state": "bogus", "sort": "nope", "dir": "x"})
|
|
assert f == {"q": "web", "stack": "arr", "host": "", "state": "", "sort": "status", "dir": "asc"}
|
|
assert _page_url(f) == "/plugins/docker/?q=web&stack=arr"
|
|
assert _page_url(f, stack="") == "/plugins/docker/?q=web"
|
|
plain = _filters({})
|
|
assert _sort_url(plain, "cpu") == "/plugins/docker/?sort=cpu" # busiest first by default
|
|
assert _sort_url(plain, "name") == "/plugins/docker/?sort=name"
|
|
cpu = _filters({"sort": "cpu"})
|
|
assert cpu["dir"] == "desc" and _sort_url(cpu, "cpu") == "/plugins/docker/?sort=cpu&dir=asc"
|
|
|
|
|
|
def test_a_task_container_name_gives_its_slot_and_task():
|
|
assert svc.task_of("web", f"web.1.{TASK}") == ("1", TASK)
|
|
assert svc.task_of("web", f"web.q8n2.{TASK}") == ("q8n2", TASK) # a global service's node slot
|
|
assert svc.task_of("web", f"web_db.1.{TASK}") is None
|
|
assert svc.task_of("arr", f"arr_radarr.1.{TASK}") is None
|
|
assert svc.task_of("web", "web.1.short") is None
|
|
|
|
|
|
def test_service_history_sums_replicas_after_averaging_each():
|
|
since = NOW - timedelta(hours=1)
|
|
t = since + timedelta(minutes=10)
|
|
samples = [("a", t, 100.0, 1000), ("a", t + timedelta(seconds=20), 300.0, 3000),
|
|
("b", t + timedelta(seconds=5), 50.0, 500),
|
|
("a", t + timedelta(minutes=5), 10.0, 100)]
|
|
out = svc.sum_series(samples, since, NOW, points=60)
|
|
assert out == [(t, 250.0, 2500), (t + timedelta(minutes=5), 10.0, 100)]
|
|
|
|
|
|
def test_a_failure_carries_the_task_whose_logs_tell_why():
|
|
sick = {"name": "codex_db", "tasks_known": True, "failures_24h": 2,
|
|
"last_error": {"state": "failed", "error": "boom", "task_id": TASK, "node": "nas", "at": None}}
|
|
(row,) = svc.failures_24h([], [sick], {})
|
|
assert row["kind"] == "service" and row["task_id"] == TASK
|
|
old = {"name": "legacy", "tasks_known": False, "failures_24h": None, "last_error": None}
|
|
died = NS(container_name=f"legacy.1.{TASK}", host_id="h1", at=NOW, event="die", detail="exit 1")
|
|
(row,) = svc.failures_24h([died], [old], {"h1": "nas"})
|
|
assert row["kind"] == "service" and row["task_id"] == TASK
|
|
|
|
|
|
def test_task_rows_say_where_each_ran_and_why_it_ended():
|
|
rows = _task_rows([
|
|
{"task_id": TASK, "slot": 2, "node_id": "n1", "state": "failed", "desired_state": "shutdown",
|
|
"error": "task: non-zero exit (1)", "exit_code": 1, "at": "2026-10-07T11:00:00Z"},
|
|
{"task_id": "y" * 25, "slot": 2, "node_id": "n9" * 10, "state": "running", "desired_state": "running"}],
|
|
{"n1": "docker02"})
|
|
assert rows[0]["node"] == "docker02" and rows[0]["failed"] and rows[0]["why"] == "task: non-zero exit (1)"
|
|
assert rows[1]["node"] == ("n9" * 10)[:12] and rows[1]["current"] and not rows[1]["failed"]
|
|
|
|
|
|
async def test_the_service_page_puts_state_first_and_offers_each_tasks_logs(app):
|
|
now = datetime.now(timezone.utc)
|
|
s = _service("arr_radarr", running=1, desired=2, state="degraded", failures_24h=3, replicas=[
|
|
dict(_replica("docker02"), name=f"arr_radarr.1.{TASK}"), {"ghost": True, "host": "edge01", "count": 1}],
|
|
last_error={"state": "failed", "error": "task: non-zero exit (1)", "exit_code": 1, "node": "docker02",
|
|
"age_s": 300, "task_id": TASK, "slot": 2})
|
|
replicas = [dict(s["replicas"][0], usage={"cores": 0.1, "cpus": 4, "cpu_share": 2.5, "mem_bytes": 100,
|
|
"mem_pct": None, "mem_of": None, "mem_limit": None},
|
|
uptime="2h"), s["replicas"][1]]
|
|
tasks = _task_rows([{"task_id": TASK, "slot": 2, "node_id": "n1", "state": "failed",
|
|
"desired_state": "shutdown", "error": "task: non-zero exit (1)", "exit_code": 1}],
|
|
{"n1": "docker02"})
|
|
svc_row = NS(image="lscr.io/linuxserver/radarr:5@sha256:abc", updated_at=now)
|
|
async with app.test_request_context(f"/plugins/docker/service/arr_radarr?task={TASK}"):
|
|
session["user_role"] = "viewer"
|
|
page = await render_plugin_template(
|
|
app, "docker", "docker/service.html", name="arr_radarr", service=s, row=service_row(s, now),
|
|
svc=svc_row, tasks=tasks, replicas=replicas, stack="arr", current_range="24h",
|
|
task_choices=[(TASK, f"2 · {TASK[:10]}"), ("y" * 25, "1 · yyyyyyyyyy")], focus_task=TASK)
|
|
page = re.sub(r">\s+<", "><", " ".join(page.split()))
|
|
assert '<nav class="breadcrumb"' in page and 'href="/plugins/docker/">Docker</a>' in page
|
|
assert page.index("Degraded") < page.index("Last failed task") < page.index('id="svc-replicas"')
|
|
assert "1 of 2 running" in page and '<a class="chip" href="/plugins/docker/?stack=arr">arr</a>' in page
|
|
assert f'href="?task={TASK}#logs">its logs →</a>' in page and "5m ago" in page
|
|
assert "radarr:5@sha256:abc" in page
|
|
assert "1 running on a node with no Steward agent" in page and '<a href="/hosts/id-docker02">docker02</a>' in page
|
|
assert page.index('id="svc-replicas"') < page.index('id="svc-history"') < page.index('id="tasks"') < page.index('id="logs"')
|
|
assert 'hx-get="/plugins/docker/service/arr_radarr/history"' in page and 'id="time-range"' in page
|
|
assert '<span class="badge badge-crit">Failed</span>' in page
|
|
assert f'<option value="{TASK}" selected>2 · {TASK[:10]}</option>' in page
|
|
assert 'hx-get="/plugins/docker/service/arr_radarr/logs/lines"' in page
|
|
|
|
|
|
async def test_a_service_nobody_reports_is_not_found(app):
|
|
async with app.test_request_context("/plugins/docker/service/gone"):
|
|
session["user_role"] = "viewer"
|
|
page = await render_plugin_template(app, "docker", "docker/service.html", name="gone", service=None)
|
|
assert "Service not found" in page and 'id="logs"' not in page
|
|
|
|
|
|
async def test_a_service_history_shows_now_and_peak(app):
|
|
async with app.test_request_context("/plugins/docker/service/web/history"):
|
|
page = await render_plugin_template(
|
|
app, "docker", "docker/_service_history.html", have_data=True, range_key="24h",
|
|
cpu_now=0.5, cpu_peak=2.0, mem_now=100 << 20, mem_peak=300 << 20,
|
|
spark_cpu="<svg></svg>", spark_mem="<svg></svg>")
|
|
empty = await render_plugin_template(
|
|
app, "docker", "docker/_service_history.html", have_data=False, range_key="1h")
|
|
assert "0.50 cores" in page and "peak 2.0 cores" in page and "peak 300.0 MiB" in page
|
|
assert "Not enough samples in the last 1h" in empty
|
|
|
|
|
|
async def test_a_service_log_line_names_its_task(app):
|
|
lines = [{"ts": NOW, "stream": "stderr", "line": "<b>boom</b>", "task": f"1 · {TASK[:10]}"}]
|
|
async with app.test_request_context("/plugins/docker/service/web/logs/lines"):
|
|
page = await render_plugin_template(app, "docker", "docker/_container_logs_lines.html",
|
|
lines=lines, name="web", filtered=False, service=True)
|
|
empty = await render_plugin_template(app, "docker", "docker/_container_logs_lines.html",
|
|
lines=[], name="web", filtered=False, service=True)
|
|
assert f"1 · {TASK[:10]}</span>" in page and "<b>boom</b>" in page
|
|
assert "one of its tasks" in empty
|
|
|
|
|
|
def _node(nid, hostname, role="worker", status="ready", availability="active", leader=False):
|
|
return NS(node_id=nid, hostname=hostname, role=role, status=status, availability=availability,
|
|
leader=leader)
|
|
|
|
|
|
def _swarm_svc(name, placement):
|
|
import json
|
|
return NS(service_name=name, placement_json=json.dumps(placement))
|
|
|
|
|
|
def _task(name, service, node, cpu, mem, status="running"):
|
|
return NS(name=name, service_name=service, node_id=node, status=status, cpu_pct=cpu,
|
|
mem_usage_bytes=mem)
|
|
|
|
|
|
def test_each_node_lists_what_runs_on_it_and_what_that_uses():
|
|
"""#5303: a node with an agent has figures; one without is Swarm's placement only."""
|
|
from plugins.docker.swarm_nodes import node_cards
|
|
nodes = [_node("n1", "docker02", role="manager", leader=True), _node("n2", "edge01"),
|
|
_node("n3", "docker03", availability="drain")]
|
|
services = [_swarm_svc("arr_radarr", [{"node_id": "n1", "running": 1}]),
|
|
_swarm_svc("web", [{"node_id": "n1", "running": 2}, {"node_id": "n2", "running": 1}]),
|
|
_swarm_svc("db", [{"node_id": "n1", "running": 1}])]
|
|
containers = [_task("arr_radarr.1.x", "arr_radarr", "n1", 150.0, 300),
|
|
_task("web.1.x", "web", "n1", 10.0, 100), _task("web.2.x", "web", "n1", 20.0, 100),
|
|
_task("web.0.old", "web", "n1", 99.0, 999, status="exited"),
|
|
_task("traefik", None, None, 5.0, 5)]
|
|
cards = node_cards(nodes, services, containers, node_host={"n1": "h1"}, host_names={"h1": "docker02"},
|
|
capacity={"h1": (8, 16 << 30)},
|
|
service_state={"arr_radarr": "healthy", "web": "healthy", "db": "down"})
|
|
assert [c["hostname"] for c in cards] == ["docker03", "docker02", "edge01"] # drained first, then managers
|
|
n1 = cards[1]
|
|
assert n1["host_id"] == "h1" and n1["leader"] and n1["agent"]
|
|
assert [t["name"] for t in n1["tasks"]] == ["db", "arr_radarr", "web"] # trouble, then busiest
|
|
assert n1["tasks"][0]["cores"] is None and not n1["tasks"][0]["measured"] # placed, not reported
|
|
web = n1["tasks"][2]
|
|
assert web["tasks"] == 2 and web["cores"] == 0.3 and web["mem_bytes"] == 200
|
|
assert n1["cores_used"] == 1.8 and n1["mem_used"] == 500 and n1["cores"] == 8
|
|
edge = cards[2]
|
|
assert not edge["agent"] and edge["cores_used"] is None
|
|
assert edge["tasks"] == [{"name": "web", "tasks": 1, "cpu_pct": None, "mem_bytes": None, "measured": False,
|
|
"state": "healthy", "cores": None}]
|
|
assert not cards[0]["ok"] and cards[0]["tasks"] == []
|
|
|
|
|
|
async def test_the_swarm_page_is_a_card_per_node(app):
|
|
from plugins.docker.swarm_nodes import node_cards
|
|
nodes = [_node("n1", "docker02", role="manager", leader=True), _node("n2", "edge01", status="down")]
|
|
cards = node_cards(nodes, [_swarm_svc("web", [{"node_id": "n2", "running": 1}])],
|
|
[_task("arr.1.x", "arr", "n1", 50.0, 1 << 30)], node_host={"n1": "h1"},
|
|
host_names={"h1": "docker02"}, capacity={"h1": (4, 8 << 30)},
|
|
service_state={"arr": "healthy", "web": "degraded"})
|
|
troubled = [{"name": "web", "running": 1, "desired": 2, "state": "degraded", "crash_looping": False,
|
|
"trouble_since": None, "last_error": {"state": "failed", "error": "no suitable node", "node": None}}]
|
|
async with app.test_request_context("/plugins/docker/swarm"):
|
|
session["user_role"] = "viewer"
|
|
page = await render_plugin_template(app, "docker", "docker/swarm.html", swarms=[{
|
|
"managers": ["docker02"], "services_total": 2, "services_healthy": 1, "troubled": troubled,
|
|
"nodes": cards}])
|
|
page = re.sub(r">\s+<", "><", " ".join(page.split()))
|
|
assert "2 need attention" in page and '<span class="tone-warn">1/2</span> services healthy' in page
|
|
problems = page[page.index('class="problem-list"'):]
|
|
assert "Node down" in problems and "edge01" in problems
|
|
assert '<a href="/plugins/docker/service/web">web</a>' in problems and "last task failed: no suitable node" in problems
|
|
assert page.index('id="node-n2"') < page.index('id="node-n1"') # the down node first
|
|
assert '<a href="/hosts/h1">docker02</a><span class="chip">leader</span>' in page
|
|
assert "Containers use <strong>0.50 cores</strong> of 4 · <strong>1.0 GiB</strong> of 8.0 GiB" in page
|
|
assert "No Steward agent" in page
|
|
assert '<a href="/plugins/docker/service/arr" title="arr">arr</a>' in page
|
|
|
|
|
|
def test_an_image_reference_splits_into_repository_and_tag():
|
|
from plugins.docker.disk_view import image_repo
|
|
assert image_repo("nginx:1.27") == ("nginx", "1.27")
|
|
assert image_repo("ghcr.io/x/app:2026.10.07") == ("ghcr.io/x/app", "2026.10.07")
|
|
assert image_repo("reg.local:5000/app") == ("reg.local:5000/app", None) # a port is not a tag
|
|
assert image_repo("reg.local:5000/app:3") == ("reg.local:5000/app", "3")
|
|
assert image_repo("app@sha256:" + "f" * 64) == ("app", "sha256:ffffffffffff")
|
|
assert image_repo("<none>") == (None, None) and image_repo("<none>:<none>") == (None, None)
|
|
assert image_repo(None) == (None, None)
|
|
|
|
|
|
def test_images_fold_under_their_repository_by_the_space_they_hold():
|
|
from plugins.docker.disk_view import group_images
|
|
repos = group_images([
|
|
{"repo_tag": "app:1", "image_id": "a", "size": 500, "shared_size": 100, "containers": 1},
|
|
{"repo_tag": "app:0", "image_id": "b", "size": 450, "shared_size": 100, "containers": 0},
|
|
{"repo_tag": "<none>", "image_id": "c", "size": 9000, "shared_size": 0, "containers": 0},
|
|
{"repo_tag": "db:16", "image_id": "d", "size": 300, "shared_size": 0, "containers": 1}])
|
|
assert [r["repo"] for r in repos] == ["app", "db", None] # untagged last
|
|
app_ = repos[0]
|
|
assert app_["own"] == 750 and app_["reclaimable"] == 350 and app_["unused"] == 1
|
|
assert [v["tag"] for v in app_["versions"]] == ["1", "0"] and app_["containers"] == 1
|
|
|
|
|
|
def test_disk_metrics_and_growth():
|
|
from plugins.docker.disk_view import disk_metrics, growth
|
|
got = disk_metrics({"images_size": 100, "containers_size": 10, "volumes_size": 5,
|
|
"build_cache_size": 1, "images_reclaimable": 40, "layers_size": 999})
|
|
assert got["disk_total_bytes"] == 116.0 and got["disk_reclaimable_bytes"] == 40.0
|
|
assert growth([(NOW, 1.0)]) is None
|
|
g = growth([(NOW - timedelta(days=7), 100.0), (NOW - timedelta(days=1), 90.0), (NOW, 125.0)])
|
|
assert g["change"] == 25.0 and g["pct"] == 25.0 and g["since"] == NOW - timedelta(days=7)
|
|
assert growth([(NOW, 0.0), (NOW, 5.0)])["pct"] is None
|
|
|
|
|
|
async def test_the_overview_has_one_disk_line_per_host(app):
|
|
gib = 1 << 30
|
|
disk = [{"host_id": "h1", "host_name": "docker02", "total": 42 * gib, "reclaimable": 5 * gib,
|
|
"growth": {"change": 2 * gib, "pct": 5.0, "since": NOW - timedelta(days=7)}},
|
|
{"host_id": "h2", "host_name": "util01", "total": gib, "reclaimable": 0, "growth": None}]
|
|
async with app.test_request_context("/plugins/docker/"):
|
|
session["user_role"] = "viewer"
|
|
page = await render_plugin_template(
|
|
app, "docker", "docker/index.html", poll_interval=10, has_swarm=False, has_disk=True, disk=disk,
|
|
f={"q": "", "stack": "", "host": "", "state": "", "sort": "status", "dir": "asc"},
|
|
choices={"stacks": [], "hosts": []}, states={}, no_stack="__none__")
|
|
page = re.sub(r">\s+<", "><", " ".join(page.split()))
|
|
line = page[page.index('class="docker-disk-line"'):]
|
|
line = line[:line.index("</p>")]
|
|
assert '<a href="/plugins/docker/disk#host-h1">docker02</a> 42.0 GiB' in line and "+2.0 GiB in 7d" in line
|
|
assert "5.0 GiB reclaimable" in line and '<a href="/plugins/docker/disk#host-h2">util01</a> 1.0 GiB' in line
|
|
assert page.index('id="docker-rows"') < page.index('class="docker-disk-line"')
|