feat(docker): the Docker page leads with a verdict, lists services and containers as filterable tables, and each service has its own page with logs across its tasks (#5298–#5302)
- #5298 "Docker overview: verdict line; only the problem tiles that aren't zero" - #5299 "Docker overview: swarm services as a filterable, sortable table with a Stack column" - #5300 "Docker overview: failures listed once, each with a link to the failed task's logs" - #5301 "Docker overview: standalone containers in one table with a Host column" - #5302 "Service page: replicas, image, task history, resource history and logs across all its tasks" overview.py holds the rows, filters and sort as plain functions. The filter bar sits outside the polled fragment and the URL keeps its state. A service's page reads every task container Steward still holds anything for, the live ones and removed ones known by their events, so a failed task's logs are reachable after Docker has removed it. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Claude Opus 5.5
@@ -0,0 +1,141 @@
|
||||
"""Integration: a swarm service's own page (#5302) and the Docker overview's
|
||||
service table (#5299, #5300) against real Postgres. The page reads the logs
|
||||
and samples of every task the service ran, the live one and one Docker has
|
||||
already removed (known only by its events), and a failure's "Logs →" picks
|
||||
that task. Requires STEWARD_DATABASE_URL."""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import importlib
|
||||
import json
|
||||
import os
|
||||
import time
|
||||
import uuid
|
||||
from datetime import datetime, timedelta, timezone
|
||||
|
||||
import pytest
|
||||
|
||||
pytestmark = pytest.mark.integration
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def app():
|
||||
if not os.environ.get("STEWARD_DATABASE_URL"):
|
||||
pytest.skip("needs Postgres")
|
||||
from steward.app import create_app
|
||||
return create_app(testing=False)
|
||||
|
||||
|
||||
def _docker(sub):
|
||||
"""The app's loaded docker plugin module (see test_docker_recent_usage)."""
|
||||
from steward.core.plugin_manager import get_loaded_plugin_modules
|
||||
loaded = get_loaded_plugin_modules().get("docker")
|
||||
base = loaded.__name__ if loaded is not None else "plugins.docker"
|
||||
return importlib.import_module(f"{base}.{sub}")
|
||||
|
||||
|
||||
def test_a_service_page_reads_every_task(app):
|
||||
from steward.models.hosts import Host
|
||||
from steward.models.users import User, UserRole
|
||||
m = _docker("models")
|
||||
|
||||
tag = uuid.uuid4().hex[:8]
|
||||
name = f"it{tag}_web"
|
||||
live_task, dead_task = "a" * 17 + tag, "b" * 17 + tag # 25 characters, as Swarm's
|
||||
live, dead = f"{name}.1.{live_task}", f"{name}.1.{dead_task}"
|
||||
hid, uid, node = str(uuid.uuid4()), str(uuid.uuid4()), f"n{tag}"
|
||||
now = datetime.now(timezone.utc)
|
||||
|
||||
def iso(dt):
|
||||
return dt.strftime("%Y-%m-%dT%H:%M:%S.%fZ")
|
||||
|
||||
tasks = [{"task_id": live_task, "slot": 1, "node_id": node, "desired_state": "running",
|
||||
"state": "running", "message": "started", "error": "", "exit_code": None,
|
||||
"at": iso(now - timedelta(minutes=9))},
|
||||
{"task_id": dead_task, "slot": 1, "node_id": node, "desired_state": "shutdown",
|
||||
"state": "failed", "message": "started", "error": "task: non-zero exit (1)",
|
||||
"exit_code": 1, "at": iso(now - timedelta(minutes=10))}]
|
||||
|
||||
def sample(cname, mins, cpu, mem):
|
||||
return m.DockerMetric(host_id=hid, container_name=cname, scraped_at=now - timedelta(minutes=mins),
|
||||
cpu_pct=cpu, mem_pct=0.0, mem_usage_bytes=mem)
|
||||
|
||||
async def go():
|
||||
async with app.db_sessionmaker() as s:
|
||||
async with s.begin():
|
||||
s.add(User(id=uid, username=f"sp-{tag}", email=f"sp-{tag}@test",
|
||||
password_hash="x", role=UserRole.viewer, is_active=True))
|
||||
s.add(Host(id=hid, name=f"dock-{tag}", address="10.9.7.1"))
|
||||
async with s.begin():
|
||||
s.add_all([
|
||||
m.DockerSwarmNode(host_id=hid, node_id=node, hostname=f"node-{tag}", role="manager",
|
||||
availability="active", status="ready", leader=True),
|
||||
m.DockerSwarmService(host_id=hid, service_name=name, mode="replicated", desired=1,
|
||||
running=1, image="nginx:1.27",
|
||||
placement_json=json.dumps([{"node_id": node, "running": 1}]),
|
||||
tasks_json=json.dumps(tasks), updated_at=now),
|
||||
m.DockerContainer(host_id=hid, name=live, image="nginx:1.27", status="running",
|
||||
cpu_pct=50.0, mem_usage_bytes=100 << 20, service_name=name,
|
||||
task_id=live_task, node_id=node, started_at=now - timedelta(minutes=9),
|
||||
scraped_at=now),
|
||||
# Docker has removed the failed task's container: only its events are left.
|
||||
m.DockerEvent(host_id=hid, container_name=dead, event="start", at=now - timedelta(minutes=11)),
|
||||
m.DockerEvent(host_id=hid, container_name=dead, event="die", detail="exit 1",
|
||||
at=now - timedelta(minutes=10)),
|
||||
m.DockerLog(host_id=hid, container_name=live, stream="stdout", line=f"serving {tag}",
|
||||
ts=now - timedelta(minutes=5)),
|
||||
m.DockerLog(host_id=hid, container_name=dead, stream="stderr", line=f"panic {tag}",
|
||||
ts=now - timedelta(minutes=10)),
|
||||
sample(dead, 40, 200.0, 300 << 20), sample(live, 30, 40.0, 90 << 20),
|
||||
sample(live, 20, 60.0, 100 << 20), sample(live, 1, 50.0, 100 << 20),
|
||||
])
|
||||
|
||||
client = app.test_client()
|
||||
async with client.session_transaction() as sess:
|
||||
sess.update(user_id=uid, user_role="viewer", username=f"sp-{tag}", login_at=time.time())
|
||||
|
||||
async def get(url):
|
||||
resp = await client.get(url)
|
||||
return resp.status_code, await resp.get_data(as_text=True)
|
||||
|
||||
out = {"page": await get(f"/plugins/docker/service/{name}"),
|
||||
"focus": await get(f"/plugins/docker/service/{name}?task={dead_task}"),
|
||||
"missing": await get(f"/plugins/docker/service/it{tag}_nothing"),
|
||||
"logs": await get(f"/plugins/docker/service/{name}/logs/lines"),
|
||||
"one": await get(f"/plugins/docker/service/{name}/logs/lines?task={dead_task}"),
|
||||
"history": await get(f"/plugins/docker/service/{name}/history?range=1h"),
|
||||
"rows": await get(f"/plugins/docker/rows?q=it{tag}"),
|
||||
"other": await get(f"/plugins/docker/rows?q=it{tag}&stack=someotherstack")}
|
||||
async with app.db_sessionmaker() as s:
|
||||
async with s.begin():
|
||||
await s.delete(await s.get(Host, hid))
|
||||
await s.delete(await s.get(User, uid))
|
||||
return out
|
||||
|
||||
got = asyncio.run(go())
|
||||
|
||||
status, page = got["page"]
|
||||
assert status == 200
|
||||
assert "nginx:1.27" in page and f"node-{tag}" in page and "1 of 1 running" in page
|
||||
assert "task: non-zero exit (1)" in page # Swarm's task record
|
||||
assert f'<option value="{live_task}">1 · {live_task[:10]}</option>' in page
|
||||
assert f'<option value="{dead_task}">1 · {dead_task[:10]}</option>' in page
|
||||
assert f'href="/plugins/docker/?stack=it{tag}"' in page
|
||||
assert f'<option value="{dead_task}" selected>' in got["focus"][1]
|
||||
assert got["missing"][0] == 404 and "Service not found" in got["missing"][1]
|
||||
|
||||
logs = got["logs"][1]
|
||||
assert f"serving {tag}" in logs and f"panic {tag}" in logs # the removed task's lines too
|
||||
assert logs.index(f"serving {tag}") < logs.index(f"panic {tag}") # newest first
|
||||
assert f"1 · {dead_task[:10]}" in logs
|
||||
one = got["one"][1]
|
||||
assert f"panic {tag}" in one and f"serving {tag}" not in one
|
||||
|
||||
history = got["history"][1]
|
||||
assert "<polyline" in history and "peak 2.0 cores" in history and "now · peak" in history
|
||||
|
||||
rows = got["rows"][1]
|
||||
assert f'href="/plugins/docker/service/{name}"' in rows
|
||||
assert f'<a class="chip" href="/plugins/docker/?stack=it{tag}"' in rows
|
||||
assert f'/plugins/docker/service/{name}?task={dead_task}#logs' in rows # the failure's logs
|
||||
assert f'href="/plugins/docker/service/{name}"' not in got["other"][1]
|
||||
@@ -6,6 +6,7 @@ from types import SimpleNamespace as NS
|
||||
|
||||
|
||||
from plugins.docker import service as svc
|
||||
from plugins.docker.overview import DOCKER_STATES, NO_STACK, container_row, service_row
|
||||
from plugins.docker.routes import _is_stale
|
||||
from tests._templates import render_plugin_template
|
||||
from tests.test_style_tokens import hex_colours
|
||||
@@ -40,28 +41,46 @@ def test_stale_is_older_than_the_agents_window():
|
||||
assert svc.stale_after_seconds({}) == 180
|
||||
|
||||
|
||||
_F = {"q": "", "stack": "", "host": "", "state": "", "sort": "status", "dir": "asc"}
|
||||
|
||||
|
||||
async def _rows(app, **over):
|
||||
"""docker/rows.html with what routes.rows() hands it (#5298–#5301),
|
||||
nothing set unless given."""
|
||||
ctx = dict(services=[], containers=[], services_total=0, containers_total=0, services_healthy=0,
|
||||
failing_services=[], failures=[], docker_hosts=0, docker_down=[], attention=[],
|
||||
says=svc.PROBLEM_SAYS, severity=svc.PROBLEM_SEVERITY, unhealthy=0, restarting=0,
|
||||
stale=0, running=0, stopped=0, f=dict(_F), filtered=False,
|
||||
sort_url=lambda key: f"/plugins/docker/?sort={key}",
|
||||
page_url=lambda **kw: "/plugins/docker/?" + "&".join(f"{k}={v}" for k, v in kw.items()))
|
||||
ctx.update(over)
|
||||
async with app.test_request_context("/plugins/docker/rows"):
|
||||
page = await render_plugin_template(app, "docker", "docker/rows.html", **ctx)
|
||||
return re.sub(r">\s+<", "><", " ".join(page.split()))
|
||||
|
||||
|
||||
def _crow(c, *, host="nas", stale=False, exit_failed=False, ports=()):
|
||||
return container_row(c, label=svc.display_name(c), host=host, usage=svc.usage(c, 1, None),
|
||||
ports=list(ports), stale=stale, exit_failed=exit_failed, now=NOW)
|
||||
|
||||
|
||||
async def test_a_crash_loop_comes_first_and_a_stale_row_says_so(app):
|
||||
loop = _c("api", status="restarting", exit_code=1, restart_count=7)
|
||||
old = _c("cache", scraped_at=NOW - timedelta(minutes=30), cpu_pct=95.0)
|
||||
attention = [{"container": loop, "label": "api", "problems": ["restarting", "crash_looping"], "replaced": False,
|
||||
"since": datetime.now(timezone.utc) - timedelta(minutes=8), "stale": False}]
|
||||
groups = [{"host": None, "host_id": "h1", "host_name": "nas", "running": 1, "stopped": 1, "grouped": False,
|
||||
"subgroups": [{"label": None, "containers": [
|
||||
{"container": loop, "ports": [], "uptime": None, "stale": False,
|
||||
"usage": svc.usage(loop, 1, None), "exit_failed": True},
|
||||
{"container": old, "ports": [], "uptime": "2h", "stale": True,
|
||||
"usage": svc.usage(old, 1, None), "exit_failed": False}]}]}]
|
||||
async with app.test_request_context("/plugins/docker/rows"):
|
||||
page = await render_plugin_template(
|
||||
app, "docker", "docker/rows.html", attention=attention, says=svc.PROBLEM_SAYS, severity=svc.PROBLEM_SEVERITY,
|
||||
unhealthy=0, restarting=1, stale=1, host_groups=groups, swarm_services=[],
|
||||
running=1, stopped=1, docker_hosts=1, failures=[], failing_services=[])
|
||||
page = re.sub(r">\s+<", "><", " ".join(page.split()))
|
||||
page = await _rows(app, attention=attention, restarting=1, stale=1, running=1, stopped=1, docker_hosts=1,
|
||||
containers=[_crow(loop, exit_failed=True), _crow(old, stale=True)], containers_total=2)
|
||||
|
||||
# #5298: the verdict first; only the tiles that aren't zero.
|
||||
assert page.index('class="docker-verdict"') < page.index("Needs attention")
|
||||
assert "1 needs attention" in page and "1 container running, 1 stopped" in page
|
||||
assert "Unhealthy</span>" not in page and "Failed (24h)" not in page
|
||||
card = page[page.index("Needs attention"):page.index("</section>")]
|
||||
assert '<span class="badge badge-crit">Restarting</span>' in card
|
||||
assert 'href="/plugins/docker/container/h1/api" title="api">api</a>' in card
|
||||
assert "is restarting and keeps dying · exit 1 · since <time" in card and ">8m ago</time>" in card
|
||||
assert 'href="/plugins/docker/container/h1/api/logs"' in card
|
||||
assert "cache" not in card
|
||||
assert page.index("Needs attention") < page.index("<table")
|
||||
|
||||
@@ -71,17 +90,14 @@ async def test_a_crash_loop_comes_first_and_a_stale_row_says_so(app):
|
||||
# CPU in cores, toned by its share of the host's CPUs through Settings → Thresholds.
|
||||
assert ('<span class="tone-crit" title="Docker counts CPU per core: 95% of one core, '
|
||||
'95.0% of this host\'s 1 core">0.95 cores</span>') in page
|
||||
assert "history" not in page.lower() and "<svg" not in page # no sparklines on the overview
|
||||
assert "<svg" not in page # no sparklines on the overview
|
||||
assert not hex_colours(page), hex_colours(page)
|
||||
|
||||
|
||||
async def test_a_healthy_board_has_no_attention_card(app):
|
||||
async with app.test_request_context("/plugins/docker/rows"):
|
||||
page = await render_plugin_template(
|
||||
app, "docker", "docker/rows.html", attention=[], says=svc.PROBLEM_SAYS, severity=svc.PROBLEM_SEVERITY,
|
||||
unhealthy=0, restarting=0, stale=0, host_groups=[], swarm_services=[], running=0, stopped=0,
|
||||
docker_hosts=0, failures=[], failing_services=[])
|
||||
page = await _rows(app)
|
||||
assert "Needs attention</h2>" not in page and "No containers reported yet" in page
|
||||
assert "Nothing needs attention" in page and "stat-strip" not in page
|
||||
|
||||
|
||||
async def test_a_host_whose_docker_stopped_answering_says_so(app):
|
||||
@@ -90,12 +106,7 @@ async def test_a_host_whose_docker_stopped_answering_says_so(app):
|
||||
"detail": "no Docker socket at /var/run/docker.sock", "since": now - timedelta(minutes=5)},
|
||||
{"host_id": "h2", "host_name": "web-1", "status": "error",
|
||||
"detail": "permission denied on /var/run/docker.sock", "since": now - timedelta(minutes=1)}]
|
||||
async with app.test_request_context("/plugins/docker/rows"):
|
||||
page = await render_plugin_template(
|
||||
app, "docker", "docker/rows.html", docker_down=down, attention=[], says=svc.PROBLEM_SAYS,
|
||||
severity=svc.PROBLEM_SEVERITY, unhealthy=0, restarting=0, stale=0, host_groups=[],
|
||||
swarm_services=[], running=0, stopped=0, docker_hosts=0, failures=[], failing_services=[])
|
||||
page = re.sub(r">\s+<", "><", " ".join(page.split()))
|
||||
page = await _rows(app, docker_down=down)
|
||||
card = page[page.index("Docker not answering"):page.index("</section>")]
|
||||
assert '<span class="badge badge-warn">No Docker</span><a href="/hosts/h1">nas</a>' in card
|
||||
assert "no Docker socket at /var/run/docker.sock · since <time" in card and ">5m " in card
|
||||
@@ -103,42 +114,96 @@ async def test_a_host_whose_docker_stopped_answering_says_so(app):
|
||||
|
||||
|
||||
async def test_no_card_while_every_docker_answers(app):
|
||||
async with app.test_request_context("/plugins/docker/rows"):
|
||||
page = await render_plugin_template(
|
||||
app, "docker", "docker/rows.html", docker_down=[], attention=[], says=svc.PROBLEM_SAYS,
|
||||
severity=svc.PROBLEM_SEVERITY, unhealthy=0, restarting=0, stale=0, host_groups=[],
|
||||
swarm_services=[], running=0, stopped=0, docker_hosts=0, failures=[], failing_services=[])
|
||||
assert "Docker not answering" not in page
|
||||
assert "Docker not answering" not in await _rows(app)
|
||||
|
||||
|
||||
async def test_a_failing_service_leads_and_the_failures_are_listed(app):
|
||||
now = datetime.now(timezone.utc)
|
||||
sick = {"name": "codex_db", "mode": "replicated", "running": 0, "desired": 1, "state": "down",
|
||||
def _sick(now):
|
||||
return {"name": "codex_db", "mode": "replicated", "running": 0, "desired": 1, "state": "down",
|
||||
"crash_looping": True, "failures_1h": 7, "failures_24h": 9, "tasks_known": True,
|
||||
"trouble_since": now - timedelta(minutes=20), "replicas": [], "previous_tasks": 7,
|
||||
"last_error": {"state": "failed", "error": "task: non-zero exit (1)", "exit_code": 1,
|
||||
"node": "nas", "at": None, "age_s": 120}}
|
||||
"node": "nas", "at": None, "age_s": 120, "task_id": "t" * 25, "slot": 1}}
|
||||
|
||||
|
||||
async def test_a_failing_service_leads_and_the_failures_are_listed_once_with_their_logs(app):
|
||||
"""#5300: each failure links to the logs of what died; a service's to its
|
||||
failed task's lines on the service page."""
|
||||
now = datetime.now(timezone.utc)
|
||||
sick = _sick(now)
|
||||
failures = [{"kind": "service", "name": "codex_db", "host_id": None, "host": "nas", "count": 9,
|
||||
"at": now - timedelta(minutes=2), "detail": "task: non-zero exit (1)"},
|
||||
"at": now - timedelta(minutes=2), "detail": "task: non-zero exit (1)", "task_id": "t" * 25},
|
||||
{"kind": "container", "name": "backup", "host_id": "h1", "host": "nas", "count": 1,
|
||||
"at": now - timedelta(hours=3), "detail": "exit 2"}]
|
||||
async with app.test_request_context("/plugins/docker/rows"):
|
||||
page = await render_plugin_template(
|
||||
app, "docker", "docker/rows.html", docker_down=[], attention=[], says=svc.PROBLEM_SAYS,
|
||||
severity=svc.PROBLEM_SEVERITY, unhealthy=0, restarting=0, stale=0, host_groups=[],
|
||||
swarm_services=[sick], services_healthy=0, failing_services=[sick], failures=failures,
|
||||
running=0, stopped=0, docker_hosts=1)
|
||||
page = re.sub(r">\s+<", "><", " ".join(page.split()))
|
||||
"at": now - timedelta(hours=3), "detail": "exit 2", "task_id": None}]
|
||||
page = await _rows(app, services=[service_row(sick, now)], services_total=1, failing_services=[sick],
|
||||
failures=failures, docker_hosts=1)
|
||||
card = page[page.index("Needs attention"):page.index("</section>")]
|
||||
assert '<span class="badge badge-crit">Crash-looping</span><a href="/plugins/docker/swarm">codex_db</a>' in card
|
||||
assert ('<span class="badge badge-crit">Crash-looping</span>'
|
||||
'<a href="/plugins/docker/service/codex_db">codex_db</a>') in card
|
||||
assert "0 of 1 running · 7 tasks failed in the last hour · last task failed: task: non-zero exit (1)" in card
|
||||
assert f'href="/plugins/docker/service/codex_db?task={"t" * 25}#logs"' in card
|
||||
assert "since <time" in card
|
||||
assert '<span class="stat-tile-label">Swarm services</span><span class="stat-tile-value">0/1</span>' in page
|
||||
assert '<span class="tone-warn">0/1</span> services healthy' in page
|
||||
listed = page[page.index('id="failures"'):]
|
||||
listed = listed[:listed.index("</section>")]
|
||||
assert listed.index("codex_db") < listed.index("backup") # newest first
|
||||
assert "9 failures · last: task: non-zero exit (1) · nas" in listed
|
||||
assert 'href="/plugins/docker/container/h1/backup">backup</a>' in listed
|
||||
assert f'/plugins/docker/service/codex_db?task={"t" * 25}#logs"' in listed and "Logs of that task" in listed
|
||||
assert 'href="/plugins/docker/container/h1/backup/logs"' in listed
|
||||
assert '<span class="stat-tile-label">Failed (24h)</span><span class="stat-tile-value">2</span>' in page
|
||||
|
||||
|
||||
async def test_services_and_containers_are_tables_that_sort_and_filter(app):
|
||||
"""#5299, #5301: one row per service with its stack; every standalone
|
||||
container in one table with its host; headers sort; a stack chip filters."""
|
||||
now = datetime.now(timezone.utc)
|
||||
web = {"name": "arr_radarr", "mode": "replicated", "running": 2, "desired": 2, "state": "healthy",
|
||||
"crash_looping": False, "failures_24h": 0, "last_error": None, "trouble_since": None,
|
||||
"image": "lscr.io/linuxserver/radarr:5", "replicas": [
|
||||
{"ghost": False, "host": "docker02", "host_id": "h2", "name": "arr_radarr.1.x", "status": "running",
|
||||
"cpu_pct": 25.0, "mem_bytes": 300 << 20, "started_at": now - timedelta(hours=3),
|
||||
"image": "lscr.io/linuxserver/radarr:5", "health": None},
|
||||
{"ghost": True, "host": "edge01", "count": 1}]}
|
||||
proxy = _c("traefik", host_id="h1", compose_project="edge", image="traefik:v3.1",
|
||||
ports_json=None, cpu_pct=3.0)
|
||||
page = await _rows(app, services=[service_row(web, now)], services_total=3, services_healthy=3,
|
||||
containers=[_crow(proxy, ports=[{"host_port": 443, "container_port": 443, "protocol": "tcp"}])],
|
||||
containers_total=1, running=1, docker_hosts=2,
|
||||
f=dict(_F, sort="cpu", dir="desc"))
|
||||
services = page[page.index('id="docker-services"'):]
|
||||
services = services[:services.index("</section>")]
|
||||
assert "1 of 3" in services and 'href="/plugins/docker/swarm"' in services
|
||||
assert 'href="/plugins/docker/service/arr_radarr" title="arr_radarr · lscr.io/linuxserver/radarr:5">arr_radarr</a>' in services
|
||||
assert '<a class="chip" href="/plugins/docker/?stack=arr"' in services
|
||||
assert "docker02, " in services and "edge01 (no agent)" in services
|
||||
assert ">0.25 cores<" in services and ">300.0 MiB<" in services and ">3h" in services
|
||||
assert 'aria-sort="descending"' in services and 'href="/plugins/docker/?sort=mem"' in services
|
||||
containers = page[page.index('id="docker-containers"'):]
|
||||
assert "Other containers" in containers and ">nas<" in containers
|
||||
assert '<a class="chip" href="/plugins/docker/?stack=edge"' in containers
|
||||
assert '<td class="cell-image" title="traefik:v3.1">traefik:v3.1</td>' in containers
|
||||
assert '<div class="cell-sub mono">443:443/tcp</div>' in containers
|
||||
assert not hex_colours(page), hex_colours(page)
|
||||
|
||||
|
||||
async def test_a_filter_that_matches_nothing_says_so(app):
|
||||
page = await _rows(app, services_total=4, containers_total=2, filtered=True, f=dict(_F, q="zzz"))
|
||||
assert "Nothing matches these filters" in page and "No containers reported yet" not in page
|
||||
|
||||
|
||||
async def test_the_docker_page_has_a_filter_bar_the_rows_read(app):
|
||||
from quart import session
|
||||
async with app.test_request_context("/plugins/docker/?stack=arr"):
|
||||
session["user_role"] = "viewer"
|
||||
page = await render_plugin_template(
|
||||
app, "docker", "docker/index.html", poll_interval=10, has_swarm=True, has_disk=False,
|
||||
f=dict(_F, stack="arr", sort="mem", dir="desc"), choices={"stacks": ["arr", "edge"], "hosts": ["a", "b"]},
|
||||
states=DOCKER_STATES, no_stack=NO_STACK)
|
||||
bar = re.search(r'<form id="docker-filters".*?</form>', page, re.S).group(0)
|
||||
assert 'type="search" id="docker-q" name="q"' in bar
|
||||
assert '<option value="arr" selected>arr</option>' in bar and f'<option value="{NO_STACK}"' in bar
|
||||
assert 'aria-label="Host or node"' in bar and 'aria-label="Status"' in bar
|
||||
assert '<input type="hidden" name="sort" value="mem">' in bar
|
||||
assert 'hx-include="#docker-filters"' in page and "change from:#docker-filters" in page
|
||||
|
||||
|
||||
async def test_the_disk_page_explains_its_terms_and_names_untagged_images(app):
|
||||
|
||||
@@ -0,0 +1,207 @@
|
||||
"""The Docker page's tables (#5299, #5301) and a service's page (#5302): the
|
||||
rows, their search, filters and sort, and the helpers that find a service's
|
||||
tasks, logs and summed history."""
|
||||
import re
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from types import SimpleNamespace as NS
|
||||
|
||||
from quart import session
|
||||
|
||||
from plugins.docker import service as svc
|
||||
from plugins.docker.overview import (
|
||||
NO_STACK, container_row, filter_options, filter_rows, service_row, sort_rows, stack_of,
|
||||
)
|
||||
from plugins.docker.routes import _filters, _page_url, _sort_url, _task_rows
|
||||
from tests._templates import render_plugin_template
|
||||
|
||||
NOW = datetime(2026, 10, 7, 12, 0, tzinfo=timezone.utc)
|
||||
TASK = "x7q2m9k4p1r8s5t3v6w0y2z4a" # 25 characters, like Swarm's
|
||||
|
||||
|
||||
def _replica(host, cpu=10.0, mem=100, started=2, status="running", **kw):
|
||||
return dict(ghost=False, host=host, host_id=f"id-{host}", name=f"r.{host}", status=status,
|
||||
cpu_pct=cpu, mem_bytes=mem, started_at=NOW - timedelta(hours=started), image="img:1",
|
||||
health=kw.get("health"))
|
||||
|
||||
|
||||
def _service(name, state="healthy", running=1, desired=1, replicas=None, **kw):
|
||||
s = dict(name=name, mode="replicated", running=running, desired=desired, state=state,
|
||||
crash_looping=False, failures_24h=0, last_error=None, trouble_since=None,
|
||||
replicas=replicas if replicas is not None else [_replica("a")])
|
||||
s.update(kw)
|
||||
return s
|
||||
|
||||
|
||||
def _container(name, **kw):
|
||||
base = dict(name=name, host_id="h1", image="nginx:1", status="running", health=None, exit_code=None,
|
||||
oom_killed=False, restart_count=0, compose_project=None, started_at=NOW - timedelta(hours=1),
|
||||
scraped_at=NOW)
|
||||
base.update(kw)
|
||||
c = NS(**base)
|
||||
usage = {"cores": 0.5, "mem_bytes": 2048}
|
||||
return container_row(c, label=name, host="nas", usage=usage, ports=[], stale=False,
|
||||
exit_failed=bool(kw.get("exit_code")), now=NOW)
|
||||
|
||||
|
||||
def test_a_service_reads_its_stack_from_its_name():
|
||||
assert stack_of("arr_radarr") == "arr" and stack_of("steward_app_db") == "steward"
|
||||
assert stack_of("portainer") is None and stack_of("_x") is None and stack_of("x_") is None
|
||||
|
||||
|
||||
def test_a_service_row_adds_up_its_running_replicas():
|
||||
s = _service("arr_radarr", running=2, desired=3, state="degraded", replicas=[
|
||||
_replica("b", cpu=50.0, mem=300, started=5), _replica("a", cpu=25.0, mem=100, started=1),
|
||||
_replica("c", cpu=99.0, mem=999, status="exited"), {"ghost": True, "host": "edge", "count": 1}])
|
||||
row = service_row(s, NOW)
|
||||
assert row["stack"] == "arr" and row["trouble"] == 2
|
||||
assert row["cpu_cores"] == 0.75 and row["mem_bytes"] == 400 # the exited one is not counted
|
||||
assert row["started_at"] == NOW - timedelta(hours=1) and row["up_s"] == 3600
|
||||
assert row["nodes"] == ["a", "b", "c"] and row["ghost_nodes"] == ["edge"]
|
||||
assert row["image"] == "img:1"
|
||||
empty = service_row(_service("solo", state="down", running=0, replicas=[]), NOW)
|
||||
assert empty["cpu_cores"] is None and empty["mem_bytes"] is None and empty["up_s"] is None
|
||||
|
||||
|
||||
def test_rows_filter_by_search_stack_host_and_status():
|
||||
rows = [service_row(_service("arr_radarr"), NOW),
|
||||
service_row(_service("arr_sonarr", state="down", running=0, replicas=[]), NOW),
|
||||
service_row(_service("portainer", replicas=[_replica("b")]), NOW),
|
||||
_container("traefik", compose_project="edge"),
|
||||
_container("backup", status="exited", exit_code=0)]
|
||||
|
||||
def names(**kw):
|
||||
return [r["name"] for r in filter_rows(rows, **kw)]
|
||||
|
||||
assert names(q="SONARR") == ["arr_sonarr"] and names(q="nginx") == ["traefik", "backup"]
|
||||
assert names(stack="arr") == ["arr_radarr", "arr_sonarr"] and names(q="edge") == ["traefik"]
|
||||
assert names(stack=NO_STACK) == ["portainer", "backup"]
|
||||
assert names(host="b") == ["portainer"] and names(host="nas") == ["traefik", "backup"]
|
||||
assert names(state="problem") == ["arr_sonarr"]
|
||||
assert names(state="stopped") == ["arr_sonarr", "backup"]
|
||||
assert names(state="healthy") == ["arr_radarr", "portainer", "traefik"]
|
||||
assert filter_options(rows[:3], rows[3:]) == {"stacks": ["arr", "edge"], "hosts": ["a", "b", "nas"]}
|
||||
|
||||
|
||||
def test_rows_sort_by_column_with_missing_values_last():
|
||||
busy = service_row(_service("busy", replicas=[_replica("a", cpu=300.0)]), NOW)
|
||||
idle = service_row(_service("idle", replicas=[_replica("a", cpu=5.0)]), NOW)
|
||||
down = service_row(_service("down", state="down", running=0, replicas=[]), NOW)
|
||||
rows = [idle, down, busy]
|
||||
assert [r["name"] for r in sort_rows(rows, "cpu", True)] == ["busy", "idle", "down"]
|
||||
assert [r["name"] for r in sort_rows(rows, "cpu", False)] == ["idle", "busy", "down"]
|
||||
assert [r["name"] for r in sort_rows(rows, "status", False)] == ["down", "busy", "idle"]
|
||||
assert [r["name"] for r in sort_rows(rows, "name", True)] == ["idle", "down", "busy"]
|
||||
|
||||
|
||||
def test_the_filters_live_in_the_url_without_their_defaults():
|
||||
f = _filters({"q": " web ", "stack": "arr", "state": "bogus", "sort": "nope", "dir": "x"})
|
||||
assert f == {"q": "web", "stack": "arr", "host": "", "state": "", "sort": "status", "dir": "asc"}
|
||||
assert _page_url(f) == "/plugins/docker/?q=web&stack=arr"
|
||||
assert _page_url(f, stack="") == "/plugins/docker/?q=web"
|
||||
plain = _filters({})
|
||||
assert _sort_url(plain, "cpu") == "/plugins/docker/?sort=cpu" # busiest first by default
|
||||
assert _sort_url(plain, "name") == "/plugins/docker/?sort=name"
|
||||
cpu = _filters({"sort": "cpu"})
|
||||
assert cpu["dir"] == "desc" and _sort_url(cpu, "cpu") == "/plugins/docker/?sort=cpu&dir=asc"
|
||||
|
||||
|
||||
def test_a_task_container_name_gives_its_slot_and_task():
|
||||
assert svc.task_of("web", f"web.1.{TASK}") == ("1", TASK)
|
||||
assert svc.task_of("web", f"web.q8n2.{TASK}") == ("q8n2", TASK) # a global service's node slot
|
||||
assert svc.task_of("web", f"web_db.1.{TASK}") is None
|
||||
assert svc.task_of("arr", f"arr_radarr.1.{TASK}") is None
|
||||
assert svc.task_of("web", "web.1.short") is None
|
||||
|
||||
|
||||
def test_service_history_sums_replicas_after_averaging_each():
|
||||
since = NOW - timedelta(hours=1)
|
||||
t = since + timedelta(minutes=10)
|
||||
samples = [("a", t, 100.0, 1000), ("a", t + timedelta(seconds=20), 300.0, 3000),
|
||||
("b", t + timedelta(seconds=5), 50.0, 500),
|
||||
("a", t + timedelta(minutes=5), 10.0, 100)]
|
||||
out = svc.sum_series(samples, since, NOW, points=60)
|
||||
assert out == [(t, 250.0, 2500), (t + timedelta(minutes=5), 10.0, 100)]
|
||||
|
||||
|
||||
def test_a_failure_carries_the_task_whose_logs_tell_why():
|
||||
sick = {"name": "codex_db", "tasks_known": True, "failures_24h": 2,
|
||||
"last_error": {"state": "failed", "error": "boom", "task_id": TASK, "node": "nas", "at": None}}
|
||||
(row,) = svc.failures_24h([], [sick], {})
|
||||
assert row["kind"] == "service" and row["task_id"] == TASK
|
||||
old = {"name": "legacy", "tasks_known": False, "failures_24h": None, "last_error": None}
|
||||
died = NS(container_name=f"legacy.1.{TASK}", host_id="h1", at=NOW, event="die", detail="exit 1")
|
||||
(row,) = svc.failures_24h([died], [old], {"h1": "nas"})
|
||||
assert row["kind"] == "service" and row["task_id"] == TASK
|
||||
|
||||
|
||||
def test_task_rows_say_where_each_ran_and_why_it_ended():
|
||||
rows = _task_rows([
|
||||
{"task_id": TASK, "slot": 2, "node_id": "n1", "state": "failed", "desired_state": "shutdown",
|
||||
"error": "task: non-zero exit (1)", "exit_code": 1, "at": "2026-10-07T11:00:00Z"},
|
||||
{"task_id": "y" * 25, "slot": 2, "node_id": "n9" * 10, "state": "running", "desired_state": "running"}],
|
||||
{"n1": "docker02"})
|
||||
assert rows[0]["node"] == "docker02" and rows[0]["failed"] and rows[0]["why"] == "task: non-zero exit (1)"
|
||||
assert rows[1]["node"] == ("n9" * 10)[:12] and rows[1]["current"] and not rows[1]["failed"]
|
||||
|
||||
|
||||
async def test_the_service_page_puts_state_first_and_offers_each_tasks_logs(app):
|
||||
now = datetime.now(timezone.utc)
|
||||
s = _service("arr_radarr", running=1, desired=2, state="degraded", failures_24h=3, replicas=[
|
||||
dict(_replica("docker02"), name=f"arr_radarr.1.{TASK}"), {"ghost": True, "host": "edge01", "count": 1}],
|
||||
last_error={"state": "failed", "error": "task: non-zero exit (1)", "exit_code": 1, "node": "docker02",
|
||||
"age_s": 300, "task_id": TASK, "slot": 2})
|
||||
replicas = [dict(s["replicas"][0], usage={"cores": 0.1, "cpus": 4, "cpu_share": 2.5, "mem_bytes": 100,
|
||||
"mem_pct": None, "mem_of": None, "mem_limit": None},
|
||||
uptime="2h"), s["replicas"][1]]
|
||||
tasks = _task_rows([{"task_id": TASK, "slot": 2, "node_id": "n1", "state": "failed",
|
||||
"desired_state": "shutdown", "error": "task: non-zero exit (1)", "exit_code": 1}],
|
||||
{"n1": "docker02"})
|
||||
svc_row = NS(image="lscr.io/linuxserver/radarr:5@sha256:abc", updated_at=now)
|
||||
async with app.test_request_context(f"/plugins/docker/service/arr_radarr?task={TASK}"):
|
||||
session["user_role"] = "viewer"
|
||||
page = await render_plugin_template(
|
||||
app, "docker", "docker/service.html", name="arr_radarr", service=s, row=service_row(s, now),
|
||||
svc=svc_row, tasks=tasks, replicas=replicas, stack="arr", current_range="24h",
|
||||
task_choices=[(TASK, f"2 · {TASK[:10]}"), ("y" * 25, "1 · yyyyyyyyyy")], focus_task=TASK)
|
||||
page = re.sub(r">\s+<", "><", " ".join(page.split()))
|
||||
assert '<nav class="breadcrumb"' in page and 'href="/plugins/docker/">Docker</a>' in page
|
||||
assert page.index("Degraded") < page.index("Last failed task") < page.index('id="svc-replicas"')
|
||||
assert "1 of 2 running" in page and '<a class="chip" href="/plugins/docker/?stack=arr">arr</a>' in page
|
||||
assert f'href="?task={TASK}#logs">its logs →</a>' in page and "5m ago" in page
|
||||
assert "radarr:5@sha256:abc" in page
|
||||
assert "1 running on a node with no Steward agent" in page and '<a href="/hosts/id-docker02">docker02</a>' in page
|
||||
assert page.index('id="svc-replicas"') < page.index('id="svc-history"') < page.index('id="tasks"') < page.index('id="logs"')
|
||||
assert 'hx-get="/plugins/docker/service/arr_radarr/history"' in page and 'id="time-range"' in page
|
||||
assert '<span class="badge badge-crit">Failed</span>' in page
|
||||
assert f'<option value="{TASK}" selected>2 · {TASK[:10]}</option>' in page
|
||||
assert 'hx-get="/plugins/docker/service/arr_radarr/logs/lines"' in page
|
||||
|
||||
|
||||
async def test_a_service_nobody_reports_is_not_found(app):
|
||||
async with app.test_request_context("/plugins/docker/service/gone"):
|
||||
session["user_role"] = "viewer"
|
||||
page = await render_plugin_template(app, "docker", "docker/service.html", name="gone", service=None)
|
||||
assert "Service not found" in page and 'id="logs"' not in page
|
||||
|
||||
|
||||
async def test_a_service_history_shows_now_and_peak(app):
|
||||
async with app.test_request_context("/plugins/docker/service/web/history"):
|
||||
page = await render_plugin_template(
|
||||
app, "docker", "docker/_service_history.html", have_data=True, range_key="24h",
|
||||
cpu_now=0.5, cpu_peak=2.0, mem_now=100 << 20, mem_peak=300 << 20,
|
||||
spark_cpu="<svg></svg>", spark_mem="<svg></svg>")
|
||||
empty = await render_plugin_template(
|
||||
app, "docker", "docker/_service_history.html", have_data=False, range_key="1h")
|
||||
assert "0.50 cores" in page and "peak 2.0 cores" in page and "peak 300.0 MiB" in page
|
||||
assert "Not enough samples in the last 1h" in empty
|
||||
|
||||
|
||||
async def test_a_service_log_line_names_its_task(app):
|
||||
lines = [{"ts": NOW, "stream": "stderr", "line": "<b>boom</b>", "task": f"1 · {TASK[:10]}"}]
|
||||
async with app.test_request_context("/plugins/docker/service/web/logs/lines"):
|
||||
page = await render_plugin_template(app, "docker", "docker/_container_logs_lines.html",
|
||||
lines=lines, name="web", filtered=False, service=True)
|
||||
empty = await render_plugin_template(app, "docker", "docker/_container_logs_lines.html",
|
||||
lines=[], name="web", filtered=False, service=True)
|
||||
assert f"1 · {TASK[:10]}</span>" in page and "<b>boom</b>" in page
|
||||
assert "one of its tasks" in empty
|
||||
Reference in New Issue
Block a user