|
|
|
@@ -11,7 +11,8 @@ from plugins.docker import service as svc
|
|
|
|
|
from plugins.docker.overview import (
|
|
|
|
|
NO_STACK, container_row, filter_options, filter_rows, service_row, sort_rows, stack_of,
|
|
|
|
|
)
|
|
|
|
|
from plugins.docker.routes import _filters, _page_url, _sort_url, _task_rows
|
|
|
|
|
from plugins.docker.routes import _filters, _page_url, _service_tasks, _sort_url, _task_rows
|
|
|
|
|
from tests.test_style_tokens import hex_colours
|
|
|
|
|
from tests._templates import render_plugin_template
|
|
|
|
|
|
|
|
|
|
NOW = datetime(2026, 10, 7, 12, 0, tzinfo=timezone.utc)
|
|
|
|
@@ -144,37 +145,100 @@ def test_task_rows_say_where_each_ran_and_why_it_ended():
|
|
|
|
|
assert rows[1]["node"] == ("n9" * 10)[:12] and rows[1]["current"] and not rows[1]["failed"]
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
async def test_the_service_page_puts_state_first_and_offers_each_tasks_logs(app):
|
|
|
|
|
now = datetime.now(timezone.utc)
|
|
|
|
|
LIVE = "l7q2m9k4p1r8s5t3v6w0y2z4b" # the running task
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def _degraded_radarr(now, **over):
|
|
|
|
|
"""arr_radarr at 1 of 2: a running task on docker02 with figures, one on a
|
|
|
|
|
node with no agent, and a failed task Swarm replaced."""
|
|
|
|
|
usage = {"cores": 0.1, "cpus": 4, "cpu_share": 2.5, "mem_bytes": 100 << 20,
|
|
|
|
|
"mem_pct": None, "mem_of": None, "mem_limit": None}
|
|
|
|
|
s = _service("arr_radarr", running=1, desired=2, state="degraded", failures_24h=3, replicas=[
|
|
|
|
|
dict(_replica("docker02"), name=f"arr_radarr.1.{TASK}"), {"ghost": True, "host": "edge01", "count": 1}],
|
|
|
|
|
dict(_replica("docker02"), name=f"arr_radarr.1.{LIVE}"), {"ghost": True, "host": "edge01", "count": 1}],
|
|
|
|
|
last_error={"state": "failed", "error": "task: non-zero exit (1)", "exit_code": 1, "node": "docker02",
|
|
|
|
|
"age_s": 300, "task_id": TASK, "slot": 2})
|
|
|
|
|
replicas = [dict(s["replicas"][0], usage={"cores": 0.1, "cpus": 4, "cpu_share": 2.5, "mem_bytes": 100,
|
|
|
|
|
"mem_pct": None, "mem_of": None, "mem_limit": None},
|
|
|
|
|
uptime="2h"), s["replicas"][1]]
|
|
|
|
|
tasks = _task_rows([{"task_id": TASK, "slot": 2, "node_id": "n1", "state": "failed",
|
|
|
|
|
"desired_state": "shutdown", "error": "task: non-zero exit (1)", "exit_code": 1}],
|
|
|
|
|
{"n1": "docker02"})
|
|
|
|
|
svc_row = NS(image="lscr.io/linuxserver/radarr:5@sha256:abc", updated_at=now)
|
|
|
|
|
"age_s": 300, "task_id": TASK, "slot": 2},
|
|
|
|
|
trouble_since=now - timedelta(minutes=10))
|
|
|
|
|
s.update(over)
|
|
|
|
|
replicas = [dict(s["replicas"][0], usage=usage), s["replicas"][1]]
|
|
|
|
|
tasks = _task_rows([
|
|
|
|
|
{"task_id": LIVE, "slot": 1, "node_id": "n1", "state": "running", "desired_state": "running",
|
|
|
|
|
"message": "started", "at": "2026-10-07T10:00:00Z"},
|
|
|
|
|
{"task_id": TASK, "slot": 2, "node_id": "n1", "state": "failed", "desired_state": "shutdown",
|
|
|
|
|
"error": "task: non-zero exit (1)", "exit_code": 1, "at": "2026-10-07T09:00:00Z"}],
|
|
|
|
|
{"n1": "vdnt-docker02"})
|
|
|
|
|
return s, _service_tasks("arr_radarr", tasks, replicas, {"n1": "id-docker02"}, {"id-docker02": "docker02"})
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def test_a_services_tasks_are_one_table_running_first():
|
|
|
|
|
"""#5310: the running task carries its container's figures; the node reads
|
|
|
|
|
as its Steward host wherever one is known, Swarm's hostname otherwise; a
|
|
|
|
|
node with no agent still gets a row."""
|
|
|
|
|
_, rows = _degraded_radarr(NOW)
|
|
|
|
|
assert [r["task_id"] for r in rows] == [LIVE, "", TASK]
|
|
|
|
|
live, ghost, failed = rows
|
|
|
|
|
assert live["current"] and live["usage"]["cores"] == 0.1 and live["container"] == f"arr_radarr.1.{LIVE}"
|
|
|
|
|
assert live["node"] == failed["node"] == "docker02" and failed["host_id"] == "id-docker02"
|
|
|
|
|
assert live["at"] == NOW - timedelta(hours=2) # its container's start
|
|
|
|
|
assert ghost["node"] == "edge01" and ghost["current"] and ghost["usage"] is None
|
|
|
|
|
assert "no Steward agent" in ghost["why"]
|
|
|
|
|
assert failed["failed"] and failed["usage"] is None and failed["container"] is None
|
|
|
|
|
# A running replica Swarm's record does not list still gets its row.
|
|
|
|
|
orphan = _service_tasks("arr_radarr", [], [dict(_replica("docker02"), name=f"arr_radarr.3.{LIVE}",
|
|
|
|
|
usage=None)], {}, {})
|
|
|
|
|
assert orphan[0]["slot"] == "3" and orphan[0]["task_id"] == LIVE and orphan[0]["current"]
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
async def _service_page(app, s, tasks, now):
|
|
|
|
|
async with app.test_request_context(f"/plugins/docker/service/arr_radarr?task={TASK}"):
|
|
|
|
|
session["user_role"] = "viewer"
|
|
|
|
|
page = await render_plugin_template(
|
|
|
|
|
app, "docker", "docker/service.html", name="arr_radarr", service=s, row=service_row(s, now),
|
|
|
|
|
svc=svc_row, tasks=tasks, replicas=replicas, stack="arr", current_range="24h",
|
|
|
|
|
task_choices=[(TASK, f"2 · {TASK[:10]}"), ("y" * 25, "1 · yyyyyyyyyy")], focus_task=TASK)
|
|
|
|
|
page = re.sub(r">\s+<", "><", " ".join(page.split()))
|
|
|
|
|
svc=NS(image="lscr.io/linuxserver/radarr:5@sha256:abc", updated_at=now), tasks=tasks,
|
|
|
|
|
stack="arr", current_range="24h", focus_task=TASK,
|
|
|
|
|
task_choices=[(TASK, f"2 · {TASK[:10]}"), (LIVE, f"1 · {LIVE[:10]}")])
|
|
|
|
|
return re.sub(r">\s+<", "><", " ".join(page.split()))
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
async def test_the_service_page_says_each_fact_once(app):
|
|
|
|
|
"""#5310: the verdict and what it runs in the header; the failure card only
|
|
|
|
|
while in trouble; history; one Tasks table; logs. No tiles, no kv card,
|
|
|
|
|
no separate Replicas table."""
|
|
|
|
|
now = datetime.now(timezone.utc)
|
|
|
|
|
s, tasks = _degraded_radarr(now)
|
|
|
|
|
page = await _service_page(app, s, tasks, now)
|
|
|
|
|
assert '<nav class="breadcrumb"' in page and 'href="/plugins/docker/">Docker</a>' in page
|
|
|
|
|
assert page.index("Degraded") < page.index("Last failed task") < page.index('id="svc-replicas"')
|
|
|
|
|
assert "1 of 2 running" in page and '<a class="chip" href="/plugins/docker/?stack=arr">arr</a>' in page
|
|
|
|
|
assert page.index("Degraded") < page.index("Last failed task") < page.index('id="svc-history"') \
|
|
|
|
|
< page.index('id="tasks"') < page.index('id="logs"')
|
|
|
|
|
assert "1 of 2 running" in page and "in trouble since <time" in page
|
|
|
|
|
assert '<a class="chip" href="/plugins/docker/?stack=arr">arr</a>' in page
|
|
|
|
|
assert 'title="lscr.io/linuxserver/radarr:5@sha256:abc">lscr.io/linuxserver/radarr:5</span>' in page # digest on hover
|
|
|
|
|
assert f'href="?task={TASK}#logs">its logs →</a>' in page and "5m ago" in page
|
|
|
|
|
assert "radarr:5@sha256:abc" in page
|
|
|
|
|
assert "1 running on a node with no Steward agent" in page and '<a href="/hosts/id-docker02">docker02</a>' in page
|
|
|
|
|
assert page.index('id="svc-replicas"') < page.index('id="svc-history"') < page.index('id="tasks"') < page.index('id="logs"')
|
|
|
|
|
assert "stat-tile" not in page and 'class="kv"' not in page and "svc-replicas" not in page
|
|
|
|
|
tasks_html = page[page.index('id="tasks"'):page.index('id="logs"')]
|
|
|
|
|
assert '<span class="tone-warn">· 3 failed in 24h</span>' in tasks_html
|
|
|
|
|
assert f'href="/plugins/docker/container/id-docker02/arr_radarr.1.{LIVE}"' in tasks_html
|
|
|
|
|
assert '<a href="/hosts/id-docker02">docker02</a>' in tasks_html and "vdnt-docker02" not in tasks_html
|
|
|
|
|
assert "0.10 cores" in tasks_html and "100.0 MiB" in tasks_html
|
|
|
|
|
assert "1 running on a node with no Steward agent" in tasks_html
|
|
|
|
|
assert tasks_html.index(LIVE[:10]) < tasks_html.index("edge01") < tasks_html.index(TASK[:10])
|
|
|
|
|
assert '<span class="badge badge-crit">Failed</span>' in tasks_html
|
|
|
|
|
assert 'hx-get="/plugins/docker/service/arr_radarr/history"' in page and 'id="time-range"' in page
|
|
|
|
|
assert '<span class="badge badge-crit">Failed</span>' in page
|
|
|
|
|
assert f'<option value="{TASK}" selected>2 · {TASK[:10]}</option>' in page
|
|
|
|
|
assert 'hx-get="/plugins/docker/service/arr_radarr/logs/lines"' in page
|
|
|
|
|
assert not hex_colours(page), hex_colours(page)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
async def test_a_recovered_service_is_not_in_trouble(app):
|
|
|
|
|
"""#5310: a healthy service whose last task failed hours ago says when it
|
|
|
|
|
last failed, quietly, with its logs; no red card, no "in trouble"."""
|
|
|
|
|
now = datetime.now(timezone.utc)
|
|
|
|
|
s, tasks = _degraded_radarr(now, running=2, state="healthy", failures_24h=1)
|
|
|
|
|
s["last_error"] = dict(s["last_error"], age_s=5 * 3600, exit_code=137)
|
|
|
|
|
page = await _service_page(app, s, tasks, now)
|
|
|
|
|
assert "Healthy" in page and "2 of 2 running" in page
|
|
|
|
|
assert "in trouble" not in page and "Last failed task" not in page and "problem-card" not in page
|
|
|
|
|
assert "Last failed " in page and "(exit 137)" in page and f'href="?task={TASK}#logs">its logs</a>' in page
|
|
|
|
|
assert "reported <time" in page
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
async def test_a_service_nobody_reports_is_not_found(app):
|
|
|
|
@@ -184,18 +248,69 @@ async def test_a_service_nobody_reports_is_not_found(app):
|
|
|
|
|
assert "Service not found" in page and 'id="logs"' not in page
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
async def test_a_service_history_shows_now_and_peak(app):
|
|
|
|
|
async def test_a_usage_history_shows_now_and_peak(app):
|
|
|
|
|
async with app.test_request_context("/plugins/docker/service/web/history"):
|
|
|
|
|
page = await render_plugin_template(
|
|
|
|
|
app, "docker", "docker/_service_history.html", have_data=True, range_key="24h",
|
|
|
|
|
app, "docker", "docker/_usage_history.html", have_data=True, range_key="24h",
|
|
|
|
|
cpu_now=0.5, cpu_peak=2.0, mem_now=100 << 20, mem_peak=300 << 20,
|
|
|
|
|
spark_cpu="<svg></svg>", spark_mem="<svg></svg>")
|
|
|
|
|
cpu_floor=1.0, mem_floor=1 << 30, spark_cpu="<svg></svg>", spark_mem="<svg></svg>")
|
|
|
|
|
empty = await render_plugin_template(
|
|
|
|
|
app, "docker", "docker/_service_history.html", have_data=False, range_key="1h")
|
|
|
|
|
app, "docker", "docker/_usage_history.html", have_data=False, range_key="1h")
|
|
|
|
|
assert "0.50 cores" in page and "peak 2.0 cores" in page and "peak 300.0 MiB" in page
|
|
|
|
|
assert "or to 1.0 core when the peak is lower" in page # the floor is said, not hidden
|
|
|
|
|
assert "Not enough samples in the last 1h" in empty
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def _ctr(**kw):
|
|
|
|
|
base = dict(name="plex", host_id="h1", image="plexinc/pms:1.40", status="exited", health=None,
|
|
|
|
|
exit_code=137, oom_killed=True, restart_count=2, compose_project="media",
|
|
|
|
|
service_name=None, node_id=None, scraped_at=NOW - timedelta(seconds=30),
|
|
|
|
|
started_at=NOW - timedelta(hours=3))
|
|
|
|
|
base.update(kw)
|
|
|
|
|
return NS(**base)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
async def _container_page(app, c, **over):
|
|
|
|
|
ctx = dict(container=c, host=NS(id="h1", name="nas"), host_id="h1", name=c.name, label=c.name,
|
|
|
|
|
node_name=None, exit_failed=True, ports=[], uptime=None,
|
|
|
|
|
usage={"mem_limit": None}, net_rx="1.0 KiB", net_tx="2.0 KiB", blk_read="3.0 MiB",
|
|
|
|
|
blk_write="4.0 MiB", died_at=NOW - timedelta(minutes=20), current_range="24h",
|
|
|
|
|
events=[{"event": "health_change", "detail": "healthy→unhealthy", "at": NOW,
|
|
|
|
|
"glyph": "◐", "tone": "warn"}])
|
|
|
|
|
ctx.update(over)
|
|
|
|
|
async with app.test_request_context(f"/plugins/docker/container/h1/{c.name}"):
|
|
|
|
|
session["user_role"] = "viewer"
|
|
|
|
|
page = await render_plugin_template(app, "docker", "docker/container_detail.html", **ctx)
|
|
|
|
|
return re.sub(r">\s+<", "><", " ".join(page.split()))
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
async def test_a_dead_container_says_why_first(app):
|
|
|
|
|
"""#5310: a stopped container that exited badly leads with how; what it
|
|
|
|
|
uses lives in Resource history alone; lifecycle is a table."""
|
|
|
|
|
page = await _container_page(app, _ctr())
|
|
|
|
|
assert 'href="/plugins/docker/">Docker</a>' in page
|
|
|
|
|
assert '<span class="badge badge-crit">Exited</span>' in page and 'href="/hosts/h1">nas</a>' in page
|
|
|
|
|
assert '<span class="chip">media</span>' in page and "plexinc/pms:1.40" in page
|
|
|
|
|
assert page.index("Last exit") < page.index('id="ctr-history"') < page.index('id="ctr-lifecycle"')
|
|
|
|
|
assert "killed for running out of memory" in page and "20m ago" in page
|
|
|
|
|
assert "<dt>Restarts</dt>" in page and "none set" in page and "1.0 KiB in · 2.0 KiB out" in page
|
|
|
|
|
assert 'hx-get="/plugins/docker/container/h1/plex/history"' in page
|
|
|
|
|
assert "<td><span class=\"tone-warn\" aria-hidden=\"true\">◐</span> health change</td>" in page
|
|
|
|
|
assert "stat-tile" not in page and "Net in since start" not in page
|
|
|
|
|
assert not hex_colours(page), hex_colours(page)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
async def test_a_running_swarm_task_reads_under_its_service(app):
|
|
|
|
|
c = _ctr(name=f"arr_radarr.1.{LIVE}", status="running", exit_code=0, oom_killed=False,
|
|
|
|
|
service_name="arr_radarr", compose_project=None, node_id="n1")
|
|
|
|
|
page = await _container_page(app, c, label="arr_radarr · slot 1", exit_failed=False, uptime="3h",
|
|
|
|
|
node_name="docker02")
|
|
|
|
|
assert 'href="/plugins/docker/service/arr_radarr">arr_radarr</a>' in page # crumb and "task of"
|
|
|
|
|
assert "task of <a" in page and "· up 3h" in page and f"arr_radarr.1.{LIVE}" in page
|
|
|
|
|
assert "Last exit" not in page and "<dt>Restarts</dt>" not in page
|
|
|
|
|
assert "<dt>Swarm node</dt>" in page and "docker02" in page
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
async def test_a_service_log_line_names_its_task(app):
|
|
|
|
|
lines = [{"ts": NOW, "stream": "stderr", "line": "<b>boom</b>", "task": f"1 · {TASK[:10]}"}]
|
|
|
|
|
async with app.test_request_context("/plugins/docker/service/web/logs/lines"):
|
|
|
|
|