diff --git a/plugins/docker/routes.py b/plugins/docker/routes.py index 95eb331..a9e1050 100644 --- a/plugins/docker/routes.py +++ b/plugins/docker/routes.py @@ -20,6 +20,7 @@ from .swarm_view import ( build_swarm_services, failing_services, parse_tasks, replaced_tasks, services_problems_first, task_summary, trouble_since, ) +from .swarm_nodes import node_cards from .overview import ( DOCKER_SORTS, DOCKER_STATES, NO_STACK, container_row, filter_rows, service_row, sort_rows, stack_of, @@ -497,7 +498,9 @@ async def container_detail(host_id: str, name: str): @docker_bp.get("/swarm") @require_role(UserRole.viewer) async def swarm(): - """Swarm topology page: services with replica health, nodes, placement. + """The Swarm page, laid out by node (#5303): each node's role, whether it + takes work, what its containers use against what the machine has, and the + services placed on it; services short of replicas first. Swarm services/nodes are cluster-global — every manager's API returns the same list — but each manager reports them independently (rows keyed by @@ -515,6 +518,20 @@ async def swarm(): DockerSwarmNode.role, DockerSwarmNode.hostname) )).scalars()) hosts = await docker_svc.host_map(db, {s.host_id for s in services} | {n.host_id for n in nodes}) + containers = await docker_svc.list_containers(db) + # The Steward host on each node: the one whose agent reports its + # containers, else the host named as the node is. + node_host = {c.node_id: c.host_id for c in containers if c.node_id} + unmatched = {(n.hostname or "").lower() for n in nodes if n.node_id not in node_host} - {""} + if unmatched: + by_name = {h.name.lower(): h for h in (await db.execute( + select(Host).where(func.lower(Host.name).in_(unmatched)))).scalars()} + for n in nodes: + h = by_name.get((n.hostname or "").lower()) + if n.node_id not in node_host and h is not None: + node_host[n.node_id] = h.id + hosts.update(await docker_svc.host_map(db, set(node_host.values()) - set(hosts))) + capacity = await docker_svc.host_capacity(db, set(node_host.values())) # ── Group reporting managers into swarms by shared node membership ────────── manager_ids = {s.host_id for s in services} | {n.host_id for n in nodes} @@ -586,17 +603,16 @@ async def swarm(): **summary, "trouble_since": trouble_since(tasks, state, summary["crash_looping"], now), }) - # Problems first: a node that is down or not taking work, then by role. - node_rows = sorted( - node_by_id.values(), key=lambda n: ( - n.status == "ready" and n.availability == "active", - n.role, (n.hostname or n.node_id).lower()) - ) manager_names = sorted(hosts[m].name if m in hosts else m for m in members) swarms.append({ "managers": manager_names, - "services": services_problems_first(svc_dicts), - "nodes": node_rows, + "services_total": len(svc_dicts), + "services_healthy": sum(d["state"] == "healthy" for d in svc_dicts), + "troubled": failing_services(svc_dicts), + "nodes": node_cards( + node_by_id.values(), svc_by_name.values(), containers, node_host=node_host, + host_names={h.id: h.name for h in hosts.values()}, capacity=capacity, + service_state={d["name"]: d["state"] for d in svc_dicts}), }) swarms.sort(key=lambda g: g["managers"][0].lower() if g["managers"] else "") diff --git a/plugins/docker/swarm_nodes.py b/plugins/docker/swarm_nodes.py new file mode 100644 index 0000000..c8f7a75 --- /dev/null +++ b/plugins/docker/swarm_nodes.py @@ -0,0 +1,71 @@ +# plugins/docker/swarm_nodes.py +"""The Swarm page laid out by node (#5303): one card per node with its role, +whether it takes work, what its containers use against what the machine has, +and the services Swarm placed on it. Model-free, so it is unit-tested without +a database; routes.swarm() feeds it. + +A node with a Steward agent reports its own containers, so its services carry +CPU and memory. A node without one is known only from the managers' placement: +its services are listed by count, with no figures. +""" +from __future__ import annotations + +from .swarm_view import _placement + + +def node_ok(n) -> bool: + """A node that is up and taking work.""" + return n.status == "ready" and n.availability == "active" + + +def node_cards(nodes, services, containers, *, node_host: dict, host_names: dict, + capacity: dict, service_state: dict) -> list[dict]: + """One card per node, a node in trouble first, then managers, then by name. + + `nodes` and `services` are one swarm's deduplicated rows; `containers` + every container row (only running swarm tasks on these nodes count); + `node_host` maps a node id to the Steward host on it, `capacity` a host id + to (cores, memory bytes), `service_state` a service name to healthy, + degraded or down.""" + running_on: dict[str, dict[str, dict]] = {} + for c in containers: + if not (c.node_id and c.service_name and c.status == "running"): + continue + row = running_on.setdefault(c.node_id, {}).setdefault( + c.service_name, {"name": c.service_name, "tasks": 0, "cpu_pct": 0.0, "mem_bytes": 0, + "measured": True}) + row["tasks"] += 1 + row["cpu_pct"] += c.cpu_pct or 0.0 + row["mem_bytes"] += c.mem_usage_bytes or 0 + placed: dict[str, dict[str, int]] = {} + for s in services: + for p in _placement(s): + count = int(p.get("running") or 0) + if count > 0 and p.get("node_id"): + placed.setdefault(p["node_id"], {})[s.service_name] = count + + cards = [] + for n in nodes: + rows = dict(running_on.get(n.node_id, {})) + for name, count in placed.get(n.node_id, {}).items(): + rows.setdefault(name, {"name": name, "tasks": count, "cpu_pct": None, "mem_bytes": None, + "measured": False}) + for r in rows.values(): + r["state"] = service_state.get(r["name"], "healthy") + r["cores"] = r["cpu_pct"] / 100 if r["cpu_pct"] is not None else None + tasks = sorted(rows.values(), key=lambda r: (r["state"] == "healthy", -(r["cores"] or 0.0), + r["name"].lower())) + host_id = node_host.get(n.node_id) + cores, mem = capacity.get(host_id, (None, None)) if host_id else (None, None) + measured = [r for r in tasks if r["measured"]] + cards.append({ + "node_id": n.node_id, "hostname": n.hostname or n.node_id[:12], "role": n.role, + "leader": n.leader, "status": n.status, "availability": n.availability, "ok": node_ok(n), + "host_id": host_id, "host_name": host_names.get(host_id) if host_id else None, + "agent": bool(running_on.get(n.node_id)) or host_id is not None, + "tasks": tasks, "task_count": sum(r["tasks"] for r in tasks), + "cores_used": sum(r["cores"] for r in measured) if measured else None, + "mem_used": sum(r["mem_bytes"] for r in measured) if measured else None, + "cores": cores, "mem": mem, + }) + return sorted(cards, key=lambda c: (c["ok"], c["role"] != "manager", c["hostname"].lower())) diff --git a/plugins/docker/templates/docker/swarm.html b/plugins/docker/templates/docker/swarm.html index 4da31c3..9009e02 100644 --- a/plugins/docker/templates/docker/swarm.html +++ b/plugins/docker/templates/docker/swarm.html @@ -1,95 +1,103 @@ -{# docker/swarm.html — Swarm topology: services, replica health, nodes, placement. - Services in trouble come first (crash loop, down, short), with since-times; - nodes that are down or not taking work come first. #} +{# docker/swarm.html — the Swarm laid out by node (#5303). A verdict, the + services short of replicas, then one card per node: a node that is down or + not taking work first, then managers. Each card says what its containers use + against what the machine has, and lists the services placed on it. #} {% extends "base.html" %} -{% from "_macros.html" import crumbs %} +{% from "_macros.html" import crumbs, status_badge, status_dot %} +{% from "docker/_usage.html" import cores as fmt_cores %} {% block title %}Swarm — Docker — Steward{% endblock %} {% block breadcrumb %}{{ crumbs([("Docker", "/plugins/docker/"), ("Swarm", "")]) }}{% endblock %} {% block content %} -

Swarm

+

Swarm

{% if swarms %} {% for g in swarms %} -
-
- Swarm - - reported by {{ g.managers | length }} manager{{ 's' if g.managers | length != 1 }} · {{ g.managers | join(", ") }} - -
+{% set down_nodes = g.nodes|rejectattr("ok")|list %} +
+

+ {% if down_nodes or g.troubled %}{{ down_nodes|length + g.troubled|length }} need{{ "s" if (down_nodes|length + g.troubled|length) == 1 }} attention + {% else %}Every node is up and taking work{% endif %} + · {{ g.nodes|length }} node{{ "s" if g.nodes|length != 1 }} + · {{ g.services_healthy }}/{{ g.services_total }} services healthy + · reported by {{ g.managers|join(", ") }} +

- {# ── Services ─────────────────────────────────────────────────────────── #} -
- - - - - - - - {% for s in g.services %} - - - - - - - - - {% else %} - - {% endfor %} - -
ServiceModeReplicasImagePlacementTask failures
{{ s.name }}{{ s.mode }} - - {{ s.running }}/{{ s.desired }} - - {% if s.state != 'healthy' and s.trouble_since %}
since {{ s.trouble_since|ago }}
{% endif %} - 		{{ s.image or "—" }} - {% for p in s.placement %}{{ p.node }} ({{ p.running }}){% if not loop.last %}, {% endif %}{% else %}—{% endfor %} - - {% include "docker/_swarm_failures.html" %} - {%- if not s.tasks_known %}— - {%- elif not (s.crash_looping or s.failures_24h or (s.last_error and s.state != 'healthy')) %}none in 24h{% endif %} -
No services in this swarm.
-
+ {% if down_nodes or g.troubled %} +
+
    + {% for n in down_nodes %} +
  • + {{ status_badge('crit', 'Node ' ~ n.status) if n.status != 'ready' else status_badge('warn', n.availability|capitalize) }} + {{ n.hostname }} + {{ n.role }} · {{ n.task_count }} task{{ "s" if n.task_count != 1 }} placed on it +
    • + {% endfor %} + {% for s in g.troubled %} +
  • + {% if s.crash_looping %}{{ status_badge('crit', 'Crash-looping') }}{% elif s.state == 'down' %}{{ status_badge('crit', 'Down') }}{% else %}{{ status_badge('warn', 'Degraded') }}{% endif %} + {{ s.name }} + {% set e = s.last_error %} + + {{- s.running }} of {{ s.desired }} running + {%- if e %} · last task {{ e.state }}: {{ e.error or e.message or e.state }}{% if e.node %} on {{ e.node }}{% endif %}{% endif %} + {%- if s.trouble_since %} · since {{ s.trouble_since|ago }}{% endif -%} + +
    • + {% endfor %} +
+
+ {% endif %} - {# ── Nodes ────────────────────────────────────────────────────────────── #} -
- - - - - - {% for n in g.nodes %} - - - - - - +
+ {% for n in g.nodes %} +
+

+ {{ status_dot('up' if n.status == 'ready' else 'down') }} + {% if n.host_id %}{{ n.hostname }}{% else %}{{ n.hostname }}{% endif %} + {{ "leader" if n.leader else n.role }} + {% if n.availability != 'active' %}{{ status_badge('warn', n.availability|capitalize, title="Swarm places no new tasks here") }}{% endif %} +

+

+ {% if n.cores_used is not none %} + Containers use {{ fmt_cores(n.cores_used) }}{% if n.cores %} of {{ n.cores }}{% endif %} + · {{ n.mem_used|bytes }}{% if n.mem %} of {{ n.mem|bytes }}{% endif %} + {% elif n.agent %} + No swarm task reports figures here yet. {% else %} -

- {% endfor %} - -
NodeRoleAvailabilityStatus
- {{ n.hostname or n.node_id }} - {% if n.leader %}leader{% endif %} - {{ n.role }}{{ n.availability }} - - {{ n.status }} -
No nodes reported.
+ No Steward agent: what runs here is Swarm's placement, without figures. + {% endif %} +

+ {% if n.tasks %} + + + + {% for t in n.tasks %} + + + + + + + {% endfor %} + +
ServiceTasksCPUMemory
+ {%- if t.state != 'healthy' %}{{ status_dot('down' if t.state == 'down' else 'warn') }} {% endif -%} + {{ t.name }} + {{ t.tasks }}{% if t.cores is not none %}{{ fmt_cores(t.cores) }}{% else %}—{% endif %}{% if t.mem_bytes is not none %}{{ t.mem_bytes|bytes }}{% else %}—{% endif %}
+ {% else %} +

No tasks placed here.

+ {% endif %} +
+ {% endfor %}
- + {% endfor %} {% else %} -
-
- No Swarm topology reported. A Steward host agent running on a Swarm - manager reports services, nodes, and task placement here — - workers and non-swarm hosts contribute only their local containers. -
+
+ No Swarm topology reported. A Steward host agent running on a Swarm + manager reports services, nodes and task placement here; + workers and hosts outside a swarm contribute only their own containers.
{% endif %} {% endblock %} diff --git a/steward/static/steward.css b/steward/static/steward.css index 8ebcfce..17f405f 100644 --- a/steward/static/steward.css +++ b/steward/static/steward.css @@ -213,6 +213,17 @@ strong, b, th { font-weight: 500; } over a line that spans its half (docker/_service_history.html). */ .history-pair { display: grid; grid-template-columns: repeat(auto-fit, minmax(16rem, 1fr)); gap: 1.25rem; } .history-pair .spark { color: var(--blue); margin-top: 0.4rem; } +/* The Swarm page (#5303): one card per node, two or three a row on a wide + screen; each card's own services table. */ +.swarm-group { margin-bottom: 2rem; } +.node-grid { display: grid; grid-template-columns: repeat(auto-fill, minmax(min(100%, 22rem), 1fr)); gap: 1rem; align-items: start; } +.node-card { margin-bottom: 0; display: flex; flex-direction: column; gap: 0.6rem; } +.node-card-head { display: flex; align-items: center; flex-wrap: wrap; gap: 0.4rem; margin: 0; font-family: var(--fs-font-body); font-size: 1rem; font-weight: 500; } +.node-card-head .chip { margin: 0; } +.node-card-figures { margin: 0; font-size: 0.85rem; } +.node-tasks th, .node-tasks td { padding: 0.35rem 0.5rem; font-size: 0.82rem; } +.node-tasks .cell-name { max-width: 12rem; } +.node-tasks .cell-name > a { display: inline-block; max-width: 100%; vertical-align: bottom; } /* Forms */ .form-group { margin-bottom: 1rem; } diff --git a/tests/plugins/docker/test_overview.py b/tests/plugins/docker/test_overview.py index d970f7e..f5c9c05 100644 --- a/tests/plugins/docker/test_overview.py +++ b/tests/plugins/docker/test_overview.py @@ -205,3 +205,74 @@ async def test_a_service_log_line_names_its_task(app): lines=[], name="web", filtered=False, service=True) assert f"1 · {TASK[:10]}" in page and "<b>boom</b>" in page assert "one of its tasks" in empty + + +def _node(nid, hostname, role="worker", status="ready", availability="active", leader=False): + return NS(node_id=nid, hostname=hostname, role=role, status=status, availability=availability, + leader=leader) + + +def _swarm_svc(name, placement): + import json + return NS(service_name=name, placement_json=json.dumps(placement)) + + +def _task(name, service, node, cpu, mem, status="running"): + return NS(name=name, service_name=service, node_id=node, status=status, cpu_pct=cpu, + mem_usage_bytes=mem) + + +def test_each_node_lists_what_runs_on_it_and_what_that_uses(): + """#5303: a node with an agent has figures; one without is Swarm's placement only.""" + from plugins.docker.swarm_nodes import node_cards + nodes = [_node("n1", "docker02", role="manager", leader=True), _node("n2", "edge01"), + _node("n3", "docker03", availability="drain")] + services = [_swarm_svc("arr_radarr", [{"node_id": "n1", "running": 1}]), + _swarm_svc("web", [{"node_id": "n1", "running": 2}, {"node_id": "n2", "running": 1}]), + _swarm_svc("db", [{"node_id": "n1", "running": 1}])] + containers = [_task("arr_radarr.1.x", "arr_radarr", "n1", 150.0, 300), + _task("web.1.x", "web", "n1", 10.0, 100), _task("web.2.x", "web", "n1", 20.0, 100), + _task("web.0.old", "web", "n1", 99.0, 999, status="exited"), + _task("traefik", None, None, 5.0, 5)] + cards = node_cards(nodes, services, containers, node_host={"n1": "h1"}, host_names={"h1": "docker02"}, + capacity={"h1": (8, 16 << 30)}, + service_state={"arr_radarr": "healthy", "web": "healthy", "db": "down"}) + assert [c["hostname"] for c in cards] == ["docker03", "docker02", "edge01"] # drained first, then managers + n1 = cards[1] + assert n1["host_id"] == "h1" and n1["leader"] and n1["agent"] + assert [t["name"] for t in n1["tasks"]] == ["db", "arr_radarr", "web"] # trouble, then busiest + assert n1["tasks"][0]["cores"] is None and not n1["tasks"][0]["measured"] # placed, not reported + web = n1["tasks"][2] + assert web["tasks"] == 2 and web["cores"] == 0.3 and web["mem_bytes"] == 200 + assert n1["cores_used"] == 1.8 and n1["mem_used"] == 500 and n1["cores"] == 8 + edge = cards[2] + assert not edge["agent"] and edge["cores_used"] is None + assert edge["tasks"] == [{"name": "web", "tasks": 1, "cpu_pct": None, "mem_bytes": None, "measured": False, + "state": "healthy", "cores": None}] + assert not cards[0]["ok"] and cards[0]["tasks"] == [] + + +async def test_the_swarm_page_is_a_card_per_node(app): + from plugins.docker.swarm_nodes import node_cards + nodes = [_node("n1", "docker02", role="manager", leader=True), _node("n2", "edge01", status="down")] + cards = node_cards(nodes, [_swarm_svc("web", [{"node_id": "n2", "running": 1}])], + [_task("arr.1.x", "arr", "n1", 50.0, 1 << 30)], node_host={"n1": "h1"}, + host_names={"h1": "docker02"}, capacity={"h1": (4, 8 << 30)}, + service_state={"arr": "healthy", "web": "degraded"}) + troubled = [{"name": "web", "running": 1, "desired": 2, "state": "degraded", "crash_looping": False, + "trouble_since": None, "last_error": {"state": "failed", "error": "no suitable node", "node": None}}] + async with app.test_request_context("/plugins/docker/swarm"): + session["user_role"] = "viewer" + page = await render_plugin_template(app, "docker", "docker/swarm.html", swarms=[{ + "managers": ["docker02"], "services_total": 2, "services_healthy": 1, "troubled": troubled, + "nodes": cards}]) + page = re.sub(r">\s+<", "><", " ".join(page.split())) + assert "2 need attention" in page and '1/2 services healthy' in page + problems = page[page.index('class="problem-list"'):] + assert "Node down" in problems and "edge01" in problems + assert 'web' in problems and "last task failed: no suitable node" in problems + assert page.index('id="node-n2"') < page.index('id="node-n1"') # the down node first + assert 'docker02leader' in page + assert "Containers use 0.50 cores of 4 · 1.0 GiB of 8.0 GiB" in page + assert "No Steward agent" in page + assert 'arr' in page