diff --git a/plugins/docker/mcp_tools.py b/plugins/docker/mcp_tools.py index 494a91f..31109d8 100644 --- a/plugins/docker/mcp_tools.py +++ b/plugins/docker/mcp_tools.py @@ -25,58 +25,17 @@ from steward.models.hosts import Host from steward.models.users import UserRole from . import service as docker_svc +from .service import PROBLEM_SAYS, PROBLEM_SEVERITY, PROBLEMS, container_problems, problem_since from .dedup import dedup_by_container_id from .models import DockerDiskUsage, DockerSwarmNode, DockerSwarmService from .swarm_view import build_swarm_services, replaced_tasks, task_time -PROBLEMS = ("restarting", "crash_looping", "unhealthy", "oom_killed", "exited_nonzero") CRASH_LOOP_DEATHS = docker_svc.CRASH_LOOP_DEATHS LOG_LINE_CHARS = 2000 # ── pure helpers ───────────────────────────────────────────────────────────── -def container_problems(c, failures_1h: int, failed_24h: bool, - crash_loop_deaths: int = CRASH_LOOP_DEATHS) -> list[str]: - """What is wrong with one container, most severe first; [] when nothing is.""" - status = (c.status or "").lower() - out = [] - if status == "restarting": - out.append("restarting") - if failures_1h >= crash_loop_deaths: - out.append("crash_looping") - if status == "running" and (c.health or "") == "unhealthy": - out.append("unhealthy") - if c.oom_killed: - out.append("oom_killed") - if status in ("exited", "dead") and (c.exit_code or 0) != 0 and failed_24h: - out.append("exited_nonzero") - return out - - -def problem_since(problems: list[str], deaths: list, hour_ago: datetime, - unhealthy_at: datetime | None = None) -> datetime | None: - """When a container's worst problem began; None when the retained events - cannot say. `deaths` are its die/oom events of the last 24h, newest first. - - A restart or crash loop dates from its first death in the last hour, an - OOM kill or error exit from the death that ended it, and a failing - healthcheck from when its health last turned unhealthy. - """ - if not problems: - return None - worst = problems[0] - if worst in ("restarting", "crash_looping"): - in_hour = [e.at for e in deaths if e.at >= hour_ago] - return min(in_hour) if in_hour else (deaths[0].at if deaths else None) - if worst == "unhealthy": - return unhealthy_at - if worst == "oom_killed": - ooms = [e.at for e in deaths if e.event == "oom"] - return ooms[0] if ooms else (deaths[0].at if deaths else None) - return deaths[0].at if deaths else None - - def cut_middle(text: str, limit: int = LOG_LINE_CHARS) -> str: """Keep both ends of an over-long line and say how much of the middle went.""" if text is None or len(text) <= limit: @@ -117,13 +76,6 @@ def container_row(c, host_names: dict, problems: list[str], now: datetime, } -_SEVERITY = {"restarting": "critical", "crash_looping": "critical", "unhealthy": "critical", - "oom_killed": "warning", "exited_nonzero": "warning"} -_SAYS = {"restarting": "is restarting", "crash_looping": "keeps dying", - "unhealthy": "fails its healthcheck", "oom_killed": "was OOM-killed", - "exited_nonzero": "exited with an error"} - - def _from_iso(text: str | None) -> datetime | None: return datetime.fromisoformat(text.replace("Z", "+00:00")) if text else None @@ -137,8 +89,8 @@ def container_findings(rows: list[dict]) -> list[dict]: where = f" on {r['host_name']}" if r["host_name"] else "" detail = f" (exit {r['exit_code']})" if r["exit_code"] not in (None, 0) else "" out.append(finding( - _SEVERITY[worst], f"container_{worst}", r["container_name"], - f"container {r['container_name']}{where} {' and '.join(_SAYS[p] for p in r['problems'])}{detail}", + PROBLEM_SEVERITY[worst], f"container_{worst}", r["container_name"], + f"container {r['container_name']}{where} {' and '.join(PROBLEM_SAYS[p] for p in r['problems'])}{detail}", since=_from_iso(r["problem_since"]), ids={"host_id": r["host_id"], "container_name": r["container_name"]}, next=f'get_container(container="{r["container_name"]}", host="{r["host_id"]}")')) @@ -204,24 +156,10 @@ async def _replaced(db, containers) -> set[tuple]: async def _rows(db, now: datetime) -> list[dict]: """Every container with its problems. A swarm task Swarm replaced has none: its failure, if it was one, belongs to its service (service_findings).""" - containers = dedup_by_container_id(await docker_svc.list_containers(db)) - replaced = await _replaced(db, containers) - failures = await docker_svc.recent_failures_by_container(db, now - timedelta(hours=24)) - hour_ago = now - timedelta(hours=1) host_names = await _host_names(db) - deaths = _crash_loop_deaths() - sick = await docker_svc.unhealthy_since( - db, [(c.host_id, c.name) for c in containers if (c.health or "") == "unhealthy"]) - rows = [] - for c in containers: - key = (c.host_id, c.name) - gone = key in replaced - events = failures.get(key, []) - recent = sum(1 for e in events if e.at >= hour_ago) - problems = [] if gone else container_problems(c, recent, bool(events), deaths) - since = problem_since(problems, events, hour_ago, sick.get(key)) - rows.append(container_row(c, host_names, problems, now, replaced=gone, since=since)) - return rows + return [container_row(r["container"], host_names, r["problems"], now, + replaced=r["replaced"], since=r["since"]) + for r in await docker_svc.ranked_containers(db, now, _crash_loop_deaths())] async def _swarm(db, now: datetime): diff --git a/plugins/docker/routes.py b/plugins/docker/routes.py index 264cc82..df12571 100644 --- a/plugins/docker/routes.py +++ b/plugins/docker/routes.py @@ -63,7 +63,6 @@ def _sparkline(values: list[float], width: int = 80, height: int = 20) -> str: @require_role(UserRole.viewer) async def index(): poll_interval = current_app.config.get("MONITORS_POLL_INTERVAL", 60) - current_range = request.args.get("range", DEFAULT_RANGE) # Show the Swarm link only when a manager has actually reported topology, so # non-swarm installs aren't offered an always-empty page. async with current_app.db_sessionmaker() as db: @@ -74,7 +73,6 @@ async def index(): return await render_template( "docker/index.html", poll_interval=poll_interval, - current_range=current_range, has_swarm=has_swarm, has_disk=has_disk, ) @@ -83,11 +81,14 @@ async def index(): @docker_bp.get("/rows") @require_role(UserRole.viewer) async def rows(): - """HTMX fragment: swarm services (collapsed, cluster-complete) on top, then - non-swarm containers grouped by host with resource sparklines.""" - since, range_key = parse_range(request.args.get("range")) + """HTMX fragment: what needs attention first, then swarm services + (collapsed, cluster-complete), then non-swarm containers grouped by host.""" + now = datetime.now(timezone.utc) + stale_after = docker_svc.stale_after_seconds(current_app.config) async with current_app.db_sessionmaker() as db: + ranked = await docker_svc.ranked_containers( + db, now, docker_svc.crash_loop_deaths(current_app.config)) containers = await docker_svc.list_containers(db) services = list((await db.execute(select(DockerSwarmService))).scalars()) nodes = list((await db.execute(select(DockerSwarmNode))).scalars()) @@ -118,13 +119,11 @@ async def rows(): "host_name": host.name if host else c.host_id, "containers": [], "running": 0, "stopped": 0, } - cpu_hist, mem_hist = await docker_svc.container_history(db, c.host_id, c.name, since) g["containers"].append({ "container": c, "ports": json.loads(c.ports_json) if c.ports_json else [], "uptime": _uptime(c.started_at) if c.status == "running" else None, - "sparkline_cpu": _sparkline(cpu_hist), - "sparkline_mem": _sparkline(mem_hist), + "stale": _is_stale(c, now, stale_after), }) if c.status == "running": g["running"] += 1 @@ -152,17 +151,28 @@ async def rows(): swarm_services = swarm_view["services"] standalone_running = sum(g["running"] for g in host_groups) ghost_total = sum(r["count"] for s in swarm_services for r in s["replicas"] if r["ghost"]) + attention = [dict(r, stale=_is_stale(r["container"], now, stale_after)) + for r in docker_svc.needs_attention(ranked)] return await render_template( "docker/rows.html", host_groups=host_groups, swarm_services=swarm_services, + attention=attention, says=docker_svc.PROBLEM_SAYS, severity=docker_svc.PROBLEM_SEVERITY, + unhealthy=sum("unhealthy" in r["problems"] for r in attention), + restarting=sum(bool({"restarting", "crash_looping"} & set(r["problems"])) for r in attention), + stale=sum(1 for g in host_groups for i in g["containers"] if i["stale"]), running=standalone_running + sum(s["running"] for s in swarm_services), stopped=len(standalone) - standalone_running, total=len(containers) + ghost_total, - range_key=range_key, ) +def _is_stale(c, now: datetime, stale_after: int) -> bool: + """A container whose last report is older than the agent's staleness + window: what the page shows may no longer be true.""" + return c.scraped_at is None or (now - c.scraped_at).total_seconds() > stale_after + + @docker_bp.get("/widget") @require_role(UserRole.viewer) async def widget(): diff --git a/plugins/docker/service.py b/plugins/docker/service.py index 531e294..82ed4dc 100644 --- a/plugins/docker/service.py +++ b/plugins/docker/service.py @@ -6,14 +6,18 @@ Quart's pipeline) call the same code. """ from __future__ import annotations -from datetime import datetime +from datetime import datetime, timedelta from sqlalchemy import func, select from steward.core.time_range import bucket_seconds from steward.models.hosts import Host -from .models import DockerContainer, DockerEvent, DockerLog, DockerMetric, DockerMetricHourly +from .dedup import dedup_by_container_id +from .models import ( + DockerContainer, DockerEvent, DockerLog, DockerMetric, DockerMetricHourly, DockerSwarmService, +) +from .swarm_view import replaced_tasks CRASH_LOOP_DEATHS = 3 # default; Settings → Thresholds sets it @@ -278,3 +282,94 @@ def prune_extra_vars(prune_target: str) -> dict: if prune_target == "images": extra_vars["prune_all_images"] = True return extra_vars + + +# ── what is wrong with a container ─────────────────────────────────────────── +# Shared by the Docker page's "Needs attention" and the MCP tools, so both +# rank the same containers the same way. + +PROBLEMS = ("restarting", "crash_looping", "unhealthy", "oom_killed", "exited_nonzero") + +def container_problems(c, failures_1h: int, failed_24h: bool, + crash_loop_deaths: int = CRASH_LOOP_DEATHS) -> list[str]: + """What is wrong with one container, most severe first; [] when nothing is.""" + status = (c.status or "").lower() + out = [] + if status == "restarting": + out.append("restarting") + if failures_1h >= crash_loop_deaths: + out.append("crash_looping") + if status == "running" and (c.health or "") == "unhealthy": + out.append("unhealthy") + if c.oom_killed: + out.append("oom_killed") + if status in ("exited", "dead") and (c.exit_code or 0) != 0 and failed_24h: + out.append("exited_nonzero") + return out + + +def problem_since(problems: list[str], deaths: list, hour_ago: datetime, + unhealthy_at: datetime | None = None) -> datetime | None: + """When a container's worst problem began; None when the retained events + cannot say. `deaths` are its die/oom events of the last 24h, newest first. + + A restart or crash loop dates from its first death in the last hour, an + OOM kill or error exit from the death that ended it, and a failing + healthcheck from when its health last turned unhealthy. + """ + if not problems: + return None + worst = problems[0] + if worst in ("restarting", "crash_looping"): + in_hour = [e.at for e in deaths if e.at >= hour_ago] + return min(in_hour) if in_hour else (deaths[0].at if deaths else None) + if worst == "unhealthy": + return unhealthy_at + if worst == "oom_killed": + ooms = [e.at for e in deaths if e.event == "oom"] + return ooms[0] if ooms else (deaths[0].at if deaths else None) + return deaths[0].at if deaths else None + + +PROBLEM_SEVERITY = {"restarting": "critical", "crash_looping": "critical", "unhealthy": "critical", + "oom_killed": "warning", "exited_nonzero": "warning"} +PROBLEM_SAYS = {"restarting": "is restarting", "crash_looping": "keeps dying", + "unhealthy": "fails its healthcheck", "oom_killed": "was OOM-killed", + "exited_nonzero": "exited with an error"} + + + +async def ranked_containers(db, now: datetime, crash_loop_deaths: int) -> list[dict]: + """Every container (deduplicated) with what is wrong with it, most severe + first: {container, problems, since, replaced}. A swarm task Swarm has + replaced has no problems; its failure belongs to its service.""" + containers = dedup_by_container_id(await list_containers(db)) + services = list((await db.execute(select(DockerSwarmService))).scalars()) + replaced = replaced_tasks(containers, services) + failures = await recent_failures_by_container(db, now - timedelta(hours=24)) + hour_ago = now - timedelta(hours=1) + sick = await unhealthy_since( + db, [(c.host_id, c.name) for c in containers if (c.health or "") == "unhealthy"]) + out = [] + for c in containers: + key = (c.host_id, c.name) + gone = key in replaced + events = failures.get(key, []) + recent = sum(1 for e in events if e.at >= hour_ago) + problems = [] if gone else container_problems(c, recent, bool(events), crash_loop_deaths) + out.append({"container": c, "problems": problems, "replaced": gone, + "since": problem_since(problems, events, hour_ago, sick.get(key))}) + return out + + +def needs_attention(ranked: list[dict]) -> list[dict]: + """The containers with something wrong, worst problem first, then by host + and name.""" + return sorted((r for r in ranked if r["problems"]), + key=lambda r: (PROBLEMS.index(r["problems"][0]), r["container"].host_id, r["container"].name)) + + +def stale_after_seconds(config) -> int: + """How old a container's last report may be before the page calls it + stale: the host agent's own staleness window, since the agent reports it.""" + return int(((config.get("PLUGINS") or {}).get("host_agent") or {}).get("stale_after_seconds", 180)) diff --git a/plugins/docker/templates/docker/index.html b/plugins/docker/templates/docker/index.html index 950dd06..62dfce9 100644 --- a/plugins/docker/templates/docker/index.html +++ b/plugins/docker/templates/docker/index.html @@ -1,24 +1,16 @@ {% extends "base.html" %} +{% from "_macros.html" import page_header %} {% block title %}Docker — Steward{% endblock %} {% block content %} -
-
-

Docker

- {% if has_swarm %} - Swarm → - {% endif %} - {% if has_disk %} - Disk → - {% endif %} -
- {% include "_time_range.html" %} -
+{% call page_header("Docker", "What needs attention first, then every service and container.") %} + {% if has_swarm %}Swarm{% endif %} + {% if has_disk %}Disk{% endif %} +{% endcall %}
-
Loading...
+

Loading…

{% endblock %} diff --git a/plugins/docker/templates/docker/rows.html b/plugins/docker/templates/docker/rows.html index 8748207..b599b00 100644 --- a/plugins/docker/templates/docker/rows.html +++ b/plugins/docker/templates/docker/rows.html @@ -1,29 +1,43 @@ {# docker/rows.html — HTMX fragment for the Docker main page, grouped by host #} -{# ── Summary strip ─────────────────────────────────────────────────────────── #} -
-
-
Running
- {{ running }} -
-
-
Stopped
- {{ stopped }} -
-
-
Total
- {{ total }} -
- {% if swarm_services %} -
-
Services
- {{ swarm_services | length }} -
- {% endif %} -
-
Hosts
- {{ host_groups | length }} -
+{# docker/rows.html — HTMX fragment for the Docker page: what needs attention + first (service.needs_attention, the ranking the MCP tools use), the counts, + then swarm services and each host's containers. #} +{% from "_macros.html" import stat_tile, status_badge, status_dot %} + +{# ── Needs attention ──────────────────────────────────────────────────────── #} +{% if attention %} +
+

Needs attention

+ +
+{% endif %} + +{# ── Counts ───────────────────────────────────────────────────────────────── #} +
+ {{ stat_tile("Needs attention", attention|length, state="crit" if attention else "inactive") }} + {{ stat_tile("Unhealthy", unhealthy, state="crit" if unhealthy else "inactive") }} + {{ stat_tile("Restarting", restarting, state="crit" if restarting else "inactive", hint="restarting or crash-looping") }} + {{ stat_tile("Running", running, state="up" if running else "inactive") }} + {{ stat_tile("Stopped", stopped, state="inactive") }} + {% if stale %}{{ stat_tile("Stale", stale, state="stale", hint="no recent report") }}{% endif %} + {% if swarm_services %}{{ stat_tile("Services", swarm_services|length, href="/plugins/docker/swarm") }}{% endif %} + {{ stat_tile("Hosts", host_groups|length) }}
{# ── Swarm services: collapsed per-service, each current replica labelled with @@ -43,7 +57,7 @@ {{ s.name }} {{ s.mode }} - + {{ s.running }}/{{ s.desired }} running
@@ -92,35 +106,33 @@ Image Ports CPU % - CPU history Mem % - Mem history {% for sub in g.subgroups %} {% if g.grouped and sub.label %} - {{ sub.label }} + {{ sub.label }} {% endif %} {% for item in sub.containers %} {% set c = item.container %}
- + {{ status_dot('stale' if item.stale else ('up' if c.status == 'running' else ('pending' if c.status == 'paused' else 'down')), 'Stale' if item.stale else c.status|capitalize) }}
{{ c.name }} - {% if c.health == 'healthy' %}● - {% elif c.health == 'unhealthy' %}● - {% elif c.health == 'starting' %}◐{% endif %} + {% if c.health == 'unhealthy' %} {{ status_badge('crit', 'Unhealthy') }} + {% elif c.health == 'starting' %} {{ status_badge('pending', 'Starting') }}{% endif %} + {% if item.stale %} {{ status_badge('stale', title="Last report " ~ (c.scraped_at.strftime('%Y-%m-%d %H:%M UTC') if c.scraped_at else 'never') ~ "; what is shown may no longer be true") }}{% endif %}
{{ c.status }}{% if item.uptime %} · up {{ item.uptime }}{% endif %} {% if c.status != 'running' and c.exit_code is not none and c.exit_code != 0 %} - · exit {{ c.exit_code }}{% if c.oom_killed %} (OOM){% endif %} + · exit {{ c.exit_code }}{% if c.oom_killed %} (OOM){% endif %} {% endif %} - {% if c.restart_count %} · ⟳{{ c.restart_count }}{% endif %} + {% if c.restart_count %} · ⟳{{ c.restart_count }}{% endif %}
{% if c.service_name or c.compose_project %}
@@ -140,24 +152,18 @@ {% if c.cpu_pct is not none %} - - {{ "%.1f" | format(c.cpu_pct) }}% - + {{ "%.1f" | format(c.cpu_pct) }}% {% else %} — {% endif %} - {{ item.sparkline_cpu | safe }} {% if c.mem_pct is not none %} - - {{ "%.1f" | format(c.mem_pct) }}% - + {{ "%.1f" | format(c.mem_pct) }}% {% else %} — {% endif %} - {{ item.sparkline_mem | safe }} {% endfor %} {% endfor %} diff --git a/tests/plugins/docker/test_docker_page.py b/tests/plugins/docker/test_docker_page.py new file mode 100644 index 0000000..94ee224 --- /dev/null +++ b/tests/plugins/docker/test_docker_page.py @@ -0,0 +1,76 @@ +"""The Docker page leads with what needs attention, ranked the way the MCP +tools rank it, and marks a container whose report is old as stale (#5123).""" +import re +from datetime import datetime, timedelta, timezone +from types import SimpleNamespace as NS + + +from plugins.docker import service as svc +from plugins.docker.routes import _is_stale +from tests._templates import render_plugin_template +from tests.test_style_tokens import hex_colours + +NOW = datetime.now(timezone.utc) + + +def _c(name, **kw): + base = dict(name=name, host_id="h1", image="nginx:1", status="running", health=None, exit_code=None, + oom_killed=False, restart_count=0, cpu_pct=5.0, mem_pct=10.0, compose_project=None, + service_name=None, ports_json=None, started_at=NOW - timedelta(hours=2), + scraped_at=NOW - timedelta(seconds=20)) + base.update(kw) + return NS(**base) + + +def test_needs_attention_ranks_the_worst_problem_first(): + ranked = [{"container": _c(n), "problems": p, "since": None, "replaced": False} for n, p in [ + ("fine", []), ("oom", ["oom_killed"]), ("sick", ["unhealthy"]), + ("loop", ["crash_looping"]), ("exit", ["exited_nonzero"]), ("bounce", ["restarting", "crash_looping"])]] + assert [r["container"].name for r in svc.needs_attention(ranked)] == ["bounce", "loop", "sick", "oom", "exit"] + + +def test_stale_is_older_than_the_agents_window(): + assert not _is_stale(_c("a"), NOW, 180) + assert _is_stale(_c("a", scraped_at=NOW - timedelta(minutes=10)), NOW, 180) + assert _is_stale(_c("a", scraped_at=None), NOW, 180) + assert svc.stale_after_seconds({"PLUGINS": {"host_agent": {"stale_after_seconds": 60}}}) == 60 + assert svc.stale_after_seconds({}) == 180 + + +async def test_a_crash_loop_comes_first_and_a_stale_row_says_so(app): + loop = _c("api", status="restarting", exit_code=1, restart_count=7) + old = _c("cache", scraped_at=NOW - timedelta(minutes=30), cpu_pct=95.0) + attention = [{"container": loop, "problems": ["restarting", "crash_looping"], "replaced": False, + "since": datetime.now(timezone.utc) - timedelta(minutes=8), "stale": False}] + groups = [{"host": None, "host_id": "h1", "host_name": "nas", "running": 1, "stopped": 1, "grouped": False, + "subgroups": [{"label": None, "containers": [ + {"container": loop, "ports": [], "uptime": None, "stale": False}, + {"container": old, "ports": [], "uptime": "2h", "stale": True}]}]}] + async with app.test_request_context("/plugins/docker/rows"): + page = await render_plugin_template( + app, "docker", "docker/rows.html", attention=attention, says=svc.PROBLEM_SAYS, severity=svc.PROBLEM_SEVERITY, + unhealthy=0, restarting=1, stale=1, host_groups=groups, swarm_services=[], + running=1, stopped=1, total=2) + page = re.sub(r">\s+<", "><", " ".join(page.split())) + + card = page[page.index("Needs attention"):page.index("")] + assert 'Restarting' in card + assert 'href="/plugins/docker/container/h1/api">api' in card + assert "is restarting and keeps dying · exit 1 · since 8m ago" in card + assert "cache" not in card + assert page.index("Needs attention") < page.index("Restarting1' in page + assert 'Stale1' in page + assert 'class="badge badge-warn" title="Last report' in page # the stale row + assert '95.0%' in page # CPU through Settings → Thresholds + assert "history" not in page.lower() and "" not in page and "No containers reported yet" in page