diff --git a/plugins/docker/mcp_tools.py b/plugins/docker/mcp_tools.py index 494a91f..31109d8 100644 --- a/plugins/docker/mcp_tools.py +++ b/plugins/docker/mcp_tools.py @@ -25,58 +25,17 @@ from steward.models.hosts import Host from steward.models.users import UserRole from . import service as docker_svc +from .service import PROBLEM_SAYS, PROBLEM_SEVERITY, PROBLEMS, container_problems, problem_since from .dedup import dedup_by_container_id from .models import DockerDiskUsage, DockerSwarmNode, DockerSwarmService from .swarm_view import build_swarm_services, replaced_tasks, task_time -PROBLEMS = ("restarting", "crash_looping", "unhealthy", "oom_killed", "exited_nonzero") CRASH_LOOP_DEATHS = docker_svc.CRASH_LOOP_DEATHS LOG_LINE_CHARS = 2000 # ── pure helpers ───────────────────────────────────────────────────────────── -def container_problems(c, failures_1h: int, failed_24h: bool, - crash_loop_deaths: int = CRASH_LOOP_DEATHS) -> list[str]: - """What is wrong with one container, most severe first; [] when nothing is.""" - status = (c.status or "").lower() - out = [] - if status == "restarting": - out.append("restarting") - if failures_1h >= crash_loop_deaths: - out.append("crash_looping") - if status == "running" and (c.health or "") == "unhealthy": - out.append("unhealthy") - if c.oom_killed: - out.append("oom_killed") - if status in ("exited", "dead") and (c.exit_code or 0) != 0 and failed_24h: - out.append("exited_nonzero") - return out - - -def problem_since(problems: list[str], deaths: list, hour_ago: datetime, - unhealthy_at: datetime | None = None) -> datetime | None: - """When a container's worst problem began; None when the retained events - cannot say. `deaths` are its die/oom events of the last 24h, newest first. - - A restart or crash loop dates from its first death in the last hour, an - OOM kill or error exit from the death that ended it, and a failing - healthcheck from when its health last turned unhealthy. - """ - if not problems: - return None - worst = problems[0] - if worst in ("restarting", "crash_looping"): - in_hour = [e.at for e in deaths if e.at >= hour_ago] - return min(in_hour) if in_hour else (deaths[0].at if deaths else None) - if worst == "unhealthy": - return unhealthy_at - if worst == "oom_killed": - ooms = [e.at for e in deaths if e.event == "oom"] - return ooms[0] if ooms else (deaths[0].at if deaths else None) - return deaths[0].at if deaths else None - - def cut_middle(text: str, limit: int = LOG_LINE_CHARS) -> str: """Keep both ends of an over-long line and say how much of the middle went.""" if text is None or len(text) <= limit: @@ -117,13 +76,6 @@ def container_row(c, host_names: dict, problems: list[str], now: datetime, } -_SEVERITY = {"restarting": "critical", "crash_looping": "critical", "unhealthy": "critical", - "oom_killed": "warning", "exited_nonzero": "warning"} -_SAYS = {"restarting": "is restarting", "crash_looping": "keeps dying", - "unhealthy": "fails its healthcheck", "oom_killed": "was OOM-killed", - "exited_nonzero": "exited with an error"} - - def _from_iso(text: str | None) -> datetime | None: return datetime.fromisoformat(text.replace("Z", "+00:00")) if text else None @@ -137,8 +89,8 @@ def container_findings(rows: list[dict]) -> list[dict]: where = f" on {r['host_name']}" if r["host_name"] else "" detail = f" (exit {r['exit_code']})" if r["exit_code"] not in (None, 0) else "" out.append(finding( - _SEVERITY[worst], f"container_{worst}", r["container_name"], - f"container {r['container_name']}{where} {' and '.join(_SAYS[p] for p in r['problems'])}{detail}", + PROBLEM_SEVERITY[worst], f"container_{worst}", r["container_name"], + f"container {r['container_name']}{where} {' and '.join(PROBLEM_SAYS[p] for p in r['problems'])}{detail}", since=_from_iso(r["problem_since"]), ids={"host_id": r["host_id"], "container_name": r["container_name"]}, next=f'get_container(container="{r["container_name"]}", host="{r["host_id"]}")')) @@ -204,24 +156,10 @@ async def _replaced(db, containers) -> set[tuple]: async def _rows(db, now: datetime) -> list[dict]: """Every container with its problems. A swarm task Swarm replaced has none: its failure, if it was one, belongs to its service (service_findings).""" - containers = dedup_by_container_id(await docker_svc.list_containers(db)) - replaced = await _replaced(db, containers) - failures = await docker_svc.recent_failures_by_container(db, now - timedelta(hours=24)) - hour_ago = now - timedelta(hours=1) host_names = await _host_names(db) - deaths = _crash_loop_deaths() - sick = await docker_svc.unhealthy_since( - db, [(c.host_id, c.name) for c in containers if (c.health or "") == "unhealthy"]) - rows = [] - for c in containers: - key = (c.host_id, c.name) - gone = key in replaced - events = failures.get(key, []) - recent = sum(1 for e in events if e.at >= hour_ago) - problems = [] if gone else container_problems(c, recent, bool(events), deaths) - since = problem_since(problems, events, hour_ago, sick.get(key)) - rows.append(container_row(c, host_names, problems, now, replaced=gone, since=since)) - return rows + return [container_row(r["container"], host_names, r["problems"], now, + replaced=r["replaced"], since=r["since"]) + for r in await docker_svc.ranked_containers(db, now, _crash_loop_deaths())] async def _swarm(db, now: datetime): diff --git a/plugins/docker/routes.py b/plugins/docker/routes.py index 264cc82..df12571 100644 --- a/plugins/docker/routes.py +++ b/plugins/docker/routes.py @@ -63,7 +63,6 @@ def _sparkline(values: list[float], width: int = 80, height: int = 20) -> str: @require_role(UserRole.viewer) async def index(): poll_interval = current_app.config.get("MONITORS_POLL_INTERVAL", 60) - current_range = request.args.get("range", DEFAULT_RANGE) # Show the Swarm link only when a manager has actually reported topology, so # non-swarm installs aren't offered an always-empty page. async with current_app.db_sessionmaker() as db: @@ -74,7 +73,6 @@ async def index(): return await render_template( "docker/index.html", poll_interval=poll_interval, - current_range=current_range, has_swarm=has_swarm, has_disk=has_disk, ) @@ -83,11 +81,14 @@ async def index(): @docker_bp.get("/rows") @require_role(UserRole.viewer) async def rows(): - """HTMX fragment: swarm services (collapsed, cluster-complete) on top, then - non-swarm containers grouped by host with resource sparklines.""" - since, range_key = parse_range(request.args.get("range")) + """HTMX fragment: what needs attention first, then swarm services + (collapsed, cluster-complete), then non-swarm containers grouped by host.""" + now = datetime.now(timezone.utc) + stale_after = docker_svc.stale_after_seconds(current_app.config) async with current_app.db_sessionmaker() as db: + ranked = await docker_svc.ranked_containers( + db, now, docker_svc.crash_loop_deaths(current_app.config)) containers = await docker_svc.list_containers(db) services = list((await db.execute(select(DockerSwarmService))).scalars()) nodes = list((await db.execute(select(DockerSwarmNode))).scalars()) @@ -118,13 +119,11 @@ async def rows(): "host_name": host.name if host else c.host_id, "containers": [], "running": 0, "stopped": 0, } - cpu_hist, mem_hist = await docker_svc.container_history(db, c.host_id, c.name, since) g["containers"].append({ "container": c, "ports": json.loads(c.ports_json) if c.ports_json else [], "uptime": _uptime(c.started_at) if c.status == "running" else None, - "sparkline_cpu": _sparkline(cpu_hist), - "sparkline_mem": _sparkline(mem_hist), + "stale": _is_stale(c, now, stale_after), }) if c.status == "running": g["running"] += 1 @@ -152,17 +151,28 @@ async def rows(): swarm_services = swarm_view["services"] standalone_running = sum(g["running"] for g in host_groups) ghost_total = sum(r["count"] for s in swarm_services for r in s["replicas"] if r["ghost"]) + attention = [dict(r, stale=_is_stale(r["container"], now, stale_after)) + for r in docker_svc.needs_attention(ranked)] return await render_template( "docker/rows.html", host_groups=host_groups, swarm_services=swarm_services, + attention=attention, says=docker_svc.PROBLEM_SAYS, severity=docker_svc.PROBLEM_SEVERITY, + unhealthy=sum("unhealthy" in r["problems"] for r in attention), + restarting=sum(bool({"restarting", "crash_looping"} & set(r["problems"])) for r in attention), + stale=sum(1 for g in host_groups for i in g["containers"] if i["stale"]), running=standalone_running + sum(s["running"] for s in swarm_services), stopped=len(standalone) - standalone_running, total=len(containers) + ghost_total, - range_key=range_key, ) +def _is_stale(c, now: datetime, stale_after: int) -> bool: + """A container whose last report is older than the agent's staleness + window: what the page shows may no longer be true.""" + return c.scraped_at is None or (now - c.scraped_at).total_seconds() > stale_after + + @docker_bp.get("/widget") @require_role(UserRole.viewer) async def widget(): diff --git a/plugins/docker/service.py b/plugins/docker/service.py index 531e294..82ed4dc 100644 --- a/plugins/docker/service.py +++ b/plugins/docker/service.py @@ -6,14 +6,18 @@ Quart's pipeline) call the same code. """ from __future__ import annotations -from datetime import datetime +from datetime import datetime, timedelta from sqlalchemy import func, select from steward.core.time_range import bucket_seconds from steward.models.hosts import Host -from .models import DockerContainer, DockerEvent, DockerLog, DockerMetric, DockerMetricHourly +from .dedup import dedup_by_container_id +from .models import ( + DockerContainer, DockerEvent, DockerLog, DockerMetric, DockerMetricHourly, DockerSwarmService, +) +from .swarm_view import replaced_tasks CRASH_LOOP_DEATHS = 3 # default; Settings → Thresholds sets it @@ -278,3 +282,94 @@ def prune_extra_vars(prune_target: str) -> dict: if prune_target == "images": extra_vars["prune_all_images"] = True return extra_vars + + +# ── what is wrong with a container ─────────────────────────────────────────── +# Shared by the Docker page's "Needs attention" and the MCP tools, so both +# rank the same containers the same way. + +PROBLEMS = ("restarting", "crash_looping", "unhealthy", "oom_killed", "exited_nonzero") + +def container_problems(c, failures_1h: int, failed_24h: bool, + crash_loop_deaths: int = CRASH_LOOP_DEATHS) -> list[str]: + """What is wrong with one container, most severe first; [] when nothing is.""" + status = (c.status or "").lower() + out = [] + if status == "restarting": + out.append("restarting") + if failures_1h >= crash_loop_deaths: + out.append("crash_looping") + if status == "running" and (c.health or "") == "unhealthy": + out.append("unhealthy") + if c.oom_killed: + out.append("oom_killed") + if status in ("exited", "dead") and (c.exit_code or 0) != 0 and failed_24h: + out.append("exited_nonzero") + return out + + +def problem_since(problems: list[str], deaths: list, hour_ago: datetime, + unhealthy_at: datetime | None = None) -> datetime | None: + """When a container's worst problem began; None when the retained events + cannot say. `deaths` are its die/oom events of the last 24h, newest first. + + A restart or crash loop dates from its first death in the last hour, an + OOM kill or error exit from the death that ended it, and a failing + healthcheck from when its health last turned unhealthy. + """ + if not problems: + return None + worst = problems[0] + if worst in ("restarting", "crash_looping"): + in_hour = [e.at for e in deaths if e.at >= hour_ago] + return min(in_hour) if in_hour else (deaths[0].at if deaths else None) + if worst == "unhealthy": + return unhealthy_at + if worst == "oom_killed": + ooms = [e.at for e in deaths if e.event == "oom"] + return ooms[0] if ooms else (deaths[0].at if deaths else None) + return deaths[0].at if deaths else None + + +PROBLEM_SEVERITY = {"restarting": "critical", "crash_looping": "critical", "unhealthy": "critical", + "oom_killed": "warning", "exited_nonzero": "warning"} +PROBLEM_SAYS = {"restarting": "is restarting", "crash_looping": "keeps dying", + "unhealthy": "fails its healthcheck", "oom_killed": "was OOM-killed", + "exited_nonzero": "exited with an error"} + + + +async def ranked_containers(db, now: datetime, crash_loop_deaths: int) -> list[dict]: + """Every container (deduplicated) with what is wrong with it, most severe + first: {container, problems, since, replaced}. A swarm task Swarm has + replaced has no problems; its failure belongs to its service.""" + containers = dedup_by_container_id(await list_containers(db)) + services = list((await db.execute(select(DockerSwarmService))).scalars()) + replaced = replaced_tasks(containers, services) + failures = await recent_failures_by_container(db, now - timedelta(hours=24)) + hour_ago = now - timedelta(hours=1) + sick = await unhealthy_since( + db, [(c.host_id, c.name) for c in containers if (c.health or "") == "unhealthy"]) + out = [] + for c in containers: + key = (c.host_id, c.name) + gone = key in replaced + events = failures.get(key, []) + recent = sum(1 for e in events if e.at >= hour_ago) + problems = [] if gone else container_problems(c, recent, bool(events), crash_loop_deaths) + out.append({"container": c, "problems": problems, "replaced": gone, + "since": problem_since(problems, events, hour_ago, sick.get(key))}) + return out + + +def needs_attention(ranked: list[dict]) -> list[dict]: + """The containers with something wrong, worst problem first, then by host + and name.""" + return sorted((r for r in ranked if r["problems"]), + key=lambda r: (PROBLEMS.index(r["problems"][0]), r["container"].host_id, r["container"].name)) + + +def stale_after_seconds(config) -> int: + """How old a container's last report may be before the page calls it + stale: the host agent's own staleness window, since the agent reports it.""" + return int(((config.get("PLUGINS") or {}).get("host_agent") or {}).get("stale_after_seconds", 180)) diff --git a/plugins/docker/templates/docker/index.html b/plugins/docker/templates/docker/index.html index 950dd06..62dfce9 100644 --- a/plugins/docker/templates/docker/index.html +++ b/plugins/docker/templates/docker/index.html @@ -1,24 +1,16 @@ {% extends "base.html" %} +{% from "_macros.html" import page_header %} {% block title %}Docker — Steward{% endblock %} {% block content %} -
Loading…