feat(docker): needs attention first, stale rows say so, no sparklines on the overview (#5123)
The Docker page opens with a "Needs attention" card: every container
with something wrong, worst problem first — restarting, crash-looping,
unhealthy, OOM-killed, exited non-zero — each with what it does ("is
restarting and keeps dying · exit 1 · since 8m ago"), linked to its page.
It is ranked by the same code as the MCP tools: container_problems,
problem_since and the per-container ranking move from mcp_tools into
service.py (ranked_containers, needs_attention), and mcp_tools' _rows and
findings now read them from there.
Counts follow as stat tiles: needs attention, unhealthy, restarting (or
crash-looping), running, stopped, stale, services, hosts.
A container whose last report is older than the host agent's staleness
window (stale_after_seconds, 180 s by default) carries a Stale badge with
when it last reported, instead of a live-looking green dot.
The per-container CPU and memory history columns are gone from the
overview, and with them one history query per container per refresh; the
container's own page keeps its charts. With nothing left to range over,
the time-range control goes too, and the page now refreshes on the poll
interval. CPU and memory are graded by Settings → Thresholds rather than
50/80 and 70/90 written into the template.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -25,58 +25,17 @@ from steward.models.hosts import Host
|
|||||||
from steward.models.users import UserRole
|
from steward.models.users import UserRole
|
||||||
|
|
||||||
from . import service as docker_svc
|
from . import service as docker_svc
|
||||||
|
from .service import PROBLEM_SAYS, PROBLEM_SEVERITY, PROBLEMS, container_problems, problem_since
|
||||||
from .dedup import dedup_by_container_id
|
from .dedup import dedup_by_container_id
|
||||||
from .models import DockerDiskUsage, DockerSwarmNode, DockerSwarmService
|
from .models import DockerDiskUsage, DockerSwarmNode, DockerSwarmService
|
||||||
from .swarm_view import build_swarm_services, replaced_tasks, task_time
|
from .swarm_view import build_swarm_services, replaced_tasks, task_time
|
||||||
|
|
||||||
PROBLEMS = ("restarting", "crash_looping", "unhealthy", "oom_killed", "exited_nonzero")
|
|
||||||
CRASH_LOOP_DEATHS = docker_svc.CRASH_LOOP_DEATHS
|
CRASH_LOOP_DEATHS = docker_svc.CRASH_LOOP_DEATHS
|
||||||
LOG_LINE_CHARS = 2000
|
LOG_LINE_CHARS = 2000
|
||||||
|
|
||||||
|
|
||||||
# ── pure helpers ─────────────────────────────────────────────────────────────
|
# ── pure helpers ─────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
def container_problems(c, failures_1h: int, failed_24h: bool,
|
|
||||||
crash_loop_deaths: int = CRASH_LOOP_DEATHS) -> list[str]:
|
|
||||||
"""What is wrong with one container, most severe first; [] when nothing is."""
|
|
||||||
status = (c.status or "").lower()
|
|
||||||
out = []
|
|
||||||
if status == "restarting":
|
|
||||||
out.append("restarting")
|
|
||||||
if failures_1h >= crash_loop_deaths:
|
|
||||||
out.append("crash_looping")
|
|
||||||
if status == "running" and (c.health or "") == "unhealthy":
|
|
||||||
out.append("unhealthy")
|
|
||||||
if c.oom_killed:
|
|
||||||
out.append("oom_killed")
|
|
||||||
if status in ("exited", "dead") and (c.exit_code or 0) != 0 and failed_24h:
|
|
||||||
out.append("exited_nonzero")
|
|
||||||
return out
|
|
||||||
|
|
||||||
|
|
||||||
def problem_since(problems: list[str], deaths: list, hour_ago: datetime,
|
|
||||||
unhealthy_at: datetime | None = None) -> datetime | None:
|
|
||||||
"""When a container's worst problem began; None when the retained events
|
|
||||||
cannot say. `deaths` are its die/oom events of the last 24h, newest first.
|
|
||||||
|
|
||||||
A restart or crash loop dates from its first death in the last hour, an
|
|
||||||
OOM kill or error exit from the death that ended it, and a failing
|
|
||||||
healthcheck from when its health last turned unhealthy.
|
|
||||||
"""
|
|
||||||
if not problems:
|
|
||||||
return None
|
|
||||||
worst = problems[0]
|
|
||||||
if worst in ("restarting", "crash_looping"):
|
|
||||||
in_hour = [e.at for e in deaths if e.at >= hour_ago]
|
|
||||||
return min(in_hour) if in_hour else (deaths[0].at if deaths else None)
|
|
||||||
if worst == "unhealthy":
|
|
||||||
return unhealthy_at
|
|
||||||
if worst == "oom_killed":
|
|
||||||
ooms = [e.at for e in deaths if e.event == "oom"]
|
|
||||||
return ooms[0] if ooms else (deaths[0].at if deaths else None)
|
|
||||||
return deaths[0].at if deaths else None
|
|
||||||
|
|
||||||
|
|
||||||
def cut_middle(text: str, limit: int = LOG_LINE_CHARS) -> str:
|
def cut_middle(text: str, limit: int = LOG_LINE_CHARS) -> str:
|
||||||
"""Keep both ends of an over-long line and say how much of the middle went."""
|
"""Keep both ends of an over-long line and say how much of the middle went."""
|
||||||
if text is None or len(text) <= limit:
|
if text is None or len(text) <= limit:
|
||||||
@@ -117,13 +76,6 @@ def container_row(c, host_names: dict, problems: list[str], now: datetime,
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
_SEVERITY = {"restarting": "critical", "crash_looping": "critical", "unhealthy": "critical",
|
|
||||||
"oom_killed": "warning", "exited_nonzero": "warning"}
|
|
||||||
_SAYS = {"restarting": "is restarting", "crash_looping": "keeps dying",
|
|
||||||
"unhealthy": "fails its healthcheck", "oom_killed": "was OOM-killed",
|
|
||||||
"exited_nonzero": "exited with an error"}
|
|
||||||
|
|
||||||
|
|
||||||
def _from_iso(text: str | None) -> datetime | None:
|
def _from_iso(text: str | None) -> datetime | None:
|
||||||
return datetime.fromisoformat(text.replace("Z", "+00:00")) if text else None
|
return datetime.fromisoformat(text.replace("Z", "+00:00")) if text else None
|
||||||
|
|
||||||
@@ -137,8 +89,8 @@ def container_findings(rows: list[dict]) -> list[dict]:
|
|||||||
where = f" on {r['host_name']}" if r["host_name"] else ""
|
where = f" on {r['host_name']}" if r["host_name"] else ""
|
||||||
detail = f" (exit {r['exit_code']})" if r["exit_code"] not in (None, 0) else ""
|
detail = f" (exit {r['exit_code']})" if r["exit_code"] not in (None, 0) else ""
|
||||||
out.append(finding(
|
out.append(finding(
|
||||||
_SEVERITY[worst], f"container_{worst}", r["container_name"],
|
PROBLEM_SEVERITY[worst], f"container_{worst}", r["container_name"],
|
||||||
f"container {r['container_name']}{where} {' and '.join(_SAYS[p] for p in r['problems'])}{detail}",
|
f"container {r['container_name']}{where} {' and '.join(PROBLEM_SAYS[p] for p in r['problems'])}{detail}",
|
||||||
since=_from_iso(r["problem_since"]),
|
since=_from_iso(r["problem_since"]),
|
||||||
ids={"host_id": r["host_id"], "container_name": r["container_name"]},
|
ids={"host_id": r["host_id"], "container_name": r["container_name"]},
|
||||||
next=f'get_container(container="{r["container_name"]}", host="{r["host_id"]}")'))
|
next=f'get_container(container="{r["container_name"]}", host="{r["host_id"]}")'))
|
||||||
@@ -204,24 +156,10 @@ async def _replaced(db, containers) -> set[tuple]:
|
|||||||
async def _rows(db, now: datetime) -> list[dict]:
|
async def _rows(db, now: datetime) -> list[dict]:
|
||||||
"""Every container with its problems. A swarm task Swarm replaced has none:
|
"""Every container with its problems. A swarm task Swarm replaced has none:
|
||||||
its failure, if it was one, belongs to its service (service_findings)."""
|
its failure, if it was one, belongs to its service (service_findings)."""
|
||||||
containers = dedup_by_container_id(await docker_svc.list_containers(db))
|
|
||||||
replaced = await _replaced(db, containers)
|
|
||||||
failures = await docker_svc.recent_failures_by_container(db, now - timedelta(hours=24))
|
|
||||||
hour_ago = now - timedelta(hours=1)
|
|
||||||
host_names = await _host_names(db)
|
host_names = await _host_names(db)
|
||||||
deaths = _crash_loop_deaths()
|
return [container_row(r["container"], host_names, r["problems"], now,
|
||||||
sick = await docker_svc.unhealthy_since(
|
replaced=r["replaced"], since=r["since"])
|
||||||
db, [(c.host_id, c.name) for c in containers if (c.health or "") == "unhealthy"])
|
for r in await docker_svc.ranked_containers(db, now, _crash_loop_deaths())]
|
||||||
rows = []
|
|
||||||
for c in containers:
|
|
||||||
key = (c.host_id, c.name)
|
|
||||||
gone = key in replaced
|
|
||||||
events = failures.get(key, [])
|
|
||||||
recent = sum(1 for e in events if e.at >= hour_ago)
|
|
||||||
problems = [] if gone else container_problems(c, recent, bool(events), deaths)
|
|
||||||
since = problem_since(problems, events, hour_ago, sick.get(key))
|
|
||||||
rows.append(container_row(c, host_names, problems, now, replaced=gone, since=since))
|
|
||||||
return rows
|
|
||||||
|
|
||||||
|
|
||||||
async def _swarm(db, now: datetime):
|
async def _swarm(db, now: datetime):
|
||||||
|
|||||||
@@ -63,7 +63,6 @@ def _sparkline(values: list[float], width: int = 80, height: int = 20) -> str:
|
|||||||
@require_role(UserRole.viewer)
|
@require_role(UserRole.viewer)
|
||||||
async def index():
|
async def index():
|
||||||
poll_interval = current_app.config.get("MONITORS_POLL_INTERVAL", 60)
|
poll_interval = current_app.config.get("MONITORS_POLL_INTERVAL", 60)
|
||||||
current_range = request.args.get("range", DEFAULT_RANGE)
|
|
||||||
# Show the Swarm link only when a manager has actually reported topology, so
|
# Show the Swarm link only when a manager has actually reported topology, so
|
||||||
# non-swarm installs aren't offered an always-empty page.
|
# non-swarm installs aren't offered an always-empty page.
|
||||||
async with current_app.db_sessionmaker() as db:
|
async with current_app.db_sessionmaker() as db:
|
||||||
@@ -74,7 +73,6 @@ async def index():
|
|||||||
return await render_template(
|
return await render_template(
|
||||||
"docker/index.html",
|
"docker/index.html",
|
||||||
poll_interval=poll_interval,
|
poll_interval=poll_interval,
|
||||||
current_range=current_range,
|
|
||||||
has_swarm=has_swarm,
|
has_swarm=has_swarm,
|
||||||
has_disk=has_disk,
|
has_disk=has_disk,
|
||||||
)
|
)
|
||||||
@@ -83,11 +81,14 @@ async def index():
|
|||||||
@docker_bp.get("/rows")
|
@docker_bp.get("/rows")
|
||||||
@require_role(UserRole.viewer)
|
@require_role(UserRole.viewer)
|
||||||
async def rows():
|
async def rows():
|
||||||
"""HTMX fragment: swarm services (collapsed, cluster-complete) on top, then
|
"""HTMX fragment: what needs attention first, then swarm services
|
||||||
non-swarm containers grouped by host with resource sparklines."""
|
(collapsed, cluster-complete), then non-swarm containers grouped by host."""
|
||||||
since, range_key = parse_range(request.args.get("range"))
|
now = datetime.now(timezone.utc)
|
||||||
|
stale_after = docker_svc.stale_after_seconds(current_app.config)
|
||||||
|
|
||||||
async with current_app.db_sessionmaker() as db:
|
async with current_app.db_sessionmaker() as db:
|
||||||
|
ranked = await docker_svc.ranked_containers(
|
||||||
|
db, now, docker_svc.crash_loop_deaths(current_app.config))
|
||||||
containers = await docker_svc.list_containers(db)
|
containers = await docker_svc.list_containers(db)
|
||||||
services = list((await db.execute(select(DockerSwarmService))).scalars())
|
services = list((await db.execute(select(DockerSwarmService))).scalars())
|
||||||
nodes = list((await db.execute(select(DockerSwarmNode))).scalars())
|
nodes = list((await db.execute(select(DockerSwarmNode))).scalars())
|
||||||
@@ -118,13 +119,11 @@ async def rows():
|
|||||||
"host_name": host.name if host else c.host_id,
|
"host_name": host.name if host else c.host_id,
|
||||||
"containers": [], "running": 0, "stopped": 0,
|
"containers": [], "running": 0, "stopped": 0,
|
||||||
}
|
}
|
||||||
cpu_hist, mem_hist = await docker_svc.container_history(db, c.host_id, c.name, since)
|
|
||||||
g["containers"].append({
|
g["containers"].append({
|
||||||
"container": c,
|
"container": c,
|
||||||
"ports": json.loads(c.ports_json) if c.ports_json else [],
|
"ports": json.loads(c.ports_json) if c.ports_json else [],
|
||||||
"uptime": _uptime(c.started_at) if c.status == "running" else None,
|
"uptime": _uptime(c.started_at) if c.status == "running" else None,
|
||||||
"sparkline_cpu": _sparkline(cpu_hist),
|
"stale": _is_stale(c, now, stale_after),
|
||||||
"sparkline_mem": _sparkline(mem_hist),
|
|
||||||
})
|
})
|
||||||
if c.status == "running":
|
if c.status == "running":
|
||||||
g["running"] += 1
|
g["running"] += 1
|
||||||
@@ -152,17 +151,28 @@ async def rows():
|
|||||||
swarm_services = swarm_view["services"]
|
swarm_services = swarm_view["services"]
|
||||||
standalone_running = sum(g["running"] for g in host_groups)
|
standalone_running = sum(g["running"] for g in host_groups)
|
||||||
ghost_total = sum(r["count"] for s in swarm_services for r in s["replicas"] if r["ghost"])
|
ghost_total = sum(r["count"] for s in swarm_services for r in s["replicas"] if r["ghost"])
|
||||||
|
attention = [dict(r, stale=_is_stale(r["container"], now, stale_after))
|
||||||
|
for r in docker_svc.needs_attention(ranked)]
|
||||||
return await render_template(
|
return await render_template(
|
||||||
"docker/rows.html",
|
"docker/rows.html",
|
||||||
host_groups=host_groups,
|
host_groups=host_groups,
|
||||||
swarm_services=swarm_services,
|
swarm_services=swarm_services,
|
||||||
|
attention=attention, says=docker_svc.PROBLEM_SAYS, severity=docker_svc.PROBLEM_SEVERITY,
|
||||||
|
unhealthy=sum("unhealthy" in r["problems"] for r in attention),
|
||||||
|
restarting=sum(bool({"restarting", "crash_looping"} & set(r["problems"])) for r in attention),
|
||||||
|
stale=sum(1 for g in host_groups for i in g["containers"] if i["stale"]),
|
||||||
running=standalone_running + sum(s["running"] for s in swarm_services),
|
running=standalone_running + sum(s["running"] for s in swarm_services),
|
||||||
stopped=len(standalone) - standalone_running,
|
stopped=len(standalone) - standalone_running,
|
||||||
total=len(containers) + ghost_total,
|
total=len(containers) + ghost_total,
|
||||||
range_key=range_key,
|
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _is_stale(c, now: datetime, stale_after: int) -> bool:
|
||||||
|
"""A container whose last report is older than the agent's staleness
|
||||||
|
window: what the page shows may no longer be true."""
|
||||||
|
return c.scraped_at is None or (now - c.scraped_at).total_seconds() > stale_after
|
||||||
|
|
||||||
|
|
||||||
@docker_bp.get("/widget")
|
@docker_bp.get("/widget")
|
||||||
@require_role(UserRole.viewer)
|
@require_role(UserRole.viewer)
|
||||||
async def widget():
|
async def widget():
|
||||||
|
|||||||
@@ -6,14 +6,18 @@ Quart's pipeline) call the same code.
|
|||||||
"""
|
"""
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
from datetime import datetime
|
from datetime import datetime, timedelta
|
||||||
|
|
||||||
from sqlalchemy import func, select
|
from sqlalchemy import func, select
|
||||||
|
|
||||||
from steward.core.time_range import bucket_seconds
|
from steward.core.time_range import bucket_seconds
|
||||||
from steward.models.hosts import Host
|
from steward.models.hosts import Host
|
||||||
|
|
||||||
from .models import DockerContainer, DockerEvent, DockerLog, DockerMetric, DockerMetricHourly
|
from .dedup import dedup_by_container_id
|
||||||
|
from .models import (
|
||||||
|
DockerContainer, DockerEvent, DockerLog, DockerMetric, DockerMetricHourly, DockerSwarmService,
|
||||||
|
)
|
||||||
|
from .swarm_view import replaced_tasks
|
||||||
|
|
||||||
|
|
||||||
CRASH_LOOP_DEATHS = 3 # default; Settings → Thresholds sets it
|
CRASH_LOOP_DEATHS = 3 # default; Settings → Thresholds sets it
|
||||||
@@ -278,3 +282,94 @@ def prune_extra_vars(prune_target: str) -> dict:
|
|||||||
if prune_target == "images":
|
if prune_target == "images":
|
||||||
extra_vars["prune_all_images"] = True
|
extra_vars["prune_all_images"] = True
|
||||||
return extra_vars
|
return extra_vars
|
||||||
|
|
||||||
|
|
||||||
|
# ── what is wrong with a container ───────────────────────────────────────────
|
||||||
|
# Shared by the Docker page's "Needs attention" and the MCP tools, so both
|
||||||
|
# rank the same containers the same way.
|
||||||
|
|
||||||
|
PROBLEMS = ("restarting", "crash_looping", "unhealthy", "oom_killed", "exited_nonzero")
|
||||||
|
|
||||||
|
def container_problems(c, failures_1h: int, failed_24h: bool,
|
||||||
|
crash_loop_deaths: int = CRASH_LOOP_DEATHS) -> list[str]:
|
||||||
|
"""What is wrong with one container, most severe first; [] when nothing is."""
|
||||||
|
status = (c.status or "").lower()
|
||||||
|
out = []
|
||||||
|
if status == "restarting":
|
||||||
|
out.append("restarting")
|
||||||
|
if failures_1h >= crash_loop_deaths:
|
||||||
|
out.append("crash_looping")
|
||||||
|
if status == "running" and (c.health or "") == "unhealthy":
|
||||||
|
out.append("unhealthy")
|
||||||
|
if c.oom_killed:
|
||||||
|
out.append("oom_killed")
|
||||||
|
if status in ("exited", "dead") and (c.exit_code or 0) != 0 and failed_24h:
|
||||||
|
out.append("exited_nonzero")
|
||||||
|
return out
|
||||||
|
|
||||||
|
|
||||||
|
def problem_since(problems: list[str], deaths: list, hour_ago: datetime,
|
||||||
|
unhealthy_at: datetime | None = None) -> datetime | None:
|
||||||
|
"""When a container's worst problem began; None when the retained events
|
||||||
|
cannot say. `deaths` are its die/oom events of the last 24h, newest first.
|
||||||
|
|
||||||
|
A restart or crash loop dates from its first death in the last hour, an
|
||||||
|
OOM kill or error exit from the death that ended it, and a failing
|
||||||
|
healthcheck from when its health last turned unhealthy.
|
||||||
|
"""
|
||||||
|
if not problems:
|
||||||
|
return None
|
||||||
|
worst = problems[0]
|
||||||
|
if worst in ("restarting", "crash_looping"):
|
||||||
|
in_hour = [e.at for e in deaths if e.at >= hour_ago]
|
||||||
|
return min(in_hour) if in_hour else (deaths[0].at if deaths else None)
|
||||||
|
if worst == "unhealthy":
|
||||||
|
return unhealthy_at
|
||||||
|
if worst == "oom_killed":
|
||||||
|
ooms = [e.at for e in deaths if e.event == "oom"]
|
||||||
|
return ooms[0] if ooms else (deaths[0].at if deaths else None)
|
||||||
|
return deaths[0].at if deaths else None
|
||||||
|
|
||||||
|
|
||||||
|
PROBLEM_SEVERITY = {"restarting": "critical", "crash_looping": "critical", "unhealthy": "critical",
|
||||||
|
"oom_killed": "warning", "exited_nonzero": "warning"}
|
||||||
|
PROBLEM_SAYS = {"restarting": "is restarting", "crash_looping": "keeps dying",
|
||||||
|
"unhealthy": "fails its healthcheck", "oom_killed": "was OOM-killed",
|
||||||
|
"exited_nonzero": "exited with an error"}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
async def ranked_containers(db, now: datetime, crash_loop_deaths: int) -> list[dict]:
|
||||||
|
"""Every container (deduplicated) with what is wrong with it, most severe
|
||||||
|
first: {container, problems, since, replaced}. A swarm task Swarm has
|
||||||
|
replaced has no problems; its failure belongs to its service."""
|
||||||
|
containers = dedup_by_container_id(await list_containers(db))
|
||||||
|
services = list((await db.execute(select(DockerSwarmService))).scalars())
|
||||||
|
replaced = replaced_tasks(containers, services)
|
||||||
|
failures = await recent_failures_by_container(db, now - timedelta(hours=24))
|
||||||
|
hour_ago = now - timedelta(hours=1)
|
||||||
|
sick = await unhealthy_since(
|
||||||
|
db, [(c.host_id, c.name) for c in containers if (c.health or "") == "unhealthy"])
|
||||||
|
out = []
|
||||||
|
for c in containers:
|
||||||
|
key = (c.host_id, c.name)
|
||||||
|
gone = key in replaced
|
||||||
|
events = failures.get(key, [])
|
||||||
|
recent = sum(1 for e in events if e.at >= hour_ago)
|
||||||
|
problems = [] if gone else container_problems(c, recent, bool(events), crash_loop_deaths)
|
||||||
|
out.append({"container": c, "problems": problems, "replaced": gone,
|
||||||
|
"since": problem_since(problems, events, hour_ago, sick.get(key))})
|
||||||
|
return out
|
||||||
|
|
||||||
|
|
||||||
|
def needs_attention(ranked: list[dict]) -> list[dict]:
|
||||||
|
"""The containers with something wrong, worst problem first, then by host
|
||||||
|
and name."""
|
||||||
|
return sorted((r for r in ranked if r["problems"]),
|
||||||
|
key=lambda r: (PROBLEMS.index(r["problems"][0]), r["container"].host_id, r["container"].name))
|
||||||
|
|
||||||
|
|
||||||
|
def stale_after_seconds(config) -> int:
|
||||||
|
"""How old a container's last report may be before the page calls it
|
||||||
|
stale: the host agent's own staleness window, since the agent reports it."""
|
||||||
|
return int(((config.get("PLUGINS") or {}).get("host_agent") or {}).get("stale_after_seconds", 180))
|
||||||
|
|||||||
@@ -1,24 +1,16 @@
|
|||||||
{% extends "base.html" %}
|
{% extends "base.html" %}
|
||||||
|
{% from "_macros.html" import page_header %}
|
||||||
{% block title %}Docker — Steward{% endblock %}
|
{% block title %}Docker — Steward{% endblock %}
|
||||||
{% block content %}
|
{% block content %}
|
||||||
<div style="display:flex;align-items:center;justify-content:space-between;margin-bottom:1.5rem;gap:1rem;flex-wrap:wrap;">
|
{% call page_header("Docker", "What needs attention first, then every service and container.") %}
|
||||||
<div style="display:flex;align-items:baseline;gap:1rem;">
|
{% if has_swarm %}<a class="btn btn-sm btn-ghost" href="/plugins/docker/swarm">Swarm</a>{% endif %}
|
||||||
<h1 class="page-title" style="margin-bottom:0;">Docker</h1>
|
{% if has_disk %}<a class="btn btn-sm btn-ghost" href="/plugins/docker/disk">Disk</a>{% endif %}
|
||||||
{% if has_swarm %}
|
{% endcall %}
|
||||||
<a href="/plugins/docker/swarm" style="font-size:0.85rem;color:var(--accent);text-decoration:none;">Swarm →</a>
|
|
||||||
{% endif %}
|
|
||||||
{% if has_disk %}
|
|
||||||
<a href="/plugins/docker/disk" style="font-size:0.85rem;color:var(--accent);text-decoration:none;">Disk →</a>
|
|
||||||
{% endif %}
|
|
||||||
</div>
|
|
||||||
{% include "_time_range.html" %}
|
|
||||||
</div>
|
|
||||||
|
|
||||||
<div id="docker-rows"
|
<div id="docker-rows"
|
||||||
hx-get="/plugins/docker/rows"
|
hx-get="/plugins/docker/rows"
|
||||||
hx-trigger="load, rangeChange from:body"
|
hx-trigger="load, every {{ poll_interval }}s"
|
||||||
hx-include="#time-range"
|
|
||||||
hx-swap="innerHTML">
|
hx-swap="innerHTML">
|
||||||
<div style="color:var(--text-muted);font-size:0.9rem;padding:2rem;">Loading...</div>
|
<div class="card"><p class="field-hint">Loading…</p></div>
|
||||||
</div>
|
</div>
|
||||||
{% endblock %}
|
{% endblock %}
|
||||||
|
|||||||
@@ -1,29 +1,43 @@
|
|||||||
{# docker/rows.html — HTMX fragment for the Docker main page, grouped by host #}
|
{# docker/rows.html — HTMX fragment for the Docker main page, grouped by host #}
|
||||||
|
|
||||||
{# ── Summary strip ─────────────────────────────────────────────────────────── #}
|
{# docker/rows.html — HTMX fragment for the Docker page: what needs attention
|
||||||
<div style="display:grid;grid-template-columns:repeat(auto-fill,minmax(140px,1fr));gap:0.75rem;margin-bottom:1.5rem;">
|
first (service.needs_attention, the ranking the MCP tools use), the counts,
|
||||||
<div class="card" style="margin-bottom:0;">
|
then swarm services and each host's containers. #}
|
||||||
<div class="section-title" style="margin-bottom:0.35rem;">Running</div>
|
{% from "_macros.html" import stat_tile, status_badge, status_dot %}
|
||||||
<span class="stat-val" style="color:var(--green);">{{ running }}</span>
|
|
||||||
</div>
|
{# ── Needs attention ──────────────────────────────────────────────────────── #}
|
||||||
<div class="card" style="margin-bottom:0;">
|
{% if attention %}
|
||||||
<div class="section-title" style="margin-bottom:0.35rem;">Stopped</div>
|
<section class="card problem-card" aria-labelledby="docker-problems">
|
||||||
<span class="stat-val" style="{% if stopped %}color:var(--text-muted){% endif %};">{{ stopped }}</span>
|
<h2 id="docker-problems" class="section-title" style="margin-bottom:0.6rem;">Needs attention</h2>
|
||||||
</div>
|
<ul class="problem-list">
|
||||||
<div class="card" style="margin-bottom:0;">
|
{% for r in attention %}
|
||||||
<div class="section-title" style="margin-bottom:0.35rem;">Total</div>
|
{% set c = r.container %}
|
||||||
<span class="stat-val">{{ total }}</span>
|
{% set worst = r.problems[0] %}
|
||||||
</div>
|
<li>
|
||||||
{% if swarm_services %}
|
{{ status_badge('crit' if severity[worst] == 'critical' else 'warn', worst|replace('_', ' ')|capitalize) }}
|
||||||
<div class="card" style="margin-bottom:0;">
|
<a href="/plugins/docker/container/{{ c.host_id }}/{{ c.name }}">{{ c.name }}</a>
|
||||||
<div class="section-title" style="margin-bottom:0.35rem;">Services</div>
|
<span class="problem-detail">
|
||||||
<span class="stat-val">{{ swarm_services | length }}</span>
|
{%- for p in r.problems %}{{ says[p] }}{{ " and " if not loop.last }}{% endfor %}
|
||||||
</div>
|
{%- if c.exit_code and worst in ('exited_nonzero', 'crash_looping', 'restarting') %} · exit {{ c.exit_code }}{% endif %}
|
||||||
{% endif %}
|
{%- if r.since %} · since {{ r.since|ago }}{% endif %}
|
||||||
<div class="card" style="margin-bottom:0;">
|
{%- if r.stale %} · {{ status_badge('stale') }} last report {{ c.scraped_at|ago if c.scraped_at else "never" }}{% endif -%}
|
||||||
<div class="section-title" style="margin-bottom:0.35rem;">Hosts</div>
|
</span>
|
||||||
<span class="stat-val">{{ host_groups | length }}</span>
|
</li>
|
||||||
</div>
|
{% endfor %}
|
||||||
|
</ul>
|
||||||
|
</section>
|
||||||
|
{% endif %}
|
||||||
|
|
||||||
|
{# ── Counts ───────────────────────────────────────────────────────────────── #}
|
||||||
|
<div class="stat-strip">
|
||||||
|
{{ stat_tile("Needs attention", attention|length, state="crit" if attention else "inactive") }}
|
||||||
|
{{ stat_tile("Unhealthy", unhealthy, state="crit" if unhealthy else "inactive") }}
|
||||||
|
{{ stat_tile("Restarting", restarting, state="crit" if restarting else "inactive", hint="restarting or crash-looping") }}
|
||||||
|
{{ stat_tile("Running", running, state="up" if running else "inactive") }}
|
||||||
|
{{ stat_tile("Stopped", stopped, state="inactive") }}
|
||||||
|
{% if stale %}{{ stat_tile("Stale", stale, state="stale", hint="no recent report") }}{% endif %}
|
||||||
|
{% if swarm_services %}{{ stat_tile("Services", swarm_services|length, href="/plugins/docker/swarm") }}{% endif %}
|
||||||
|
{{ stat_tile("Hosts", host_groups|length) }}
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
{# ── Swarm services: collapsed per-service, each current replica labelled with
|
{# ── Swarm services: collapsed per-service, each current replica labelled with
|
||||||
@@ -43,7 +57,7 @@
|
|||||||
<span class="dot {% if s.state == 'healthy' %}dot-up{% elif s.state == 'degraded' %}dot-warn{% else %}dot-down{% endif %}"></span>
|
<span class="dot {% if s.state == 'healthy' %}dot-up{% elif s.state == 'degraded' %}dot-warn{% else %}dot-down{% endif %}"></span>
|
||||||
<span style="font-weight:600;font-size:0.9rem;">{{ s.name }}</span>
|
<span style="font-weight:600;font-size:0.9rem;">{{ s.name }}</span>
|
||||||
<span style="font-size:0.72rem;color:var(--text-dim);">{{ s.mode }}</span>
|
<span style="font-size:0.72rem;color:var(--text-dim);">{{ s.mode }}</span>
|
||||||
<span style="font-size:0.78rem;color:{% if s.state == 'healthy' %}var(--green){% elif s.state == 'degraded' %}var(--orange){% else %}var(--red){% endif %};font-variant-numeric:tabular-nums;">
|
<span class="tone-{{ 'ok' if s.state == 'healthy' else ('warn' if s.state == 'degraded' else 'crit') }}" style="font-size:0.78rem;font-variant-numeric:tabular-nums;">
|
||||||
{{ s.running }}/{{ s.desired }} running
|
{{ s.running }}/{{ s.desired }} running
|
||||||
</span>
|
</span>
|
||||||
</div>
|
</div>
|
||||||
@@ -92,35 +106,33 @@
|
|||||||
<th>Image</th>
|
<th>Image</th>
|
||||||
<th>Ports</th>
|
<th>Ports</th>
|
||||||
<th>CPU %</th>
|
<th>CPU %</th>
|
||||||
<th style="min-width:100px;">CPU history</th>
|
|
||||||
<th>Mem %</th>
|
<th>Mem %</th>
|
||||||
<th style="min-width:100px;">Mem history</th>
|
|
||||||
</tr>
|
</tr>
|
||||||
</thead>
|
</thead>
|
||||||
<tbody>
|
<tbody>
|
||||||
{% for sub in g.subgroups %}
|
{% for sub in g.subgroups %}
|
||||||
{% if g.grouped and sub.label %}
|
{% if g.grouped and sub.label %}
|
||||||
<tr><td colspan="7" style="padding:0.4rem 0.75rem 0.2rem;font-size:0.72rem;color:var(--text-dim);text-transform:uppercase;letter-spacing:0.05em;font-weight:600;">{{ sub.label }}</td></tr>
|
<tr><td colspan="5" style="padding:0.4rem 0.75rem 0.2rem;font-size:0.72rem;color:var(--text-dim);text-transform:uppercase;letter-spacing:0.05em;font-weight:600;">{{ sub.label }}</td></tr>
|
||||||
{% endif %}
|
{% endif %}
|
||||||
{% for item in sub.containers %}
|
{% for item in sub.containers %}
|
||||||
{% set c = item.container %}
|
{% set c = item.container %}
|
||||||
<tr>
|
<tr>
|
||||||
<td>
|
<td>
|
||||||
<div style="display:flex;align-items:center;gap:0.5rem;">
|
<div style="display:flex;align-items:center;gap:0.5rem;">
|
||||||
<span class="dot {% if c.status == 'running' %}dot-up{% elif c.status == 'paused' %}dot-warn{% else %}dot-down{% endif %}"></span>
|
{{ status_dot('stale' if item.stale else ('up' if c.status == 'running' else ('pending' if c.status == 'paused' else 'down')), 'Stale' if item.stale else c.status|capitalize) }}
|
||||||
<div>
|
<div>
|
||||||
<div style="font-weight:500;font-size:0.9rem;">
|
<div style="font-weight:500;font-size:0.9rem;">
|
||||||
<a href="/plugins/docker/container/{{ c.host_id }}/{{ c.name }}" style="color:inherit;text-decoration:none;border-bottom:1px dotted var(--border-mid);">{{ c.name }}</a>
|
<a href="/plugins/docker/container/{{ c.host_id }}/{{ c.name }}" style="color:inherit;text-decoration:none;border-bottom:1px dotted var(--border-mid);">{{ c.name }}</a>
|
||||||
{% if c.health == 'healthy' %}<span title="healthy" style="color:var(--green);font-size:0.7rem;">●</span>
|
{% if c.health == 'unhealthy' %} {{ status_badge('crit', 'Unhealthy') }}
|
||||||
{% elif c.health == 'unhealthy' %}<span title="unhealthy" style="color:var(--red);font-size:0.7rem;">●</span>
|
{% elif c.health == 'starting' %} {{ status_badge('pending', 'Starting') }}{% endif %}
|
||||||
{% elif c.health == 'starting' %}<span title="health: starting" style="color:var(--orange);font-size:0.7rem;">◐</span>{% endif %}
|
{% if item.stale %} {{ status_badge('stale', title="Last report " ~ (c.scraped_at.strftime('%Y-%m-%d %H:%M UTC') if c.scraped_at else 'never') ~ "; what is shown may no longer be true") }}{% endif %}
|
||||||
</div>
|
</div>
|
||||||
<div style="font-size:0.73rem;color:var(--text-muted);">
|
<div style="font-size:0.73rem;color:var(--text-muted);">
|
||||||
{{ c.status }}{% if item.uptime %} · up {{ item.uptime }}{% endif %}
|
{{ c.status }}{% if item.uptime %} · up {{ item.uptime }}{% endif %}
|
||||||
{% if c.status != 'running' and c.exit_code is not none and c.exit_code != 0 %}
|
{% if c.status != 'running' and c.exit_code is not none and c.exit_code != 0 %}
|
||||||
· <span style="color:var(--red);">exit {{ c.exit_code }}{% if c.oom_killed %} (OOM){% endif %}</span>
|
· <span class="tone-crit">exit {{ c.exit_code }}{% if c.oom_killed %} (OOM){% endif %}</span>
|
||||||
{% endif %}
|
{% endif %}
|
||||||
{% if c.restart_count %} · <span title="restart count" style="color:var(--orange);">⟳{{ c.restart_count }}</span>{% endif %}
|
{% if c.restart_count %} · <span class="tone-warn" title="Restarted {{ c.restart_count }} times">⟳{{ c.restart_count }}</span>{% endif %}
|
||||||
</div>
|
</div>
|
||||||
{% if c.service_name or c.compose_project %}
|
{% if c.service_name or c.compose_project %}
|
||||||
<div style="font-size:0.68rem;color:var(--text-dim);margin-top:0.1rem;">
|
<div style="font-size:0.68rem;color:var(--text-dim);margin-top:0.1rem;">
|
||||||
@@ -140,24 +152,18 @@
|
|||||||
</td>
|
</td>
|
||||||
<td style="font-family:ui-monospace,monospace;font-size:0.88rem;">
|
<td style="font-family:ui-monospace,monospace;font-size:0.88rem;">
|
||||||
{% if c.cpu_pct is not none %}
|
{% if c.cpu_pct is not none %}
|
||||||
<span style="color:{% if c.cpu_pct > 80 %}var(--red){% elif c.cpu_pct > 50 %}var(--orange){% else %}var(--text){% endif %};">
|
<span class="tone-{{ status_class(c.cpu_pct, 'cpu') }}">{{ "%.1f" | format(c.cpu_pct) }}%</span>
|
||||||
{{ "%.1f" | format(c.cpu_pct) }}%
|
|
||||||
</span>
|
|
||||||
{% else %}
|
{% else %}
|
||||||
<span style="color:var(--text-dim);">—</span>
|
<span style="color:var(--text-dim);">—</span>
|
||||||
{% endif %}
|
{% endif %}
|
||||||
</td>
|
</td>
|
||||||
<td>{{ item.sparkline_cpu | safe }}</td>
|
|
||||||
<td style="font-family:ui-monospace,monospace;font-size:0.88rem;">
|
<td style="font-family:ui-monospace,monospace;font-size:0.88rem;">
|
||||||
{% if c.mem_pct is not none %}
|
{% if c.mem_pct is not none %}
|
||||||
<span style="color:{% if c.mem_pct > 90 %}var(--red){% elif c.mem_pct > 70 %}var(--orange){% else %}var(--text){% endif %};">
|
<span class="tone-{{ status_class(c.mem_pct, 'mem') }}">{{ "%.1f" | format(c.mem_pct) }}%</span>
|
||||||
{{ "%.1f" | format(c.mem_pct) }}%
|
|
||||||
</span>
|
|
||||||
{% else %}
|
{% else %}
|
||||||
<span style="color:var(--text-dim);">—</span>
|
<span style="color:var(--text-dim);">—</span>
|
||||||
{% endif %}
|
{% endif %}
|
||||||
</td>
|
</td>
|
||||||
<td>{{ item.sparkline_mem | safe }}</td>
|
|
||||||
</tr>
|
</tr>
|
||||||
{% endfor %}
|
{% endfor %}
|
||||||
{% endfor %}
|
{% endfor %}
|
||||||
|
|||||||
@@ -0,0 +1,76 @@
|
|||||||
|
"""The Docker page leads with what needs attention, ranked the way the MCP
|
||||||
|
tools rank it, and marks a container whose report is old as stale (#5123)."""
|
||||||
|
import re
|
||||||
|
from datetime import datetime, timedelta, timezone
|
||||||
|
from types import SimpleNamespace as NS
|
||||||
|
|
||||||
|
|
||||||
|
from plugins.docker import service as svc
|
||||||
|
from plugins.docker.routes import _is_stale
|
||||||
|
from tests._templates import render_plugin_template
|
||||||
|
from tests.test_style_tokens import hex_colours
|
||||||
|
|
||||||
|
NOW = datetime.now(timezone.utc)
|
||||||
|
|
||||||
|
|
||||||
|
def _c(name, **kw):
|
||||||
|
base = dict(name=name, host_id="h1", image="nginx:1", status="running", health=None, exit_code=None,
|
||||||
|
oom_killed=False, restart_count=0, cpu_pct=5.0, mem_pct=10.0, compose_project=None,
|
||||||
|
service_name=None, ports_json=None, started_at=NOW - timedelta(hours=2),
|
||||||
|
scraped_at=NOW - timedelta(seconds=20))
|
||||||
|
base.update(kw)
|
||||||
|
return NS(**base)
|
||||||
|
|
||||||
|
|
||||||
|
def test_needs_attention_ranks_the_worst_problem_first():
|
||||||
|
ranked = [{"container": _c(n), "problems": p, "since": None, "replaced": False} for n, p in [
|
||||||
|
("fine", []), ("oom", ["oom_killed"]), ("sick", ["unhealthy"]),
|
||||||
|
("loop", ["crash_looping"]), ("exit", ["exited_nonzero"]), ("bounce", ["restarting", "crash_looping"])]]
|
||||||
|
assert [r["container"].name for r in svc.needs_attention(ranked)] == ["bounce", "loop", "sick", "oom", "exit"]
|
||||||
|
|
||||||
|
|
||||||
|
def test_stale_is_older_than_the_agents_window():
|
||||||
|
assert not _is_stale(_c("a"), NOW, 180)
|
||||||
|
assert _is_stale(_c("a", scraped_at=NOW - timedelta(minutes=10)), NOW, 180)
|
||||||
|
assert _is_stale(_c("a", scraped_at=None), NOW, 180)
|
||||||
|
assert svc.stale_after_seconds({"PLUGINS": {"host_agent": {"stale_after_seconds": 60}}}) == 60
|
||||||
|
assert svc.stale_after_seconds({}) == 180
|
||||||
|
|
||||||
|
|
||||||
|
async def test_a_crash_loop_comes_first_and_a_stale_row_says_so(app):
|
||||||
|
loop = _c("api", status="restarting", exit_code=1, restart_count=7)
|
||||||
|
old = _c("cache", scraped_at=NOW - timedelta(minutes=30), cpu_pct=95.0)
|
||||||
|
attention = [{"container": loop, "problems": ["restarting", "crash_looping"], "replaced": False,
|
||||||
|
"since": datetime.now(timezone.utc) - timedelta(minutes=8), "stale": False}]
|
||||||
|
groups = [{"host": None, "host_id": "h1", "host_name": "nas", "running": 1, "stopped": 1, "grouped": False,
|
||||||
|
"subgroups": [{"label": None, "containers": [
|
||||||
|
{"container": loop, "ports": [], "uptime": None, "stale": False},
|
||||||
|
{"container": old, "ports": [], "uptime": "2h", "stale": True}]}]}]
|
||||||
|
async with app.test_request_context("/plugins/docker/rows"):
|
||||||
|
page = await render_plugin_template(
|
||||||
|
app, "docker", "docker/rows.html", attention=attention, says=svc.PROBLEM_SAYS, severity=svc.PROBLEM_SEVERITY,
|
||||||
|
unhealthy=0, restarting=1, stale=1, host_groups=groups, swarm_services=[],
|
||||||
|
running=1, stopped=1, total=2)
|
||||||
|
page = re.sub(r">\s+<", "><", " ".join(page.split()))
|
||||||
|
|
||||||
|
card = page[page.index("Needs attention"):page.index("</section>")]
|
||||||
|
assert '<span class="badge badge-crit">Restarting</span>' in card
|
||||||
|
assert 'href="/plugins/docker/container/h1/api">api</a>' in card
|
||||||
|
assert "is restarting and keeps dying · exit 1 · since <time" in card and ">8m ago</time>" in card
|
||||||
|
assert "cache" not in card
|
||||||
|
assert page.index("Needs attention") < page.index("<table")
|
||||||
|
|
||||||
|
assert '<span class="stat-tile-label">Restarting</span><span class="stat-tile-value">1</span>' in page
|
||||||
|
assert '<span class="stat-tile-label">Stale</span><span class="stat-tile-value">1</span>' in page
|
||||||
|
assert 'class="badge badge-warn" title="Last report' in page # the stale row
|
||||||
|
assert '<span class="tone-crit">95.0%</span>' in page # CPU through Settings → Thresholds
|
||||||
|
assert "history" not in page.lower() and "<svg" not in page # no sparklines on the overview
|
||||||
|
assert not hex_colours(page), hex_colours(page)
|
||||||
|
|
||||||
|
|
||||||
|
async def test_a_healthy_board_has_no_attention_card(app):
|
||||||
|
async with app.test_request_context("/plugins/docker/rows"):
|
||||||
|
page = await render_plugin_template(
|
||||||
|
app, "docker", "docker/rows.html", attention=[], says=svc.PROBLEM_SAYS, severity=svc.PROBLEM_SEVERITY,
|
||||||
|
unhealthy=0, restarting=0, stale=0, host_groups=[], swarm_services=[], running=0, stopped=0, total=0)
|
||||||
|
assert "Needs attention</h2>" not in page and "No containers reported yet" in page
|
||||||
Reference in New Issue
Block a user