feat(docker): needs attention first, stale rows say so, no sparklines on the overview (#5123)
The Docker page opens with a "Needs attention" card: every container
with something wrong, worst problem first — restarting, crash-looping,
unhealthy, OOM-killed, exited non-zero — each with what it does ("is
restarting and keeps dying · exit 1 · since 8m ago"), linked to its page.
It is ranked by the same code as the MCP tools: container_problems,
problem_since and the per-container ranking move from mcp_tools into
service.py (ranked_containers, needs_attention), and mcp_tools' _rows and
findings now read them from there.
Counts follow as stat tiles: needs attention, unhealthy, restarting (or
crash-looping), running, stopped, stale, services, hosts.
A container whose last report is older than the host agent's staleness
window (stale_after_seconds, 180 s by default) carries a Stale badge with
when it last reported, instead of a live-looking green dot.
The per-container CPU and memory history columns are gone from the
overview, and with them one history query per container per refresh; the
container's own page keeps its charts. With nothing left to range over,
the time-range control goes too, and the page now refreshes on the poll
interval. CPU and memory are graded by Settings → Thresholds rather than
50/80 and 70/90 written into the template.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -25,58 +25,17 @@ from steward.models.hosts import Host
|
||||
from steward.models.users import UserRole
|
||||
|
||||
from . import service as docker_svc
|
||||
from .service import PROBLEM_SAYS, PROBLEM_SEVERITY, PROBLEMS, container_problems, problem_since
|
||||
from .dedup import dedup_by_container_id
|
||||
from .models import DockerDiskUsage, DockerSwarmNode, DockerSwarmService
|
||||
from .swarm_view import build_swarm_services, replaced_tasks, task_time
|
||||
|
||||
PROBLEMS = ("restarting", "crash_looping", "unhealthy", "oom_killed", "exited_nonzero")
|
||||
CRASH_LOOP_DEATHS = docker_svc.CRASH_LOOP_DEATHS
|
||||
LOG_LINE_CHARS = 2000
|
||||
|
||||
|
||||
# ── pure helpers ─────────────────────────────────────────────────────────────
|
||||
|
||||
def container_problems(c, failures_1h: int, failed_24h: bool,
|
||||
crash_loop_deaths: int = CRASH_LOOP_DEATHS) -> list[str]:
|
||||
"""What is wrong with one container, most severe first; [] when nothing is."""
|
||||
status = (c.status or "").lower()
|
||||
out = []
|
||||
if status == "restarting":
|
||||
out.append("restarting")
|
||||
if failures_1h >= crash_loop_deaths:
|
||||
out.append("crash_looping")
|
||||
if status == "running" and (c.health or "") == "unhealthy":
|
||||
out.append("unhealthy")
|
||||
if c.oom_killed:
|
||||
out.append("oom_killed")
|
||||
if status in ("exited", "dead") and (c.exit_code or 0) != 0 and failed_24h:
|
||||
out.append("exited_nonzero")
|
||||
return out
|
||||
|
||||
|
||||
def problem_since(problems: list[str], deaths: list, hour_ago: datetime,
|
||||
unhealthy_at: datetime | None = None) -> datetime | None:
|
||||
"""When a container's worst problem began; None when the retained events
|
||||
cannot say. `deaths` are its die/oom events of the last 24h, newest first.
|
||||
|
||||
A restart or crash loop dates from its first death in the last hour, an
|
||||
OOM kill or error exit from the death that ended it, and a failing
|
||||
healthcheck from when its health last turned unhealthy.
|
||||
"""
|
||||
if not problems:
|
||||
return None
|
||||
worst = problems[0]
|
||||
if worst in ("restarting", "crash_looping"):
|
||||
in_hour = [e.at for e in deaths if e.at >= hour_ago]
|
||||
return min(in_hour) if in_hour else (deaths[0].at if deaths else None)
|
||||
if worst == "unhealthy":
|
||||
return unhealthy_at
|
||||
if worst == "oom_killed":
|
||||
ooms = [e.at for e in deaths if e.event == "oom"]
|
||||
return ooms[0] if ooms else (deaths[0].at if deaths else None)
|
||||
return deaths[0].at if deaths else None
|
||||
|
||||
|
||||
def cut_middle(text: str, limit: int = LOG_LINE_CHARS) -> str:
|
||||
"""Keep both ends of an over-long line and say how much of the middle went."""
|
||||
if text is None or len(text) <= limit:
|
||||
@@ -117,13 +76,6 @@ def container_row(c, host_names: dict, problems: list[str], now: datetime,
|
||||
}
|
||||
|
||||
|
||||
_SEVERITY = {"restarting": "critical", "crash_looping": "critical", "unhealthy": "critical",
|
||||
"oom_killed": "warning", "exited_nonzero": "warning"}
|
||||
_SAYS = {"restarting": "is restarting", "crash_looping": "keeps dying",
|
||||
"unhealthy": "fails its healthcheck", "oom_killed": "was OOM-killed",
|
||||
"exited_nonzero": "exited with an error"}
|
||||
|
||||
|
||||
def _from_iso(text: str | None) -> datetime | None:
|
||||
return datetime.fromisoformat(text.replace("Z", "+00:00")) if text else None
|
||||
|
||||
@@ -137,8 +89,8 @@ def container_findings(rows: list[dict]) -> list[dict]:
|
||||
where = f" on {r['host_name']}" if r["host_name"] else ""
|
||||
detail = f" (exit {r['exit_code']})" if r["exit_code"] not in (None, 0) else ""
|
||||
out.append(finding(
|
||||
_SEVERITY[worst], f"container_{worst}", r["container_name"],
|
||||
f"container {r['container_name']}{where} {' and '.join(_SAYS[p] for p in r['problems'])}{detail}",
|
||||
PROBLEM_SEVERITY[worst], f"container_{worst}", r["container_name"],
|
||||
f"container {r['container_name']}{where} {' and '.join(PROBLEM_SAYS[p] for p in r['problems'])}{detail}",
|
||||
since=_from_iso(r["problem_since"]),
|
||||
ids={"host_id": r["host_id"], "container_name": r["container_name"]},
|
||||
next=f'get_container(container="{r["container_name"]}", host="{r["host_id"]}")'))
|
||||
@@ -204,24 +156,10 @@ async def _replaced(db, containers) -> set[tuple]:
|
||||
async def _rows(db, now: datetime) -> list[dict]:
|
||||
"""Every container with its problems. A swarm task Swarm replaced has none:
|
||||
its failure, if it was one, belongs to its service (service_findings)."""
|
||||
containers = dedup_by_container_id(await docker_svc.list_containers(db))
|
||||
replaced = await _replaced(db, containers)
|
||||
failures = await docker_svc.recent_failures_by_container(db, now - timedelta(hours=24))
|
||||
hour_ago = now - timedelta(hours=1)
|
||||
host_names = await _host_names(db)
|
||||
deaths = _crash_loop_deaths()
|
||||
sick = await docker_svc.unhealthy_since(
|
||||
db, [(c.host_id, c.name) for c in containers if (c.health or "") == "unhealthy"])
|
||||
rows = []
|
||||
for c in containers:
|
||||
key = (c.host_id, c.name)
|
||||
gone = key in replaced
|
||||
events = failures.get(key, [])
|
||||
recent = sum(1 for e in events if e.at >= hour_ago)
|
||||
problems = [] if gone else container_problems(c, recent, bool(events), deaths)
|
||||
since = problem_since(problems, events, hour_ago, sick.get(key))
|
||||
rows.append(container_row(c, host_names, problems, now, replaced=gone, since=since))
|
||||
return rows
|
||||
return [container_row(r["container"], host_names, r["problems"], now,
|
||||
replaced=r["replaced"], since=r["since"])
|
||||
for r in await docker_svc.ranked_containers(db, now, _crash_loop_deaths())]
|
||||
|
||||
|
||||
async def _swarm(db, now: datetime):
|
||||
|
||||
Reference in New Issue
Block a user