feat(docker): needs attention first, stale rows say so, no sparklines on the overview (#5123)

The Docker page opens with a "Needs attention" card: every container
with something wrong, worst problem first — restarting, crash-looping,
unhealthy, OOM-killed, exited non-zero — each with what it does ("is
restarting and keeps dying · exit 1 · since 8m ago"), linked to its page.
It is ranked by the same code as the MCP tools: container_problems,
problem_since and the per-container ranking move from mcp_tools into
service.py (ranked_containers, needs_attention), and mcp_tools' _rows and
findings now read them from there.

Counts follow as stat tiles: needs attention, unhealthy, restarting (or
crash-looping), running, stopped, stale, services, hosts.

A container whose last report is older than the host agent's staleness
window (stale_after_seconds, 180 s by default) carries a Stale badge with
when it last reported, instead of a live-looking green dot.

The per-container CPU and memory history columns are gone from the
overview, and with them one history query per container per refresh; the
container's own page keeps its charts. With nothing left to range over,
the time-range control goes too, and the page now refreshes on the poll
interval. CPU and memory are graded by Settings → Thresholds rather than
50/80 and 70/90 written into the template.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
2026-10-06 18:03:37 -04:00
co-authored by Claude Opus 5.5
parent 2a3af60e99
commit 6cf04f57e1
6 changed files with 253 additions and 136 deletions
+97 -2
View File
@@ -6,14 +6,18 @@ Quart's pipeline) call the same code.
"""
from __future__ import annotations
from datetime import datetime
from datetime import datetime, timedelta
from sqlalchemy import func, select
from steward.core.time_range import bucket_seconds
from steward.models.hosts import Host
from .models import DockerContainer, DockerEvent, DockerLog, DockerMetric, DockerMetricHourly
from .dedup import dedup_by_container_id
from .models import (
DockerContainer, DockerEvent, DockerLog, DockerMetric, DockerMetricHourly, DockerSwarmService,
)
from .swarm_view import replaced_tasks
CRASH_LOOP_DEATHS = 3 # default; Settings → Thresholds sets it
@@ -278,3 +282,94 @@ def prune_extra_vars(prune_target: str) -> dict:
if prune_target == "images":
extra_vars["prune_all_images"] = True
return extra_vars
# ── what is wrong with a container ───────────────────────────────────────────
# Shared by the Docker page's "Needs attention" and the MCP tools, so both
# rank the same containers the same way.
PROBLEMS = ("restarting", "crash_looping", "unhealthy", "oom_killed", "exited_nonzero")
def container_problems(c, failures_1h: int, failed_24h: bool,
crash_loop_deaths: int = CRASH_LOOP_DEATHS) -> list[str]:
"""What is wrong with one container, most severe first; [] when nothing is."""
status = (c.status or "").lower()
out = []
if status == "restarting":
out.append("restarting")
if failures_1h >= crash_loop_deaths:
out.append("crash_looping")
if status == "running" and (c.health or "") == "unhealthy":
out.append("unhealthy")
if c.oom_killed:
out.append("oom_killed")
if status in ("exited", "dead") and (c.exit_code or 0) != 0 and failed_24h:
out.append("exited_nonzero")
return out
def problem_since(problems: list[str], deaths: list, hour_ago: datetime,
unhealthy_at: datetime | None = None) -> datetime | None:
"""When a container's worst problem began; None when the retained events
cannot say. `deaths` are its die/oom events of the last 24h, newest first.
A restart or crash loop dates from its first death in the last hour, an
OOM kill or error exit from the death that ended it, and a failing
healthcheck from when its health last turned unhealthy.
"""
if not problems:
return None
worst = problems[0]
if worst in ("restarting", "crash_looping"):
in_hour = [e.at for e in deaths if e.at >= hour_ago]
return min(in_hour) if in_hour else (deaths[0].at if deaths else None)
if worst == "unhealthy":
return unhealthy_at
if worst == "oom_killed":
ooms = [e.at for e in deaths if e.event == "oom"]
return ooms[0] if ooms else (deaths[0].at if deaths else None)
return deaths[0].at if deaths else None
PROBLEM_SEVERITY = {"restarting": "critical", "crash_looping": "critical", "unhealthy": "critical",
"oom_killed": "warning", "exited_nonzero": "warning"}
PROBLEM_SAYS = {"restarting": "is restarting", "crash_looping": "keeps dying",
"unhealthy": "fails its healthcheck", "oom_killed": "was OOM-killed",
"exited_nonzero": "exited with an error"}
async def ranked_containers(db, now: datetime, crash_loop_deaths: int) -> list[dict]:
"""Every container (deduplicated) with what is wrong with it, most severe
first: {container, problems, since, replaced}. A swarm task Swarm has
replaced has no problems; its failure belongs to its service."""
containers = dedup_by_container_id(await list_containers(db))
services = list((await db.execute(select(DockerSwarmService))).scalars())
replaced = replaced_tasks(containers, services)
failures = await recent_failures_by_container(db, now - timedelta(hours=24))
hour_ago = now - timedelta(hours=1)
sick = await unhealthy_since(
db, [(c.host_id, c.name) for c in containers if (c.health or "") == "unhealthy"])
out = []
for c in containers:
key = (c.host_id, c.name)
gone = key in replaced
events = failures.get(key, [])
recent = sum(1 for e in events if e.at >= hour_ago)
problems = [] if gone else container_problems(c, recent, bool(events), crash_loop_deaths)
out.append({"container": c, "problems": problems, "replaced": gone,
"since": problem_since(problems, events, hour_ago, sick.get(key))})
return out
def needs_attention(ranked: list[dict]) -> list[dict]:
"""The containers with something wrong, worst problem first, then by host
and name."""
return sorted((r for r in ranked if r["problems"]),
key=lambda r: (PROBLEMS.index(r["problems"][0]), r["container"].host_id, r["container"].name))
def stale_after_seconds(config) -> int:
"""How old a container's last report may be before the page calls it
stale: the host agent's own staleness window, since the agent reports it."""
return int(((config.get("PLUGINS") or {}).get("host_agent") or {}).get("stale_after_seconds", 180))