bvandeusenandClaude Opus 5.5
6cf04f57e1
The Docker page opens with a "Needs attention" card: every container
with something wrong, worst problem first — restarting, crash-looping,
unhealthy, OOM-killed, exited non-zero — each with what it does ("is
restarting and keeps dying · exit 1 · since 8m ago"), linked to its page.
It is ranked by the same code as the MCP tools: container_problems,
problem_since and the per-container ranking move from mcp_tools into
service.py (ranked_containers, needs_attention), and mcp_tools' _rows and
findings now read them from there.
Counts follow as stat tiles: needs attention, unhealthy, restarting (or
crash-looping), running, stopped, stale, services, hosts.
A container whose last report is older than the host agent's staleness
window (stale_after_seconds, 180 s by default) carries a Stale badge with
when it last reported, instead of a live-looking green dot.
The per-container CPU and memory history columns are gone from the
overview, and with them one history query per container per refresh; the
container's own page keeps its charts. With nothing left to range over,
the time-range control goes too, and the page now refreshes on the poll
interval. CPU and memory are graded by Settings → Thresholds rather than
50/80 and 70/90 written into the template.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
376 lines
16 KiB
Python
376 lines
16 KiB
Python
"""Docker queries shared by the plugin's pages and its MCP tools.
|
|
|
|
Each function takes an open session and returns rows or plain data, with no
|
|
Quart request context, so the routes and the /mcp tools (which run outside
|
|
Quart's pipeline) call the same code.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
from datetime import datetime, timedelta
|
|
|
|
from sqlalchemy import func, select
|
|
|
|
from steward.core.time_range import bucket_seconds
|
|
from steward.models.hosts import Host
|
|
|
|
from .dedup import dedup_by_container_id
|
|
from .models import (
|
|
DockerContainer, DockerEvent, DockerLog, DockerMetric, DockerMetricHourly, DockerSwarmService,
|
|
)
|
|
from .swarm_view import replaced_tasks
|
|
|
|
|
|
CRASH_LOOP_DEATHS = 3 # default; Settings → Thresholds sets it
|
|
|
|
|
|
def crash_loop_deaths(config) -> int:
|
|
"""Deaths within an hour that make a container or a swarm service a crash
|
|
loop, from Settings → Thresholds (app.config["THRESHOLDS"])."""
|
|
th = (config.get("THRESHOLDS") or {}).get("crash_loop") or {}
|
|
return int(th.get("warn") or CRASH_LOOP_DEATHS)
|
|
|
|
|
|
def failed_24h(dead_names, swarm_services) -> int:
|
|
"""Failures in the last 24h for the dashboard: distinct plain containers
|
|
that died or were OOM-killed, plus each swarm service's failed tasks.
|
|
|
|
A swarm task's container dies whenever Swarm replaces it (a rolling update
|
|
kills it with 137), so its die event is not a failure. Services with
|
|
Swarm's task records count their failed tasks instead; a service without
|
|
them (an agent older than 1.8.0) falls back to its tasks' die events.
|
|
"""
|
|
by_prefix = {s["name"] + ".": s for s in swarm_services}
|
|
plain, fallback = 0, set()
|
|
for name in dead_names:
|
|
svc = next((s for p, s in by_prefix.items() if name.startswith(p)), None)
|
|
if svc is None:
|
|
plain += 1
|
|
elif not svc["tasks_known"]:
|
|
fallback.add(name)
|
|
tasks = sum(s["failures_24h"] or 0 for s in swarm_services if s["tasks_known"])
|
|
return plain + len(fallback) + tasks
|
|
|
|
|
|
def bucket(values: list[float], target: int = 40) -> list[float]:
|
|
"""Bucket-average a series down to ~target points (DB-agnostic, in Python).
|
|
|
|
Replaces the old SQL strftime() bucketing, which was SQLite-only and broke
|
|
on Postgres. Agent cadence is dense, so a multi-hour window is hundreds of
|
|
rows — averaging into a readable point count keeps the sparkline's shape.
|
|
"""
|
|
n = len(values)
|
|
if n <= target:
|
|
return values
|
|
size = (n + target - 1) // target
|
|
out: list[float] = []
|
|
for i in range(0, n, size):
|
|
chunk = values[i:i + size]
|
|
out.append(sum(chunk) / len(chunk))
|
|
return out
|
|
|
|
|
|
async def container_history(db, host_id: str, name: str, since) -> tuple[list, list]:
|
|
"""Return (cpu_series, mem_series) sparkline-ready for one host's container."""
|
|
rows = (await db.execute(
|
|
select(DockerMetric.cpu_pct, DockerMetric.mem_pct)
|
|
.where(DockerMetric.host_id == host_id)
|
|
.where(DockerMetric.container_name == name)
|
|
.where(DockerMetric.scraped_at >= since)
|
|
.order_by(DockerMetric.scraped_at)
|
|
)).all()
|
|
cpu = bucket([r.cpu_pct or 0.0 for r in rows])
|
|
mem = bucket([r.mem_pct or 0.0 for r in rows])
|
|
return cpu, mem
|
|
|
|
|
|
async def host_map(db, host_ids: set[str]) -> dict[str, Host]:
|
|
if not host_ids:
|
|
return {}
|
|
return {
|
|
h.id: h for h in (await db.execute(
|
|
select(Host).where(Host.id.in_(host_ids)))).scalars().all()
|
|
}
|
|
|
|
|
|
def escape_like(text: str) -> str:
|
|
"""Make `text` match literally inside a LIKE/ILIKE pattern (escape char \\)."""
|
|
return text.replace("\\", "\\\\").replace("%", "\\%").replace("_", "\\_")
|
|
|
|
|
|
async def list_containers(db, host_id: str | None = None) -> list[DockerContainer]:
|
|
"""Containers, running first then by name; one host's when host_id is given."""
|
|
stmt = select(DockerContainer)
|
|
if host_id is not None:
|
|
stmt = stmt.where(DockerContainer.host_id == host_id)
|
|
stmt = stmt.order_by((DockerContainer.status == "running").desc(), DockerContainer.name)
|
|
return list((await db.execute(stmt)).scalars())
|
|
|
|
|
|
async def container_events(db, host_id: str, name: str, limit: int = 50) -> list[DockerEvent]:
|
|
"""A container's lifecycle events (start/stop/die/oom/health), newest first."""
|
|
return list((await db.execute(
|
|
select(DockerEvent)
|
|
.where(DockerEvent.host_id == host_id)
|
|
.where(DockerEvent.container_name == name)
|
|
.order_by(DockerEvent.at.desc())
|
|
.limit(limit)
|
|
)).scalars())
|
|
|
|
|
|
async def query_logs(db, host_id: str, name: str, stream: str, query: str,
|
|
limit: int) -> list:
|
|
"""The recent retained lines for one container, newest-first, optionally
|
|
filtered by stream and a case-insensitive substring.
|
|
|
|
Newest-first is deliberate: the viewer replaces the list on each poll, which
|
|
resets scroll to the top — so the latest lines stay visible without any
|
|
scroll handling, and older lines are a scroll away.
|
|
"""
|
|
stmt = (select(DockerLog.ts, DockerLog.stream, DockerLog.line)
|
|
.where(DockerLog.host_id == host_id)
|
|
.where(DockerLog.container_name == name))
|
|
if stream in ("stdout", "stderr"):
|
|
stmt = stmt.where(DockerLog.stream == stream)
|
|
if query:
|
|
stmt = stmt.where(DockerLog.line.ilike(f"%{escape_like(query)}%", escape="\\"))
|
|
stmt = stmt.order_by(DockerLog.ts.desc(), DockerLog.id.desc()).limit(limit)
|
|
return (await db.execute(stmt)).all()
|
|
|
|
|
|
CONTAINER_SERIES = ("cpu_pct", "mem_pct", "mem_usage_bytes")
|
|
|
|
|
|
async def container_points(db, host_id: str, name: str, metric: str, since,
|
|
target: int = 120) -> list[tuple]:
|
|
"""(time, average) points of one container metric since `since`, oldest first.
|
|
|
|
Raw samples are bucket-averaged in SQL; the part of the range older than
|
|
the oldest raw sample (already rolled up and pruned by retention) comes
|
|
from the hourly table.
|
|
"""
|
|
if metric not in CONTAINER_SERIES:
|
|
raise ValueError(f"metric must be one of {', '.join(CONTAINER_SERIES)}")
|
|
raw_col = getattr(DockerMetric, metric)
|
|
width_s = bucket_seconds(since, target)
|
|
rbucket = func.date_bin(
|
|
func.make_interval(0, 0, 0, 0, 0, 0, width_s), DockerMetric.scraped_at, func.to_timestamp(0),
|
|
).label("bucket")
|
|
raw = (await db.execute(
|
|
select(rbucket, func.avg(raw_col))
|
|
.where(DockerMetric.host_id == host_id, DockerMetric.container_name == name,
|
|
DockerMetric.scraped_at >= since)
|
|
.group_by(rbucket).order_by(rbucket)
|
|
)).all()
|
|
oldest_raw = raw[0][0] if raw else None
|
|
hourly_stmt = (
|
|
select(DockerMetricHourly.bucket, getattr(DockerMetricHourly, metric))
|
|
.where(DockerMetricHourly.host_id == host_id,
|
|
DockerMetricHourly.container_name == name,
|
|
DockerMetricHourly.bucket >= since)
|
|
.order_by(DockerMetricHourly.bucket)
|
|
)
|
|
if oldest_raw is not None:
|
|
hourly_stmt = hourly_stmt.where(DockerMetricHourly.bucket < oldest_raw)
|
|
hourly = (await db.execute(hourly_stmt)).all()
|
|
return [(t, float(v)) for t, v in hourly] + [(t, float(v)) for t, v in raw]
|
|
|
|
|
|
async def search_logs(db, *, since, host_id: str | None = None, name: str | None = None,
|
|
text: str = "", stream: str = "", before=None, limit: int = 50) -> list:
|
|
"""Retained log lines matching a filter, newest first (limit + 1 fetched by callers
|
|
that need to know whether more exist)."""
|
|
stmt = (select(DockerLog.id, DockerLog.host_id, DockerLog.container_name,
|
|
DockerLog.ts, DockerLog.stream, DockerLog.line)
|
|
.where(DockerLog.ts >= since))
|
|
if host_id:
|
|
stmt = stmt.where(DockerLog.host_id == host_id)
|
|
if name:
|
|
stmt = stmt.where(DockerLog.container_name == name)
|
|
if stream in ("stdout", "stderr"):
|
|
stmt = stmt.where(DockerLog.stream == stream)
|
|
if text:
|
|
stmt = stmt.where(DockerLog.line.ilike(f"%{escape_like(text)}%", escape="\\"))
|
|
if before is not None:
|
|
stmt = stmt.where(DockerLog.ts < before)
|
|
stmt = stmt.order_by(DockerLog.ts.desc(), DockerLog.id.desc()).limit(limit)
|
|
return list((await db.execute(stmt)).all())
|
|
|
|
|
|
async def log_context(db, host_id: str, name: str, ts, n: int) -> tuple[list, list]:
|
|
"""Up to n lines either side of `ts` in one container's log, each oldest first."""
|
|
cols = (DockerLog.id, DockerLog.ts, DockerLog.stream, DockerLog.line)
|
|
base = select(*cols).where(DockerLog.host_id == host_id, DockerLog.container_name == name)
|
|
before = (await db.execute(
|
|
base.where(DockerLog.ts < ts).order_by(DockerLog.ts.desc(), DockerLog.id.desc()).limit(n)
|
|
)).all()
|
|
after = (await db.execute(
|
|
base.where(DockerLog.ts > ts).order_by(DockerLog.ts, DockerLog.id).limit(n)
|
|
)).all()
|
|
return list(reversed(before)), list(after)
|
|
|
|
|
|
async def recent_failures_by_container(db, since) -> dict[tuple[str, str], list]:
|
|
"""{(host_id, container_name): [die/oom events since `since`, newest first]}."""
|
|
rows = (await db.execute(
|
|
select(DockerEvent)
|
|
.where(DockerEvent.event.in_(("die", "oom")), DockerEvent.at >= since)
|
|
.order_by(DockerEvent.at.desc())
|
|
)).scalars()
|
|
out: dict[tuple[str, str], list] = {}
|
|
for e in rows:
|
|
out.setdefault((e.host_id, e.container_name), []).append(e)
|
|
return out
|
|
|
|
|
|
async def retained_names(db, want: str, *, exact: bool, host_id: str | None = None,
|
|
limit: int = 20) -> list[tuple[str, str, datetime]]:
|
|
"""(host_id, container_name, last seen) of containers Steward still holds
|
|
logs or events for, newest first — including ones Docker has removed.
|
|
|
|
exact: the name itself, from both logs and events. Otherwise names that
|
|
contain `want` (case-insensitive), from events only: a substring match
|
|
over every retained log line's name would scan the log table, and a
|
|
container Steward watched die has a die or stop event.
|
|
"""
|
|
seen: dict[tuple[str, str], datetime] = {}
|
|
sources = ((DockerLog, DockerLog.ts), (DockerEvent, DockerEvent.at)) if exact \
|
|
else ((DockerEvent, DockerEvent.at),)
|
|
for model, at in sources:
|
|
match = (model.container_name == want if exact
|
|
else model.container_name.ilike(f"%{escape_like(want)}%", escape="\\"))
|
|
stmt = (select(model.host_id, model.container_name, func.max(at))
|
|
.where(match).group_by(model.host_id, model.container_name))
|
|
if host_id:
|
|
stmt = stmt.where(model.host_id == host_id)
|
|
for hid, name, last in (await db.execute(stmt)).all():
|
|
key = (hid, name)
|
|
if key not in seen or last > seen[key]:
|
|
seen[key] = last
|
|
rows = sorted(((h, n, t) for (h, n), t in seen.items()), key=lambda r: r[2], reverse=True)
|
|
return rows[:limit]
|
|
|
|
|
|
async def unhealthy_since(db, keys) -> dict[tuple[str, str], datetime]:
|
|
"""{(host_id, container_name): when it last turned unhealthy} for the given
|
|
containers, from their health_change events; absent when none is retained."""
|
|
keys = set(keys)
|
|
if not keys:
|
|
return {}
|
|
rows = (await db.execute(
|
|
select(DockerEvent)
|
|
.where(DockerEvent.event == "health_change",
|
|
DockerEvent.detail.like("%→unhealthy"),
|
|
DockerEvent.container_name.in_({name for _, name in keys}))
|
|
.order_by(DockerEvent.at.desc())
|
|
)).scalars()
|
|
out: dict[tuple[str, str], datetime] = {}
|
|
for e in rows:
|
|
key = (e.host_id, e.container_name)
|
|
if key in keys and key not in out:
|
|
out[key] = e.at
|
|
return out
|
|
|
|
|
|
PRUNE_TARGETS = ("containers", "images", "system")
|
|
|
|
|
|
def prune_extra_vars(prune_target: str) -> dict:
|
|
"""Extra-vars for the bundled docker_prune playbook. "Prune unused images"
|
|
means ALL unused (docker image prune -a), not just dangling; the full system
|
|
prune stays conservative (-f, no -a) per the operator's choice (m78)."""
|
|
extra_vars: dict = {"prune_target": prune_target}
|
|
if prune_target == "images":
|
|
extra_vars["prune_all_images"] = True
|
|
return extra_vars
|
|
|
|
|
|
# ── what is wrong with a container ───────────────────────────────────────────
|
|
# Shared by the Docker page's "Needs attention" and the MCP tools, so both
|
|
# rank the same containers the same way.
|
|
|
|
PROBLEMS = ("restarting", "crash_looping", "unhealthy", "oom_killed", "exited_nonzero")
|
|
|
|
def container_problems(c, failures_1h: int, failed_24h: bool,
|
|
crash_loop_deaths: int = CRASH_LOOP_DEATHS) -> list[str]:
|
|
"""What is wrong with one container, most severe first; [] when nothing is."""
|
|
status = (c.status or "").lower()
|
|
out = []
|
|
if status == "restarting":
|
|
out.append("restarting")
|
|
if failures_1h >= crash_loop_deaths:
|
|
out.append("crash_looping")
|
|
if status == "running" and (c.health or "") == "unhealthy":
|
|
out.append("unhealthy")
|
|
if c.oom_killed:
|
|
out.append("oom_killed")
|
|
if status in ("exited", "dead") and (c.exit_code or 0) != 0 and failed_24h:
|
|
out.append("exited_nonzero")
|
|
return out
|
|
|
|
|
|
def problem_since(problems: list[str], deaths: list, hour_ago: datetime,
|
|
unhealthy_at: datetime | None = None) -> datetime | None:
|
|
"""When a container's worst problem began; None when the retained events
|
|
cannot say. `deaths` are its die/oom events of the last 24h, newest first.
|
|
|
|
A restart or crash loop dates from its first death in the last hour, an
|
|
OOM kill or error exit from the death that ended it, and a failing
|
|
healthcheck from when its health last turned unhealthy.
|
|
"""
|
|
if not problems:
|
|
return None
|
|
worst = problems[0]
|
|
if worst in ("restarting", "crash_looping"):
|
|
in_hour = [e.at for e in deaths if e.at >= hour_ago]
|
|
return min(in_hour) if in_hour else (deaths[0].at if deaths else None)
|
|
if worst == "unhealthy":
|
|
return unhealthy_at
|
|
if worst == "oom_killed":
|
|
ooms = [e.at for e in deaths if e.event == "oom"]
|
|
return ooms[0] if ooms else (deaths[0].at if deaths else None)
|
|
return deaths[0].at if deaths else None
|
|
|
|
|
|
PROBLEM_SEVERITY = {"restarting": "critical", "crash_looping": "critical", "unhealthy": "critical",
|
|
"oom_killed": "warning", "exited_nonzero": "warning"}
|
|
PROBLEM_SAYS = {"restarting": "is restarting", "crash_looping": "keeps dying",
|
|
"unhealthy": "fails its healthcheck", "oom_killed": "was OOM-killed",
|
|
"exited_nonzero": "exited with an error"}
|
|
|
|
|
|
|
|
async def ranked_containers(db, now: datetime, crash_loop_deaths: int) -> list[dict]:
|
|
"""Every container (deduplicated) with what is wrong with it, most severe
|
|
first: {container, problems, since, replaced}. A swarm task Swarm has
|
|
replaced has no problems; its failure belongs to its service."""
|
|
containers = dedup_by_container_id(await list_containers(db))
|
|
services = list((await db.execute(select(DockerSwarmService))).scalars())
|
|
replaced = replaced_tasks(containers, services)
|
|
failures = await recent_failures_by_container(db, now - timedelta(hours=24))
|
|
hour_ago = now - timedelta(hours=1)
|
|
sick = await unhealthy_since(
|
|
db, [(c.host_id, c.name) for c in containers if (c.health or "") == "unhealthy"])
|
|
out = []
|
|
for c in containers:
|
|
key = (c.host_id, c.name)
|
|
gone = key in replaced
|
|
events = failures.get(key, [])
|
|
recent = sum(1 for e in events if e.at >= hour_ago)
|
|
problems = [] if gone else container_problems(c, recent, bool(events), crash_loop_deaths)
|
|
out.append({"container": c, "problems": problems, "replaced": gone,
|
|
"since": problem_since(problems, events, hour_ago, sick.get(key))})
|
|
return out
|
|
|
|
|
|
def needs_attention(ranked: list[dict]) -> list[dict]:
|
|
"""The containers with something wrong, worst problem first, then by host
|
|
and name."""
|
|
return sorted((r for r in ranked if r["problems"]),
|
|
key=lambda r: (PROBLEMS.index(r["problems"][0]), r["container"].host_id, r["container"].name))
|
|
|
|
|
|
def stale_after_seconds(config) -> int:
|
|
"""How old a container's last report may be before the page calls it
|
|
stale: the host agent's own staleness window, since the agent reports it."""
|
|
return int(((config.get("PLUGINS") or {}).get("host_agent") or {}).get("stale_after_seconds", 180))
|