"""Docker queries shared by the plugin's pages and its MCP tools. Each function takes an open session and returns rows or plain data, with no Quart request context, so the routes and the /mcp tools (which run outside Quart's pipeline) call the same code. """ from __future__ import annotations from datetime import datetime, timedelta from sqlalchemy import func, select from steward.core.time_range import bucket_seconds from steward.models.hosts import Host from .dedup import dedup_by_container_id from .models import ( DockerContainer, DockerEvent, DockerLog, DockerMetric, DockerMetricHourly, DockerSwarmService, ) from .swarm_view import replaced_tasks CRASH_LOOP_DEATHS = 3 # default; Settings → Thresholds sets it def crash_loop_deaths(config) -> int: """Deaths within an hour that make a container or a swarm service a crash loop, from Settings → Thresholds (app.config["THRESHOLDS"]).""" th = (config.get("THRESHOLDS") or {}).get("crash_loop") or {} return int(th.get("warn") or CRASH_LOOP_DEATHS) def failed_24h(dead_names, swarm_services) -> int: """Failures in the last 24h for the dashboard: distinct plain containers that died or were OOM-killed, plus each swarm service's failed tasks. A swarm task's container dies whenever Swarm replaces it (a rolling update kills it with 137), so its die event is not a failure. Services with Swarm's task records count their failed tasks instead; a service without them (an agent older than 1.8.0) falls back to its tasks' die events. """ by_prefix = {s["name"] + ".": s for s in swarm_services} plain, fallback = 0, set() for name in dead_names: svc = next((s for p, s in by_prefix.items() if name.startswith(p)), None) if svc is None: plain += 1 elif not svc["tasks_known"]: fallback.add(name) tasks = sum(s["failures_24h"] or 0 for s in swarm_services if s["tasks_known"]) return plain + len(fallback) + tasks def bucket(values: list[float], target: int = 40) -> list[float]: """Bucket-average a series down to ~target points (DB-agnostic, in Python). Replaces the old SQL strftime() bucketing, which was SQLite-only and broke on Postgres. Agent cadence is dense, so a multi-hour window is hundreds of rows — averaging into a readable point count keeps the sparkline's shape. """ n = len(values) if n <= target: return values size = (n + target - 1) // target out: list[float] = [] for i in range(0, n, size): chunk = values[i:i + size] out.append(sum(chunk) / len(chunk)) return out async def container_history(db, host_id: str, name: str, since) -> tuple[list, list]: """Return (cpu_series, mem_series) sparkline-ready for one host's container.""" rows = (await db.execute( select(DockerMetric.cpu_pct, DockerMetric.mem_pct) .where(DockerMetric.host_id == host_id) .where(DockerMetric.container_name == name) .where(DockerMetric.scraped_at >= since) .order_by(DockerMetric.scraped_at) )).all() cpu = bucket([r.cpu_pct or 0.0 for r in rows]) mem = bucket([r.mem_pct or 0.0 for r in rows]) return cpu, mem async def host_map(db, host_ids: set[str]) -> dict[str, Host]: if not host_ids: return {} return { h.id: h for h in (await db.execute( select(Host).where(Host.id.in_(host_ids)))).scalars().all() } def escape_like(text: str) -> str: """Make `text` match literally inside a LIKE/ILIKE pattern (escape char \\).""" return text.replace("\\", "\\\\").replace("%", "\\%").replace("_", "\\_") async def list_containers(db, host_id: str | None = None) -> list[DockerContainer]: """Containers, running first then by name; one host's when host_id is given.""" stmt = select(DockerContainer) if host_id is not None: stmt = stmt.where(DockerContainer.host_id == host_id) stmt = stmt.order_by((DockerContainer.status == "running").desc(), DockerContainer.name) return list((await db.execute(stmt)).scalars()) async def container_events(db, host_id: str, name: str, limit: int = 50) -> list[DockerEvent]: """A container's lifecycle events (start/stop/die/oom/health), newest first.""" return list((await db.execute( select(DockerEvent) .where(DockerEvent.host_id == host_id) .where(DockerEvent.container_name == name) .order_by(DockerEvent.at.desc()) .limit(limit) )).scalars()) async def query_logs(db, host_id: str, name: str, stream: str, query: str, limit: int) -> list: """The recent retained lines for one container, newest-first, optionally filtered by stream and a case-insensitive substring. Newest-first is deliberate: the viewer replaces the list on each poll, which resets scroll to the top — so the latest lines stay visible without any scroll handling, and older lines are a scroll away. """ stmt = (select(DockerLog.ts, DockerLog.stream, DockerLog.line) .where(DockerLog.host_id == host_id) .where(DockerLog.container_name == name)) if stream in ("stdout", "stderr"): stmt = stmt.where(DockerLog.stream == stream) if query: stmt = stmt.where(DockerLog.line.ilike(f"%{escape_like(query)}%", escape="\\")) stmt = stmt.order_by(DockerLog.ts.desc(), DockerLog.id.desc()).limit(limit) return (await db.execute(stmt)).all() CONTAINER_SERIES = ("cpu_pct", "mem_pct", "mem_usage_bytes") async def container_points(db, host_id: str, name: str, metric: str, since, target: int = 120) -> list[tuple]: """(time, average) points of one container metric since `since`, oldest first. Raw samples are bucket-averaged in SQL; the part of the range older than the oldest raw sample (already rolled up and pruned by retention) comes from the hourly table. """ if metric not in CONTAINER_SERIES: raise ValueError(f"metric must be one of {', '.join(CONTAINER_SERIES)}") raw_col = getattr(DockerMetric, metric) width_s = bucket_seconds(since, target) rbucket = func.date_bin( func.make_interval(0, 0, 0, 0, 0, 0, width_s), DockerMetric.scraped_at, func.to_timestamp(0), ).label("bucket") raw = (await db.execute( select(rbucket, func.avg(raw_col)) .where(DockerMetric.host_id == host_id, DockerMetric.container_name == name, DockerMetric.scraped_at >= since) .group_by(rbucket).order_by(rbucket) )).all() oldest_raw = raw[0][0] if raw else None hourly_stmt = ( select(DockerMetricHourly.bucket, getattr(DockerMetricHourly, metric)) .where(DockerMetricHourly.host_id == host_id, DockerMetricHourly.container_name == name, DockerMetricHourly.bucket >= since) .order_by(DockerMetricHourly.bucket) ) if oldest_raw is not None: hourly_stmt = hourly_stmt.where(DockerMetricHourly.bucket < oldest_raw) hourly = (await db.execute(hourly_stmt)).all() return [(t, float(v)) for t, v in hourly] + [(t, float(v)) for t, v in raw] async def search_logs(db, *, since, host_id: str | None = None, name: str | None = None, text: str = "", stream: str = "", before=None, limit: int = 50) -> list: """Retained log lines matching a filter, newest first (limit + 1 fetched by callers that need to know whether more exist).""" stmt = (select(DockerLog.id, DockerLog.host_id, DockerLog.container_name, DockerLog.ts, DockerLog.stream, DockerLog.line) .where(DockerLog.ts >= since)) if host_id: stmt = stmt.where(DockerLog.host_id == host_id) if name: stmt = stmt.where(DockerLog.container_name == name) if stream in ("stdout", "stderr"): stmt = stmt.where(DockerLog.stream == stream) if text: stmt = stmt.where(DockerLog.line.ilike(f"%{escape_like(text)}%", escape="\\")) if before is not None: stmt = stmt.where(DockerLog.ts < before) stmt = stmt.order_by(DockerLog.ts.desc(), DockerLog.id.desc()).limit(limit) return list((await db.execute(stmt)).all()) async def log_context(db, host_id: str, name: str, ts, n: int) -> tuple[list, list]: """Up to n lines either side of `ts` in one container's log, each oldest first.""" cols = (DockerLog.id, DockerLog.ts, DockerLog.stream, DockerLog.line) base = select(*cols).where(DockerLog.host_id == host_id, DockerLog.container_name == name) before = (await db.execute( base.where(DockerLog.ts < ts).order_by(DockerLog.ts.desc(), DockerLog.id.desc()).limit(n) )).all() after = (await db.execute( base.where(DockerLog.ts > ts).order_by(DockerLog.ts, DockerLog.id).limit(n) )).all() return list(reversed(before)), list(after) async def recent_failures_by_container(db, since) -> dict[tuple[str, str], list]: """{(host_id, container_name): [die/oom events since `since`, newest first]}.""" rows = (await db.execute( select(DockerEvent) .where(DockerEvent.event.in_(("die", "oom")), DockerEvent.at >= since) .order_by(DockerEvent.at.desc()) )).scalars() out: dict[tuple[str, str], list] = {} for e in rows: out.setdefault((e.host_id, e.container_name), []).append(e) return out async def retained_names(db, want: str, *, exact: bool, host_id: str | None = None, limit: int = 20) -> list[tuple[str, str, datetime]]: """(host_id, container_name, last seen) of containers Steward still holds logs or events for, newest first — including ones Docker has removed. exact: the name itself, from both logs and events. Otherwise names that contain `want` (case-insensitive), from events only: a substring match over every retained log line's name would scan the log table, and a container Steward watched die has a die or stop event. """ seen: dict[tuple[str, str], datetime] = {} sources = ((DockerLog, DockerLog.ts), (DockerEvent, DockerEvent.at)) if exact \ else ((DockerEvent, DockerEvent.at),) for model, at in sources: match = (model.container_name == want if exact else model.container_name.ilike(f"%{escape_like(want)}%", escape="\\")) stmt = (select(model.host_id, model.container_name, func.max(at)) .where(match).group_by(model.host_id, model.container_name)) if host_id: stmt = stmt.where(model.host_id == host_id) for hid, name, last in (await db.execute(stmt)).all(): key = (hid, name) if key not in seen or last > seen[key]: seen[key] = last rows = sorted(((h, n, t) for (h, n), t in seen.items()), key=lambda r: r[2], reverse=True) return rows[:limit] async def unhealthy_since(db, keys) -> dict[tuple[str, str], datetime]: """{(host_id, container_name): when it last turned unhealthy} for the given containers, from their health_change events; absent when none is retained.""" keys = set(keys) if not keys: return {} rows = (await db.execute( select(DockerEvent) .where(DockerEvent.event == "health_change", DockerEvent.detail.like("%→unhealthy"), DockerEvent.container_name.in_({name for _, name in keys})) .order_by(DockerEvent.at.desc()) )).scalars() out: dict[tuple[str, str], datetime] = {} for e in rows: key = (e.host_id, e.container_name) if key in keys and key not in out: out[key] = e.at return out PRUNE_TARGETS = ("containers", "images", "system") def prune_extra_vars(prune_target: str) -> dict: """Extra-vars for the bundled docker_prune playbook. "Prune unused images" means ALL unused (docker image prune -a), not just dangling; the full system prune stays conservative (-f, no -a) per the operator's choice (m78).""" extra_vars: dict = {"prune_target": prune_target} if prune_target == "images": extra_vars["prune_all_images"] = True return extra_vars # ── what is wrong with a container ─────────────────────────────────────────── # Shared by the Docker page's "Needs attention" and the MCP tools, so both # rank the same containers the same way. PROBLEMS = ("restarting", "crash_looping", "unhealthy", "oom_killed", "exited_nonzero") def container_problems(c, failures_1h: int, failed_24h: bool, crash_loop_deaths: int = CRASH_LOOP_DEATHS) -> list[str]: """What is wrong with one container, most severe first; [] when nothing is.""" status = (c.status or "").lower() out = [] if status == "restarting": out.append("restarting") if failures_1h >= crash_loop_deaths: out.append("crash_looping") if status == "running" and (c.health or "") == "unhealthy": out.append("unhealthy") if c.oom_killed: out.append("oom_killed") if status in ("exited", "dead") and (c.exit_code or 0) != 0 and failed_24h: out.append("exited_nonzero") return out def problem_since(problems: list[str], deaths: list, hour_ago: datetime, unhealthy_at: datetime | None = None) -> datetime | None: """When a container's worst problem began; None when the retained events cannot say. `deaths` are its die/oom events of the last 24h, newest first. A restart or crash loop dates from its first death in the last hour, an OOM kill or error exit from the death that ended it, and a failing healthcheck from when its health last turned unhealthy. """ if not problems: return None worst = problems[0] if worst in ("restarting", "crash_looping"): in_hour = [e.at for e in deaths if e.at >= hour_ago] return min(in_hour) if in_hour else (deaths[0].at if deaths else None) if worst == "unhealthy": return unhealthy_at if worst == "oom_killed": ooms = [e.at for e in deaths if e.event == "oom"] return ooms[0] if ooms else (deaths[0].at if deaths else None) return deaths[0].at if deaths else None PROBLEM_SEVERITY = {"restarting": "critical", "crash_looping": "critical", "unhealthy": "critical", "oom_killed": "warning", "exited_nonzero": "warning"} PROBLEM_SAYS = {"restarting": "is restarting", "crash_looping": "keeps dying", "unhealthy": "fails its healthcheck", "oom_killed": "was OOM-killed", "exited_nonzero": "exited with an error"} async def ranked_containers(db, now: datetime, crash_loop_deaths: int) -> list[dict]: """Every container (deduplicated) with what is wrong with it, most severe first: {container, problems, since, replaced}. A swarm task Swarm has replaced has no problems; its failure belongs to its service.""" containers = dedup_by_container_id(await list_containers(db)) services = list((await db.execute(select(DockerSwarmService))).scalars()) replaced = replaced_tasks(containers, services) failures = await recent_failures_by_container(db, now - timedelta(hours=24)) hour_ago = now - timedelta(hours=1) sick = await unhealthy_since( db, [(c.host_id, c.name) for c in containers if (c.health or "") == "unhealthy"]) out = [] for c in containers: key = (c.host_id, c.name) gone = key in replaced events = failures.get(key, []) recent = sum(1 for e in events if e.at >= hour_ago) problems = [] if gone else container_problems(c, recent, bool(events), crash_loop_deaths) out.append({"container": c, "problems": problems, "replaced": gone, "since": problem_since(problems, events, hour_ago, sick.get(key))}) return out def needs_attention(ranked: list[dict]) -> list[dict]: """The containers with something wrong, worst problem first, then by host and name.""" return sorted((r for r in ranked if r["problems"]), key=lambda r: (PROBLEMS.index(r["problems"][0]), r["container"].host_id, r["container"].name)) def stale_after_seconds(config) -> int: """How old a container's last report may be before the page calls it stale: the host agent's own staleness window, since the agent reports it.""" return int(((config.get("PLUGINS") or {}).get("host_agent") or {}).get("stale_after_seconds", 180))