bvandeusenandClaude Opus 5.5
e9341c5639
#5013 "Overview findings for containers and Swarm services carry since=null" - Container problems are dated from their own evidence: a crash or restart loop from its first death in the hour, an OOM kill or error exit from the death that ended it, a failing healthcheck from its last turn to unhealthy. list_containers and get_container carry problem_since / problem_age_s. - Swarm services carry trouble_since: a crash loop's first failure in the hour, a short service's oldest troubled task in each failing slot, a recovered service's first failure in 24h. list_swarm_services shows it. - get_overview findings from the docker plugin now have since and age_s, so they rank by how long they have stood. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
625 lines
30 KiB
Python
625 lines
30 KiB
Python
"""The docker plugin's MCP tools and its get_overview findings.
|
|
|
|
Registered through register_mcp_tools() in __init__.py, so they exist only
|
|
while the plugin is enabled. Reads go through service.py, the same functions
|
|
the plugin's pages use.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
from datetime import datetime, timedelta
|
|
|
|
from sqlalchemy import select
|
|
|
|
from steward.ansible.runner import trigger_run
|
|
from steward.ansible.sources import BUILTIN_SOURCE_NAME
|
|
from steward.mcp._context import current_caller, mcp_app
|
|
from steward.mcp.tools._audit import audit_mcp
|
|
from steward.mcp.tools._shape import (
|
|
age_s, bounded, clamp_limit, db_session, iso, lookback, now_utc, pick, rnd, summarize,
|
|
)
|
|
from steward.mcp.tools.overview import finding
|
|
from steward.models.ansible_inventory import AnsibleTarget
|
|
from steward.models.hosts import Host
|
|
from steward.models.users import UserRole
|
|
|
|
from . import service as docker_svc
|
|
from .dedup import dedup_by_container_id
|
|
from .models import DockerDiskUsage, DockerSwarmNode, DockerSwarmService
|
|
from .swarm_view import build_swarm_services, replaced_tasks, task_time
|
|
|
|
PROBLEMS = ("restarting", "crash_looping", "unhealthy", "oom_killed", "exited_nonzero")
|
|
CRASH_LOOP_DEATHS = docker_svc.CRASH_LOOP_DEATHS
|
|
LOG_LINE_CHARS = 2000
|
|
|
|
|
|
# ── pure helpers ─────────────────────────────────────────────────────────────
|
|
|
|
def container_problems(c, failures_1h: int, failed_24h: bool,
|
|
crash_loop_deaths: int = CRASH_LOOP_DEATHS) -> list[str]:
|
|
"""What is wrong with one container, most severe first; [] when nothing is."""
|
|
status = (c.status or "").lower()
|
|
out = []
|
|
if status == "restarting":
|
|
out.append("restarting")
|
|
if failures_1h >= crash_loop_deaths:
|
|
out.append("crash_looping")
|
|
if status == "running" and (c.health or "") == "unhealthy":
|
|
out.append("unhealthy")
|
|
if c.oom_killed:
|
|
out.append("oom_killed")
|
|
if status in ("exited", "dead") and (c.exit_code or 0) != 0 and failed_24h:
|
|
out.append("exited_nonzero")
|
|
return out
|
|
|
|
|
|
def problem_since(problems: list[str], deaths: list, hour_ago: datetime,
|
|
unhealthy_at: datetime | None = None) -> datetime | None:
|
|
"""When a container's worst problem began; None when the retained events
|
|
cannot say. `deaths` are its die/oom events of the last 24h, newest first.
|
|
|
|
A restart or crash loop dates from its first death in the last hour, an
|
|
OOM kill or error exit from the death that ended it, and a failing
|
|
healthcheck from when its health last turned unhealthy.
|
|
"""
|
|
if not problems:
|
|
return None
|
|
worst = problems[0]
|
|
if worst in ("restarting", "crash_looping"):
|
|
in_hour = [e.at for e in deaths if e.at >= hour_ago]
|
|
return min(in_hour) if in_hour else (deaths[0].at if deaths else None)
|
|
if worst == "unhealthy":
|
|
return unhealthy_at
|
|
if worst == "oom_killed":
|
|
ooms = [e.at for e in deaths if e.event == "oom"]
|
|
return ooms[0] if ooms else (deaths[0].at if deaths else None)
|
|
return deaths[0].at if deaths else None
|
|
|
|
|
|
def cut_middle(text: str, limit: int = LOG_LINE_CHARS) -> str:
|
|
"""Keep both ends of an over-long line and say how much of the middle went."""
|
|
if text is None or len(text) <= limit:
|
|
return text
|
|
keep = limit // 2
|
|
return f"{text[:keep]} [… {len(text) - 2 * keep} characters omitted …] {text[-keep:]}"
|
|
|
|
|
|
def container_row(c, host_names: dict, problems: list[str], now: datetime,
|
|
replaced: bool = False, since: datetime | None = None) -> dict:
|
|
running = (c.status or "").lower() == "running"
|
|
return {
|
|
"container_name": c.name,
|
|
"host_id": c.host_id,
|
|
"host_name": host_names.get(c.host_id),
|
|
"image": c.image,
|
|
"status": c.status,
|
|
"health": c.health,
|
|
"problems": problems,
|
|
# When the worst problem began, from the container's retained events.
|
|
"problem_since": iso(since),
|
|
"problem_age_s": age_s(since, now),
|
|
"exit_code": c.exit_code,
|
|
"oom_killed": bool(c.oom_killed),
|
|
"restart_count": c.restart_count,
|
|
"cpu_pct": rnd(c.cpu_pct),
|
|
"mem_pct": rnd(c.mem_pct),
|
|
"mem_usage_bytes": c.mem_usage_bytes,
|
|
"mem_limit_bytes": c.mem_limit_bytes,
|
|
"started_at": iso(c.started_at),
|
|
"uptime_s": age_s(c.started_at, now) if running and c.started_at else None,
|
|
"compose_project": c.compose_project,
|
|
"swarm_service": c.service_name,
|
|
# A swarm task Swarm has replaced: history, judged on its service.
|
|
"swarm_replaced": replaced if c.service_name else None,
|
|
"reported_at": iso(c.scraped_at),
|
|
"reported_age_s": age_s(c.scraped_at, now),
|
|
}
|
|
|
|
|
|
_SEVERITY = {"restarting": "critical", "crash_looping": "critical", "unhealthy": "critical",
|
|
"oom_killed": "warning", "exited_nonzero": "warning"}
|
|
_SAYS = {"restarting": "is restarting", "crash_looping": "keeps dying",
|
|
"unhealthy": "fails its healthcheck", "oom_killed": "was OOM-killed",
|
|
"exited_nonzero": "exited with an error"}
|
|
|
|
|
|
def _from_iso(text: str | None) -> datetime | None:
|
|
return datetime.fromisoformat(text.replace("Z", "+00:00")) if text else None
|
|
|
|
|
|
def container_findings(rows: list[dict]) -> list[dict]:
|
|
out = []
|
|
for r in rows:
|
|
if not r["problems"]:
|
|
continue
|
|
worst = r["problems"][0]
|
|
where = f" on {r['host_name']}" if r["host_name"] else ""
|
|
detail = f" (exit {r['exit_code']})" if r["exit_code"] not in (None, 0) else ""
|
|
out.append(finding(
|
|
_SEVERITY[worst], f"container_{worst}", r["container_name"],
|
|
f"container {r['container_name']}{where} {' and '.join(_SAYS[p] for p in r['problems'])}{detail}",
|
|
since=_from_iso(r["problem_since"]),
|
|
ids={"host_id": r["host_id"], "container_name": r["container_name"]},
|
|
next=f'get_container(container="{r["container_name"]}", host="{r["host_id"]}")'))
|
|
return out
|
|
|
|
|
|
def task_error_text(err: dict | None) -> str:
|
|
"""Swarm's own account of a task's end, as one clause."""
|
|
if not err:
|
|
return ""
|
|
why = err.get("error") or err.get("message") or err.get("state") or "unknown"
|
|
code = f", exit {err['exit_code']}" if err.get("exit_code") not in (None, 0) else ""
|
|
where = f" on {err['node']}" if err.get("node") else ""
|
|
return f"{why}{code}{where}"
|
|
|
|
|
|
def service_findings(services: list[dict]) -> list[dict]:
|
|
"""Swarm services that are failing: crash-looping, under-replicated, or that
|
|
lost a task to an error in the last 24h. Each names Swarm's last task error."""
|
|
out = []
|
|
for s in services:
|
|
replicas = f"{s['running']} of {s['desired']} replicas running"
|
|
last = task_error_text(s.get("last_error"))
|
|
tail = f"; last task error: {last}" if last else ""
|
|
ids = {"service_name": s["name"]}
|
|
since = s.get("trouble_since")
|
|
if s.get("crash_looping"):
|
|
out.append(finding(
|
|
"critical", "swarm_service_crash_looping", s["name"],
|
|
f"swarm service {s['name']} keeps failing: {s['failures_1h']} tasks failed in the "
|
|
f"last hour, {replicas}{tail}", since=since, ids=ids,
|
|
next="list_swarm_services()"))
|
|
elif s["state"] != "healthy":
|
|
out.append(finding(
|
|
"critical" if s["state"] == "down" else "warning", f"swarm_service_{s['state']}",
|
|
s["name"], f"swarm service {s['name']} has {replicas}{tail}",
|
|
since=since, ids=ids, next="list_swarm_services()"))
|
|
elif s.get("failures_24h"):
|
|
n = s["failures_24h"]
|
|
out.append(finding(
|
|
"warning", "swarm_task_failed", s["name"],
|
|
f"swarm service {s['name']} had {n} failed task{'s' if n != 1 else ''} in the last "
|
|
f"24h and is now {replicas}{tail}", since=since, ids=ids,
|
|
next="list_swarm_services()"))
|
|
return out
|
|
|
|
|
|
# ── loading ──────────────────────────────────────────────────────────────────
|
|
|
|
async def _host_names(db) -> dict[str, str]:
|
|
return dict((await db.execute(select(Host.id, Host.name))).all())
|
|
|
|
|
|
def _crash_loop_deaths() -> int:
|
|
return docker_svc.crash_loop_deaths(mcp_app().config)
|
|
|
|
|
|
async def _replaced(db, containers) -> set[tuple]:
|
|
services = list((await db.execute(select(DockerSwarmService))).scalars())
|
|
return replaced_tasks(containers, services)
|
|
|
|
|
|
async def _rows(db, now: datetime) -> list[dict]:
|
|
"""Every container with its problems. A swarm task Swarm replaced has none:
|
|
its failure, if it was one, belongs to its service (service_findings)."""
|
|
containers = dedup_by_container_id(await docker_svc.list_containers(db))
|
|
replaced = await _replaced(db, containers)
|
|
failures = await docker_svc.recent_failures_by_container(db, now - timedelta(hours=24))
|
|
hour_ago = now - timedelta(hours=1)
|
|
host_names = await _host_names(db)
|
|
deaths = _crash_loop_deaths()
|
|
sick = await docker_svc.unhealthy_since(
|
|
db, [(c.host_id, c.name) for c in containers if (c.health or "") == "unhealthy"])
|
|
rows = []
|
|
for c in containers:
|
|
key = (c.host_id, c.name)
|
|
gone = key in replaced
|
|
events = failures.get(key, [])
|
|
recent = sum(1 for e in events if e.at >= hour_ago)
|
|
problems = [] if gone else container_problems(c, recent, bool(events), deaths)
|
|
since = problem_since(problems, events, hour_ago, sick.get(key))
|
|
rows.append(container_row(c, host_names, problems, now, replaced=gone, since=since))
|
|
return rows
|
|
|
|
|
|
async def _swarm(db, now: datetime):
|
|
containers = dedup_by_container_id(await docker_svc.list_containers(db))
|
|
services = list((await db.execute(select(DockerSwarmService))).scalars())
|
|
nodes = list((await db.execute(select(DockerSwarmNode))).scalars())
|
|
host_names = await _host_names(db)
|
|
node_hostname = {n.node_id: n.hostname for n in nodes if n.hostname}
|
|
built = build_swarm_services(containers, services, node_hostname, host_names,
|
|
now=now, crash_loop_deaths=_crash_loop_deaths())
|
|
return built["services"], nodes, host_names
|
|
|
|
|
|
async def mcp_overview(db) -> list[dict]:
|
|
"""get_overview findings: failing containers and under-replicated swarm services."""
|
|
now = now_utc()
|
|
rows = await _rows(db, now)
|
|
services, _, _ = await _swarm(db, now)
|
|
return container_findings(rows) + service_findings(services)
|
|
|
|
|
|
# ── tools ────────────────────────────────────────────────────────────────────
|
|
|
|
async def list_containers(host: str = "", status: str = "", problem: str = "",
|
|
compose_project: str = "", limit: int = 100) -> dict:
|
|
"""Docker containers across every host that runs a Steward agent with Docker.
|
|
|
|
host: a host's name or id (list_hosts); empty for all hosts.
|
|
status: Docker's state — "running", "exited", "restarting", "paused",
|
|
"created" or "dead"; empty for all.
|
|
problem: "any" for containers with something wrong, or one of
|
|
"restarting", "crash_looping" (died repeatedly within the last hour;
|
|
the count is set under Settings → Thresholds, default 3), "unhealthy"
|
|
(failing its healthcheck), "oom_killed", "exited_nonzero" (died with an
|
|
error in the last 24h); empty for no filter.
|
|
compose_project: only containers from this Compose project.
|
|
limit: default 100, max 500.
|
|
|
|
Containers with problems come first. Each row's `problems` lists what is
|
|
wrong, most severe first, and `problem_since` / `problem_age_s` say when
|
|
the worst of them began (null when Steward's retained events cannot say).
|
|
`reported_age_s` is how old the agent's last report is, so a large value
|
|
means the row may be stale.
|
|
|
|
A swarm task's container that Swarm has replaced (a rolling update, or a
|
|
retry after a failure) has `swarm_replaced: true` and no problems: Swarm
|
|
restarts a task as a new container, so failures are counted on the
|
|
service — list_swarm_services shows them.
|
|
"""
|
|
if problem and problem != "any" and problem not in PROBLEMS:
|
|
raise ValueError(f"problem {problem!r} is not 'any' or one of {', '.join(PROBLEMS)}.")
|
|
limit = clamp_limit(limit, 100, 500)
|
|
now = now_utc()
|
|
async with db_session() as db:
|
|
rows = await _rows(db, now)
|
|
if host:
|
|
hosts = list((await db.execute(select(Host))).scalars())
|
|
h = pick(host, hosts, kind="host", list_tool="list_hosts")
|
|
rows = [r for r in rows if r["host_id"] == h.id]
|
|
if status:
|
|
rows = [r for r in rows if (r["status"] or "").lower() == status.lower()]
|
|
if problem == "any":
|
|
rows = [r for r in rows if r["problems"]]
|
|
elif problem:
|
|
rows = [r for r in rows if problem in r["problems"]]
|
|
if compose_project:
|
|
rows = [r for r in rows if r["compose_project"] == compose_project]
|
|
rows.sort(key=lambda r: (not r["problems"], (r["status"] or "") != "running",
|
|
(r["host_name"] or "").lower(), r["container_name"]))
|
|
return bounded(rows, limit, key="containers",
|
|
more="Pass a larger `limit` (max 500), or filter by `host`, `status` or `problem`.")
|
|
|
|
|
|
async def _pick_container(db, container: str, host: str):
|
|
containers = await docker_svc.list_containers(db)
|
|
if host:
|
|
hosts = list((await db.execute(select(Host))).scalars())
|
|
h = pick(host, hosts, kind="host", list_tool="list_hosts")
|
|
containers = [c for c in containers if c.host_id == h.id]
|
|
want = (container or "").strip()
|
|
if not want:
|
|
raise ValueError("pass a container name; list_containers lists them.")
|
|
same = [c for c in containers if c.name == want] or \
|
|
[c for c in containers if c.name.lower() == want.lower()]
|
|
if len(same) == 1:
|
|
return same[0]
|
|
if same:
|
|
raise ValueError(f"{len(same)} hosts run a container named {want!r}; pass `host` "
|
|
f"(one of: {', '.join(sorted(c.host_id for c in same))}).")
|
|
# One more chance: a unique container whose name contains it (swarm task
|
|
# names look like "stack_svc.1.k2j…").
|
|
partial = [c for c in containers if want.lower() in c.name.lower()]
|
|
if len(partial) == 1:
|
|
return partial[0]
|
|
hint = (f" Containers whose names contain it: {', '.join(sorted(c.name for c in partial)[:10])}."
|
|
if partial else "")
|
|
raise ValueError(f"no container named {want!r}{' on that host' if host else ''}.{hint} "
|
|
"list_containers lists them.")
|
|
|
|
|
|
async def get_container(container: str, host: str = "", range: str = "24h", events: int = 20) -> dict:
|
|
"""One container in depth: state, resources over time and lifecycle events.
|
|
|
|
container: the container's name, as list_containers shows. A unique part of
|
|
a name also works (swarm task names are long).
|
|
host: the host's name or id. Needed only when several hosts run a
|
|
container with this name.
|
|
range: the window for `cpu_pct_history` and `mem_pct_history` (min, avg,
|
|
max, p95, last and a ≤60-point series) — "1h", "6h", "24h" (default),
|
|
"7d" or "30d". `cpu_pct` and `mem_pct` are the latest reported values.
|
|
events: how many lifecycle events (start, stop, die, oom, health_change)
|
|
to return, newest first (default 20, max 100).
|
|
"""
|
|
since, key = lookback(range)
|
|
events = clamp_limit(events, 20, 100)
|
|
now = now_utc()
|
|
async with db_session() as db:
|
|
c = await _pick_container(db, container, host)
|
|
gone = (c.host_id, c.name) in await _replaced(db, await docker_svc.list_containers(db))
|
|
failures = (await docker_svc.recent_failures_by_container(
|
|
db, now - timedelta(hours=24))).get((c.host_id, c.name), [])
|
|
host_names = await _host_names(db)
|
|
cpu = await docker_svc.container_points(db, c.host_id, c.name, "cpu_pct", since)
|
|
mem = await docker_svc.container_points(db, c.host_id, c.name, "mem_pct", since)
|
|
evs = await docker_svc.container_events(db, c.host_id, c.name, events)
|
|
sick = await docker_svc.unhealthy_since(db, [(c.host_id, c.name)])
|
|
hour_ago = now - timedelta(hours=1)
|
|
recent = sum(1 for e in failures if e.at >= hour_ago)
|
|
try:
|
|
ports = json.loads(c.ports_json or "[]")
|
|
except ValueError:
|
|
ports = []
|
|
problems = [] if gone else container_problems(c, recent, bool(failures), _crash_loop_deaths())
|
|
began = problem_since(problems, failures, hour_ago, sick.get((c.host_id, c.name)))
|
|
return {
|
|
**container_row(c, host_names, problems, now, replaced=gone, since=began),
|
|
"container_id": c.container_id,
|
|
"ports": ports,
|
|
"net_rx_bytes": c.net_rx_bytes, "net_tx_bytes": c.net_tx_bytes,
|
|
"blk_read_bytes": c.blk_read_bytes, "blk_write_bytes": c.blk_write_bytes,
|
|
"range": key,
|
|
"cpu_pct_history": summarize(cpu),
|
|
"mem_pct_history": summarize(mem),
|
|
"events": [{"event": e.event, "detail": e.detail, "at": iso(e.at), "age_s": age_s(e.at, now)}
|
|
for e in evs],
|
|
"next": f'search_container_logs(container="{c.name}", host="{c.host_id}")',
|
|
}
|
|
|
|
|
|
async def list_swarm_services(limit: int = 100) -> dict:
|
|
"""Docker Swarm services with their replicas per node, and the swarm's nodes.
|
|
|
|
`state` is "healthy" (all desired replicas running), "degraded" (some) or
|
|
"down" (none). `replicas` are the service's current tasks: each names its
|
|
host and container status, and one marked `ghost` runs on a node with no
|
|
Steward agent, so Steward knows only how many tasks are placed there.
|
|
Tasks Swarm replaced (rolling updates, retries) are left out and counted
|
|
in `previous_tasks`.
|
|
|
|
From Swarm's own task records (a manager's agent 1.8.0 or later;
|
|
`tasks_known: false` and nulls otherwise):
|
|
- `failures_1h` / `failures_24h`: tasks that failed or were rejected.
|
|
Swarm retries a failed task as a new one, so this, not any container's
|
|
restart count, is how a crash loop shows. Swarm keeps about 5 old tasks
|
|
per replica, which bounds the count.
|
|
- `crash_looping`: failures_1h reached the crash-loop count set under
|
|
Settings → Thresholds (default 3).
|
|
- `last_error`: the newest task that failed, was rejected, or carries an
|
|
error while Swarm retries it — `error` is Swarm's own text (e.g. "No
|
|
such image", "task: non-zero exit (1)"), with `exit_code`, `node`, `at`.
|
|
- `trouble_since` / `trouble_age_s`: when the current trouble began — a
|
|
crash loop's first failure in the hour, a short service's first failed
|
|
task in its failing slots, a recovered service's first failure in 24h.
|
|
Swarm keeps few old tasks, so a long outage reads as a lower bound.
|
|
|
|
Services with a problem come first. `nodes` lists every swarm node once,
|
|
with its role (manager/worker), availability and status.
|
|
limit: default 100, max 500.
|
|
"""
|
|
limit = clamp_limit(limit, 100, 500)
|
|
now = now_utc()
|
|
async with db_session() as db:
|
|
services, nodes, host_names = await _swarm(db, now)
|
|
seen, node_rows = set(), []
|
|
for n in sorted(nodes, key=lambda n: n.updated_at, reverse=True):
|
|
if n.node_id in seen:
|
|
continue
|
|
seen.add(n.node_id)
|
|
node_rows.append({"node_id": n.node_id, "hostname": n.hostname, "role": n.role,
|
|
"leader": n.leader, "availability": n.availability, "status": n.status,
|
|
"reported_by": host_names.get(n.host_id)})
|
|
node_rows.sort(key=lambda n: n["hostname"].lower())
|
|
order = {"down": 0, "degraded": 1, "healthy": 2}
|
|
|
|
def _rank(s):
|
|
return (not s["crash_looping"], order.get(s["state"], 3), not s["failures_24h"], s["name"])
|
|
|
|
def _error_row(err):
|
|
if err is None:
|
|
return None
|
|
at = task_time(err)
|
|
return {"error": err["error"], "message": err["message"], "state": err["state"],
|
|
"exit_code": err["exit_code"], "node": err["node"],
|
|
"at": iso(at), "age_s": age_s(at, now) if at else None}
|
|
|
|
rows = [
|
|
{
|
|
"service_name": s["name"], "mode": s["mode"], "state": s["state"],
|
|
"running": s["running"], "desired": s["desired"],
|
|
"crash_looping": s["crash_looping"],
|
|
"failures_1h": s["failures_1h"], "failures_24h": s["failures_24h"],
|
|
"last_error": _error_row(s["last_error"]),
|
|
"trouble_since": iso(s["trouble_since"]),
|
|
"trouble_age_s": age_s(s["trouble_since"], now),
|
|
"previous_tasks": s["previous_tasks"], "tasks_known": s["tasks_known"],
|
|
"replicas": [
|
|
{"ghost": True, "host": r["host"], "count": r["count"]} if r["ghost"] else
|
|
{"ghost": False, "host": r["host"], "host_id": r["host_id"], "container_name": r["name"],
|
|
"status": r["status"], "health": r["health"], "restart_count": r["restart_count"],
|
|
"cpu_pct": rnd(r["cpu_pct"]), "mem_pct": rnd(r["mem_pct"])}
|
|
for r in s["replicas"]
|
|
],
|
|
}
|
|
for s in sorted(services, key=_rank)
|
|
]
|
|
return {"nodes": node_rows, **bounded(rows, limit, key="services")}
|
|
|
|
|
|
async def search_container_logs(container: str = "", host: str = "", text: str = "",
|
|
range: str = "1h", stream: str = "", before: str = "",
|
|
context: int = 0, limit: int = 50) -> dict:
|
|
"""Search the container log lines Steward has retained, newest first.
|
|
|
|
container: one container's name (as list_containers shows); empty searches
|
|
every container (narrow with `host`).
|
|
host: a host's name or id; empty for all hosts.
|
|
text: case-insensitive substring to match; empty matches every line.
|
|
range: how far back — "1h" (default), "6h", "24h", "7d" or "30d". Logs are
|
|
kept per container up to a size and age limit, so older lines may be gone.
|
|
stream: "stdout" or "stderr"; empty for both.
|
|
before: an ISO time; returns only lines older than it. Pass a response's
|
|
`next_before` to page further back.
|
|
context: lines of surrounding log (0-5) to include with each match, from
|
|
the same container. Only with `container` set.
|
|
limit: matches to return, default 50, max 200.
|
|
|
|
Lines longer than 2000 characters keep both ends, and the middle is
|
|
replaced by a marker saying how much was cut.
|
|
"""
|
|
since, key = lookback(range)
|
|
limit = clamp_limit(limit, 50, 200)
|
|
context = max(0, min(int(context or 0), 5))
|
|
if context and not container:
|
|
raise ValueError("context needs `container`: it shows lines around a match within one container's log.")
|
|
if stream and stream not in ("stdout", "stderr"):
|
|
raise ValueError(f"stream {stream!r} is not 'stdout' or 'stderr'.")
|
|
before_dt = None
|
|
if before:
|
|
try:
|
|
before_dt = datetime.fromisoformat(before.replace("Z", "+00:00"))
|
|
except ValueError as exc:
|
|
raise ValueError(f"before {before!r} is not an ISO-8601 time; pass a response's next_before.") from exc
|
|
async with db_session() as db:
|
|
host_id = name = None
|
|
if container:
|
|
c = await _pick_container(db, container, host)
|
|
host_id, name = c.host_id, c.name
|
|
elif host:
|
|
hosts = list((await db.execute(select(Host))).scalars())
|
|
host_id = pick(host, hosts, kind="host", list_tool="list_hosts").id
|
|
rows = await docker_svc.search_logs(db, since=since, host_id=host_id, name=name, text=text,
|
|
stream=stream, before=before_dt, limit=limit + 1)
|
|
more = len(rows) > limit
|
|
rows = rows[:limit]
|
|
host_names = await _host_names(db)
|
|
matches = []
|
|
for r in rows:
|
|
m = {"at": iso(r.ts), "container_name": r.container_name, "host_id": r.host_id,
|
|
"host_name": host_names.get(r.host_id), "stream": r.stream, "line": cut_middle(r.line)}
|
|
if context:
|
|
pre, post = await docker_svc.log_context(db, r.host_id, r.container_name, r.ts, context)
|
|
m["before"] = [cut_middle(x.line) for x in pre]
|
|
m["after"] = [cut_middle(x.line) for x in post]
|
|
matches.append(m)
|
|
out = {"range": key, "matches": matches, "returned": len(matches), "truncated": more}
|
|
if more:
|
|
# Full precision: `at` is cut to the second for reading, and a cursor
|
|
# cut the same way would skip lines logged earlier in that second.
|
|
out["next_before"] = rows[-1].ts.isoformat()
|
|
out["more"] = "More matches are older; pass next_before as `before`, or narrow with `text`."
|
|
return out
|
|
|
|
|
|
async def get_docker_disk_usage(host: str = "") -> dict:
|
|
"""Docker disk usage per host: images, containers, volumes and build cache, in bytes.
|
|
|
|
host: a host's name or id; empty for every host. `images_reclaimable_bytes`
|
|
is what pruning unused images would free. An admin can prune from
|
|
Steward's Docker → Disk page.
|
|
"""
|
|
async with db_session() as db:
|
|
summaries = list((await db.execute(select(DockerDiskUsage))).scalars())
|
|
host_names = await _host_names(db)
|
|
if host:
|
|
hosts = list((await db.execute(select(Host))).scalars())
|
|
hid = pick(host, hosts, kind="host", list_tool="list_hosts").id
|
|
summaries = [s for s in summaries if s.host_id == hid]
|
|
rows = [
|
|
{
|
|
"host_id": s.host_id, "host_name": host_names.get(s.host_id),
|
|
"images_total": s.images_total, "images_active": s.images_active,
|
|
"images_bytes": s.images_size, "images_reclaimable_bytes": s.images_reclaimable,
|
|
"containers_count": s.containers_count, "containers_bytes": s.containers_size,
|
|
"volumes_count": s.volumes_count, "volumes_bytes": s.volumes_size,
|
|
"build_cache_bytes": s.build_cache_size,
|
|
"reported_at": iso(s.updated_at), "reported_age_s": age_s(s.updated_at),
|
|
}
|
|
for s in summaries
|
|
]
|
|
rows.sort(key=lambda r: -(r["images_reclaimable_bytes"] or 0))
|
|
return {"hosts": rows, "total": len(rows)}
|
|
|
|
|
|
PRUNE_BUILTIN_PLAYBOOK = "maintenance/docker_prune.yml"
|
|
_STOPPED = ("exited", "created", "dead")
|
|
|
|
|
|
def prune_preview(what: str, usage, stopped: int) -> dict:
|
|
"""What a prune of `what` would remove, from the host's latest disk report."""
|
|
out: dict = {}
|
|
if what in ("containers", "system"):
|
|
out["stopped_containers"] = stopped
|
|
if what == "images":
|
|
out["unused_images_bytes"] = usage.images_reclaimable if usage else None
|
|
if what == "system":
|
|
out["dangling_images"] = "yes (unused but tagged images stay)"
|
|
out["build_cache_bytes"] = usage.build_cache_size if usage else None
|
|
out["unused_networks"] = "yes"
|
|
return out
|
|
|
|
|
|
async def prune_docker(host: str, what: str, check: bool = True) -> dict:
|
|
"""Reclaim Docker disk on a host. Previews unless check=false.
|
|
|
|
host: the Steward host's name or id. It needs a linked Ansible target
|
|
(list_inventory shows the links); the prune runs through Ansible as
|
|
Steward's built-in docker_prune playbook.
|
|
what: "containers" removes stopped containers; "images" removes ALL
|
|
unused images, not only dangling ones; "system" is `docker system
|
|
prune -f`: stopped containers, dangling images, unused networks and
|
|
build cache. Volumes are never pruned.
|
|
check: true (default) starts nothing and returns `would_remove`, sized
|
|
from the host's latest Docker disk report. Ansible's own check mode
|
|
skips the prune commands, so a check-mode run would show nothing.
|
|
Pass false to run the prune.
|
|
|
|
Admin only, as on the Docker → Disk page. A real prune returns `run_id`
|
|
(follow it with wait_for_run) and is recorded in the audit log.
|
|
"""
|
|
what = (what or "").strip()
|
|
if what not in docker_svc.PRUNE_TARGETS:
|
|
raise ValueError(f"what {what!r} is not one of {', '.join(docker_svc.PRUNE_TARGETS)}.")
|
|
async with db_session() as db:
|
|
h = pick(host, (await db.execute(select(Host))).scalars(), kind="host", list_tool="list_hosts")
|
|
target = (await db.execute(
|
|
select(AnsibleTarget).where(AnsibleTarget.host_id == h.id))).scalars().first()
|
|
usage = await db.get(DockerDiskUsage, h.id)
|
|
stopped = sum(1 for c in await docker_svc.list_containers(db, h.id)
|
|
if (c.status or "").lower() in _STOPPED)
|
|
if target is None:
|
|
raise ValueError(f"host {h.name!r} has no linked Ansible target, so Steward cannot reach it "
|
|
"to prune. Link one under Ansible → Inventory. Nothing was run.")
|
|
preview = {
|
|
"host": h.name,
|
|
"what": what,
|
|
"ansible_target": target.name,
|
|
"would_remove": prune_preview(what, usage, stopped),
|
|
"disk_report_age_s": age_s(usage.updated_at) if usage else None,
|
|
}
|
|
if check:
|
|
return {**preview, "check": True,
|
|
"summary": f"Nothing was run. Pass check=false to prune {what} on {h.name}."}
|
|
caller = current_caller()
|
|
run, _src, err = await trigger_run(
|
|
mcp_app(), source_name=BUILTIN_SOURCE_NAME, playbook_path=PRUNE_BUILTIN_PLAYBOOK,
|
|
inventory_scope=f"steward:target:{target.id}",
|
|
params={"extra_vars_map": docker_svc.prune_extra_vars(what)},
|
|
triggered_by=caller.user_id)
|
|
if err:
|
|
raise ValueError(f"the prune could not start: {err}.")
|
|
await audit_mcp("docker.prune.started", "host", h.name,
|
|
{"host_id": h.id, "what": what, "run_id": run.id})
|
|
return {**preview, "check": False, "run_id": run.id,
|
|
"next": f'wait_for_run(run_id="{run.id}")'}
|
|
|
|
|
|
def register_tools(register) -> None:
|
|
for fn in (list_containers, get_container, list_swarm_services,
|
|
search_container_logs, get_docker_disk_usage):
|
|
register(fn, role=UserRole.viewer, read_only=True)
|
|
register(prune_docker, role=UserRole.admin, read_only=False)
|