bvandeusenandClaude Opus 5.5
21338dbcd9
MCP-5 of milestone 461 "MCP server: Steward as an LLM-inspectable, LLM-operable hub". Contributed through each plugin's register_mcp_tools hook, so they exist only while the plugin is enabled. docker (viewer, read-only): - list_containers: filter by host, status, compose project, or problem (restarting, crash_looping, unhealthy, oom_killed, exited_nonzero); problems sort first and each row names what is wrong. - get_container: state, CPU/memory history across the raw/hourly seam, lifecycle events. A unique part of a swarm task name is enough. - list_swarm_services: replicas per node (agent-less placements as ghosts) and the swarm's nodes. - search_container_logs: text (% and _ literal), stream, range, context lines, paging by a full-precision `before` cursor; over-long lines keep both ends and say how much of the middle was cut. - get_docker_disk_usage: bytes per host, reclaimable first. - mcp_overview: failing containers and under-replicated swarm services join get_overview. host_agent: list_agents (reporting / stale / pending, version, OS), and an overview finding for agents registered over 30 minutes ago that never checked in. Settings > Thresholds gains the TLS-expiry warn/crit days (were 14/3 in code) and the crash-loop deaths-per-hour count (3), which get_overview and the docker tools now read. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
419 lines
20 KiB
Python
419 lines
20 KiB
Python
"""The docker plugin's MCP tools and its get_overview findings.
|
|
|
|
Registered through register_mcp_tools() in __init__.py, so they exist only
|
|
while the plugin is enabled. Reads go through service.py, the same functions
|
|
the plugin's pages use.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
from datetime import datetime, timedelta
|
|
|
|
from sqlalchemy import select
|
|
|
|
from steward.mcp._context import mcp_app
|
|
from steward.mcp.tools._shape import (
|
|
age_s, bounded, clamp_limit, db_session, iso, lookback, now_utc, pick, rnd, summarize,
|
|
)
|
|
from steward.mcp.tools.overview import finding
|
|
from steward.models.hosts import Host
|
|
from steward.models.users import UserRole
|
|
|
|
from . import service as docker_svc
|
|
from .dedup import dedup_by_container_id
|
|
from .models import DockerDiskUsage, DockerSwarmNode, DockerSwarmService
|
|
from .swarm_view import build_swarm_services
|
|
|
|
PROBLEMS = ("restarting", "crash_looping", "unhealthy", "oom_killed", "exited_nonzero")
|
|
CRASH_LOOP_DEATHS = 3 # default; Settings → Thresholds sets it
|
|
LOG_LINE_CHARS = 2000
|
|
|
|
|
|
# ── pure helpers ─────────────────────────────────────────────────────────────
|
|
|
|
def container_problems(c, failures_1h: int, failed_24h: bool,
|
|
crash_loop_deaths: int = CRASH_LOOP_DEATHS) -> list[str]:
|
|
"""What is wrong with one container, most severe first; [] when nothing is."""
|
|
status = (c.status or "").lower()
|
|
out = []
|
|
if status == "restarting":
|
|
out.append("restarting")
|
|
if failures_1h >= crash_loop_deaths:
|
|
out.append("crash_looping")
|
|
if status == "running" and (c.health or "") == "unhealthy":
|
|
out.append("unhealthy")
|
|
if c.oom_killed:
|
|
out.append("oom_killed")
|
|
if status in ("exited", "dead") and (c.exit_code or 0) != 0 and failed_24h:
|
|
out.append("exited_nonzero")
|
|
return out
|
|
|
|
|
|
def cut_middle(text: str, limit: int = LOG_LINE_CHARS) -> str:
|
|
"""Keep both ends of an over-long line and say how much of the middle went."""
|
|
if text is None or len(text) <= limit:
|
|
return text
|
|
keep = limit // 2
|
|
return f"{text[:keep]} [… {len(text) - 2 * keep} characters omitted …] {text[-keep:]}"
|
|
|
|
|
|
def container_row(c, host_names: dict, problems: list[str], now: datetime) -> dict:
|
|
running = (c.status or "").lower() == "running"
|
|
return {
|
|
"container_name": c.name,
|
|
"host_id": c.host_id,
|
|
"host_name": host_names.get(c.host_id),
|
|
"image": c.image,
|
|
"status": c.status,
|
|
"health": c.health,
|
|
"problems": problems,
|
|
"exit_code": c.exit_code,
|
|
"oom_killed": bool(c.oom_killed),
|
|
"restart_count": c.restart_count,
|
|
"cpu_pct": rnd(c.cpu_pct),
|
|
"mem_pct": rnd(c.mem_pct),
|
|
"mem_usage_bytes": c.mem_usage_bytes,
|
|
"mem_limit_bytes": c.mem_limit_bytes,
|
|
"started_at": iso(c.started_at),
|
|
"uptime_s": age_s(c.started_at, now) if running and c.started_at else None,
|
|
"compose_project": c.compose_project,
|
|
"swarm_service": c.service_name,
|
|
"reported_at": iso(c.scraped_at),
|
|
"reported_age_s": age_s(c.scraped_at, now),
|
|
}
|
|
|
|
|
|
_SEVERITY = {"restarting": "critical", "crash_looping": "critical", "unhealthy": "critical",
|
|
"oom_killed": "warning", "exited_nonzero": "warning"}
|
|
_SAYS = {"restarting": "is restarting", "crash_looping": "keeps dying",
|
|
"unhealthy": "fails its healthcheck", "oom_killed": "was OOM-killed",
|
|
"exited_nonzero": "exited with an error"}
|
|
|
|
|
|
def container_findings(rows: list[dict]) -> list[dict]:
|
|
out = []
|
|
for r in rows:
|
|
if not r["problems"]:
|
|
continue
|
|
worst = r["problems"][0]
|
|
where = f" on {r['host_name']}" if r["host_name"] else ""
|
|
detail = f" (exit {r['exit_code']})" if r["exit_code"] not in (None, 0) else ""
|
|
out.append(finding(
|
|
_SEVERITY[worst], f"container_{worst}", r["container_name"],
|
|
f"container {r['container_name']}{where} {' and '.join(_SAYS[p] for p in r['problems'])}{detail}",
|
|
ids={"host_id": r["host_id"], "container_name": r["container_name"]},
|
|
next=f'get_container(container="{r["container_name"]}", host="{r["host_id"]}")'))
|
|
return out
|
|
|
|
|
|
def service_findings(services: list[dict]) -> list[dict]:
|
|
out = []
|
|
for s in services:
|
|
if s["state"] == "healthy":
|
|
continue
|
|
out.append(finding(
|
|
"critical" if s["state"] == "down" else "warning", f"swarm_service_{s['state']}",
|
|
s["name"], f"swarm service {s['name']} has {s['running']} of {s['desired']} replicas running",
|
|
ids={"service_name": s["name"]}, next="list_swarm_services()"))
|
|
return out
|
|
|
|
|
|
# ── loading ──────────────────────────────────────────────────────────────────
|
|
|
|
async def _host_names(db) -> dict[str, str]:
|
|
return dict((await db.execute(select(Host.id, Host.name))).all())
|
|
|
|
|
|
def _crash_loop_deaths() -> int:
|
|
th = (mcp_app().config.get("THRESHOLDS") or {}).get("crash_loop") or {}
|
|
return int(th.get("warn") or CRASH_LOOP_DEATHS)
|
|
|
|
|
|
async def _rows(db, now: datetime) -> list[dict]:
|
|
containers = dedup_by_container_id(await docker_svc.list_containers(db))
|
|
failures = await docker_svc.recent_failures_by_container(db, now - timedelta(hours=24))
|
|
hour_ago = now - timedelta(hours=1)
|
|
host_names = await _host_names(db)
|
|
deaths = _crash_loop_deaths()
|
|
rows = []
|
|
for c in containers:
|
|
events = failures.get((c.host_id, c.name), [])
|
|
recent = sum(1 for e in events if e.at >= hour_ago)
|
|
rows.append(container_row(c, host_names, container_problems(c, recent, bool(events), deaths), now))
|
|
return rows
|
|
|
|
|
|
async def _swarm(db):
|
|
containers = dedup_by_container_id(await docker_svc.list_containers(db))
|
|
services = list((await db.execute(select(DockerSwarmService))).scalars())
|
|
nodes = list((await db.execute(select(DockerSwarmNode))).scalars())
|
|
host_names = await _host_names(db)
|
|
node_hostname = {n.node_id: n.hostname for n in nodes if n.hostname}
|
|
built = build_swarm_services(containers, services, node_hostname, host_names)
|
|
return built["services"], nodes, host_names
|
|
|
|
|
|
async def mcp_overview(db) -> list[dict]:
|
|
"""get_overview findings: failing containers and under-replicated swarm services."""
|
|
rows = await _rows(db, now_utc())
|
|
services, _, _ = await _swarm(db)
|
|
return container_findings(rows) + service_findings(services)
|
|
|
|
|
|
# ── tools ────────────────────────────────────────────────────────────────────
|
|
|
|
async def list_containers(host: str = "", status: str = "", problem: str = "",
|
|
compose_project: str = "", limit: int = 100) -> dict:
|
|
"""Docker containers across every host that runs a Steward agent with Docker.
|
|
|
|
host: a host's name or id (list_hosts); empty for all hosts.
|
|
status: Docker's state — "running", "exited", "restarting", "paused",
|
|
"created" or "dead"; empty for all.
|
|
problem: "any" for containers with something wrong, or one of
|
|
"restarting", "crash_looping" (died repeatedly within the last hour;
|
|
the count is set under Settings → Thresholds, default 3), "unhealthy"
|
|
(failing its healthcheck), "oom_killed", "exited_nonzero" (died with an
|
|
error in the last 24h); empty for no filter.
|
|
compose_project: only containers from this Compose project.
|
|
limit: default 100, max 500.
|
|
|
|
Containers with problems come first. Each row's `problems` lists what is
|
|
wrong, most severe first. `reported_age_s` is how old the agent's last
|
|
report is, so a large value means the row may be stale.
|
|
"""
|
|
if problem and problem != "any" and problem not in PROBLEMS:
|
|
raise ValueError(f"problem {problem!r} is not 'any' or one of {', '.join(PROBLEMS)}.")
|
|
limit = clamp_limit(limit, 100, 500)
|
|
now = now_utc()
|
|
async with db_session() as db:
|
|
rows = await _rows(db, now)
|
|
if host:
|
|
hosts = list((await db.execute(select(Host))).scalars())
|
|
h = pick(host, hosts, kind="host", list_tool="list_hosts")
|
|
rows = [r for r in rows if r["host_id"] == h.id]
|
|
if status:
|
|
rows = [r for r in rows if (r["status"] or "").lower() == status.lower()]
|
|
if problem == "any":
|
|
rows = [r for r in rows if r["problems"]]
|
|
elif problem:
|
|
rows = [r for r in rows if problem in r["problems"]]
|
|
if compose_project:
|
|
rows = [r for r in rows if r["compose_project"] == compose_project]
|
|
rows.sort(key=lambda r: (not r["problems"], (r["status"] or "") != "running",
|
|
(r["host_name"] or "").lower(), r["container_name"]))
|
|
return bounded(rows, limit, key="containers",
|
|
more="Pass a larger `limit` (max 500), or filter by `host`, `status` or `problem`.")
|
|
|
|
|
|
async def _pick_container(db, container: str, host: str):
|
|
containers = await docker_svc.list_containers(db)
|
|
if host:
|
|
hosts = list((await db.execute(select(Host))).scalars())
|
|
h = pick(host, hosts, kind="host", list_tool="list_hosts")
|
|
containers = [c for c in containers if c.host_id == h.id]
|
|
want = (container or "").strip()
|
|
if not want:
|
|
raise ValueError("pass a container name; list_containers lists them.")
|
|
same = [c for c in containers if c.name == want] or \
|
|
[c for c in containers if c.name.lower() == want.lower()]
|
|
if len(same) == 1:
|
|
return same[0]
|
|
if same:
|
|
raise ValueError(f"{len(same)} hosts run a container named {want!r}; pass `host` "
|
|
f"(one of: {', '.join(sorted(c.host_id for c in same))}).")
|
|
# One more chance: a unique container whose name contains it (swarm task
|
|
# names look like "stack_svc.1.k2j…").
|
|
partial = [c for c in containers if want.lower() in c.name.lower()]
|
|
if len(partial) == 1:
|
|
return partial[0]
|
|
hint = (f" Containers whose names contain it: {', '.join(sorted(c.name for c in partial)[:10])}."
|
|
if partial else "")
|
|
raise ValueError(f"no container named {want!r}{' on that host' if host else ''}.{hint} "
|
|
"list_containers lists them.")
|
|
|
|
|
|
async def get_container(container: str, host: str = "", range: str = "24h", events: int = 20) -> dict:
|
|
"""One container in depth: state, resources over time and lifecycle events.
|
|
|
|
container: the container's name, as list_containers shows. A unique part of
|
|
a name also works (swarm task names are long).
|
|
host: the host's name or id. Needed only when several hosts run a
|
|
container with this name.
|
|
range: the window for `cpu_pct_history` and `mem_pct_history` (min, avg,
|
|
max, p95, last and a ≤60-point series) — "1h", "6h", "24h" (default),
|
|
"7d" or "30d". `cpu_pct` and `mem_pct` are the latest reported values.
|
|
events: how many lifecycle events (start, stop, die, oom, health_change)
|
|
to return, newest first (default 20, max 100).
|
|
"""
|
|
since, key = lookback(range)
|
|
events = clamp_limit(events, 20, 100)
|
|
now = now_utc()
|
|
async with db_session() as db:
|
|
c = await _pick_container(db, container, host)
|
|
failures = (await docker_svc.recent_failures_by_container(
|
|
db, now - timedelta(hours=24))).get((c.host_id, c.name), [])
|
|
host_names = await _host_names(db)
|
|
cpu = await docker_svc.container_points(db, c.host_id, c.name, "cpu_pct", since)
|
|
mem = await docker_svc.container_points(db, c.host_id, c.name, "mem_pct", since)
|
|
evs = await docker_svc.container_events(db, c.host_id, c.name, events)
|
|
recent = sum(1 for e in failures if e.at >= now - timedelta(hours=1))
|
|
try:
|
|
ports = json.loads(c.ports_json or "[]")
|
|
except ValueError:
|
|
ports = []
|
|
return {
|
|
**container_row(c, host_names,
|
|
container_problems(c, recent, bool(failures), _crash_loop_deaths()), now),
|
|
"container_id": c.container_id,
|
|
"ports": ports,
|
|
"net_rx_bytes": c.net_rx_bytes, "net_tx_bytes": c.net_tx_bytes,
|
|
"blk_read_bytes": c.blk_read_bytes, "blk_write_bytes": c.blk_write_bytes,
|
|
"range": key,
|
|
"cpu_pct_history": summarize(cpu),
|
|
"mem_pct_history": summarize(mem),
|
|
"events": [{"event": e.event, "detail": e.detail, "at": iso(e.at), "age_s": age_s(e.at, now)}
|
|
for e in evs],
|
|
"next": f'search_container_logs(container="{c.name}", host="{c.host_id}")',
|
|
}
|
|
|
|
|
|
async def list_swarm_services(limit: int = 100) -> dict:
|
|
"""Docker Swarm services with their replicas per node, and the swarm's nodes.
|
|
|
|
`state` is "healthy" (all desired replicas running), "degraded" (some) or
|
|
"down" (none). Each replica names its host and container status; a replica
|
|
marked `ghost` runs on a node with no Steward agent, so Steward knows only
|
|
how many tasks are placed there. `nodes` lists every swarm node once, with
|
|
its role (manager/worker), availability and status.
|
|
limit: default 100, max 500.
|
|
"""
|
|
limit = clamp_limit(limit, 100, 500)
|
|
async with db_session() as db:
|
|
services, nodes, host_names = await _swarm(db)
|
|
seen, node_rows = set(), []
|
|
for n in sorted(nodes, key=lambda n: n.updated_at, reverse=True):
|
|
if n.node_id in seen:
|
|
continue
|
|
seen.add(n.node_id)
|
|
node_rows.append({"node_id": n.node_id, "hostname": n.hostname, "role": n.role,
|
|
"leader": n.leader, "availability": n.availability, "status": n.status,
|
|
"reported_by": host_names.get(n.host_id)})
|
|
node_rows.sort(key=lambda n: n["hostname"].lower())
|
|
order = {"down": 0, "degraded": 1, "healthy": 2}
|
|
rows = [
|
|
{
|
|
"service_name": s["name"], "mode": s["mode"], "state": s["state"],
|
|
"running": s["running"], "desired": s["desired"],
|
|
"replicas": [
|
|
{"ghost": True, "host": r["host"], "count": r["count"]} if r["ghost"] else
|
|
{"ghost": False, "host": r["host"], "host_id": r["host_id"], "container_name": r["name"],
|
|
"status": r["status"], "health": r["health"], "restart_count": r["restart_count"],
|
|
"cpu_pct": rnd(r["cpu_pct"]), "mem_pct": rnd(r["mem_pct"])}
|
|
for r in s["replicas"]
|
|
],
|
|
}
|
|
for s in sorted(services, key=lambda s: (order.get(s["state"], 3), s["name"]))
|
|
]
|
|
return {"nodes": node_rows, **bounded(rows, limit, key="services")}
|
|
|
|
|
|
async def search_container_logs(container: str = "", host: str = "", text: str = "",
|
|
range: str = "1h", stream: str = "", before: str = "",
|
|
context: int = 0, limit: int = 50) -> dict:
|
|
"""Search the container log lines Steward has retained, newest first.
|
|
|
|
container: one container's name (as list_containers shows); empty searches
|
|
every container (narrow with `host`).
|
|
host: a host's name or id; empty for all hosts.
|
|
text: case-insensitive substring to match; empty matches every line.
|
|
range: how far back — "1h" (default), "6h", "24h", "7d" or "30d". Logs are
|
|
kept per container up to a size and age limit, so older lines may be gone.
|
|
stream: "stdout" or "stderr"; empty for both.
|
|
before: an ISO time; returns only lines older than it. Pass a response's
|
|
`next_before` to page further back.
|
|
context: lines of surrounding log (0-5) to include with each match, from
|
|
the same container. Only with `container` set.
|
|
limit: matches to return, default 50, max 200.
|
|
|
|
Lines longer than 2000 characters keep both ends, and the middle is
|
|
replaced by a marker saying how much was cut.
|
|
"""
|
|
since, key = lookback(range)
|
|
limit = clamp_limit(limit, 50, 200)
|
|
context = max(0, min(int(context or 0), 5))
|
|
if context and not container:
|
|
raise ValueError("context needs `container`: it shows lines around a match within one container's log.")
|
|
if stream and stream not in ("stdout", "stderr"):
|
|
raise ValueError(f"stream {stream!r} is not 'stdout' or 'stderr'.")
|
|
before_dt = None
|
|
if before:
|
|
try:
|
|
before_dt = datetime.fromisoformat(before.replace("Z", "+00:00"))
|
|
except ValueError as exc:
|
|
raise ValueError(f"before {before!r} is not an ISO-8601 time; pass a response's next_before.") from exc
|
|
async with db_session() as db:
|
|
host_id = name = None
|
|
if container:
|
|
c = await _pick_container(db, container, host)
|
|
host_id, name = c.host_id, c.name
|
|
elif host:
|
|
hosts = list((await db.execute(select(Host))).scalars())
|
|
host_id = pick(host, hosts, kind="host", list_tool="list_hosts").id
|
|
rows = await docker_svc.search_logs(db, since=since, host_id=host_id, name=name, text=text,
|
|
stream=stream, before=before_dt, limit=limit + 1)
|
|
more = len(rows) > limit
|
|
rows = rows[:limit]
|
|
host_names = await _host_names(db)
|
|
matches = []
|
|
for r in rows:
|
|
m = {"at": iso(r.ts), "container_name": r.container_name, "host_id": r.host_id,
|
|
"host_name": host_names.get(r.host_id), "stream": r.stream, "line": cut_middle(r.line)}
|
|
if context:
|
|
pre, post = await docker_svc.log_context(db, r.host_id, r.container_name, r.ts, context)
|
|
m["before"] = [cut_middle(x.line) for x in pre]
|
|
m["after"] = [cut_middle(x.line) for x in post]
|
|
matches.append(m)
|
|
out = {"range": key, "matches": matches, "returned": len(matches), "truncated": more}
|
|
if more:
|
|
# Full precision: `at` is cut to the second for reading, and a cursor
|
|
# cut the same way would skip lines logged earlier in that second.
|
|
out["next_before"] = rows[-1].ts.isoformat()
|
|
out["more"] = "More matches are older; pass next_before as `before`, or narrow with `text`."
|
|
return out
|
|
|
|
|
|
async def get_docker_disk_usage(host: str = "") -> dict:
|
|
"""Docker disk usage per host: images, containers, volumes and build cache, in bytes.
|
|
|
|
host: a host's name or id; empty for every host. `images_reclaimable_bytes`
|
|
is what pruning unused images would free. An admin can prune from
|
|
Steward's Docker → Disk page.
|
|
"""
|
|
async with db_session() as db:
|
|
summaries = list((await db.execute(select(DockerDiskUsage))).scalars())
|
|
host_names = await _host_names(db)
|
|
if host:
|
|
hosts = list((await db.execute(select(Host))).scalars())
|
|
hid = pick(host, hosts, kind="host", list_tool="list_hosts").id
|
|
summaries = [s for s in summaries if s.host_id == hid]
|
|
rows = [
|
|
{
|
|
"host_id": s.host_id, "host_name": host_names.get(s.host_id),
|
|
"images_total": s.images_total, "images_active": s.images_active,
|
|
"images_bytes": s.images_size, "images_reclaimable_bytes": s.images_reclaimable,
|
|
"containers_count": s.containers_count, "containers_bytes": s.containers_size,
|
|
"volumes_count": s.volumes_count, "volumes_bytes": s.volumes_size,
|
|
"build_cache_bytes": s.build_cache_size,
|
|
"reported_at": iso(s.updated_at), "reported_age_s": age_s(s.updated_at),
|
|
}
|
|
for s in summaries
|
|
]
|
|
rows.sort(key=lambda r: -(r["images_reclaimable_bytes"] or 0))
|
|
return {"hosts": rows, "total": len(rows)}
|
|
|
|
|
|
def register_tools(register) -> None:
|
|
for fn in (list_containers, get_container, list_swarm_services,
|
|
search_container_logs, get_docker_disk_usage):
|
|
register(fn, role=UserRole.viewer, read_only=True)
|