feat(mcp): docker and host-agent tools; TLS and crash-loop cutoffs in Settings
CI / lint (push) Successful in 2s
CI / integration (push) Successful in 37s
CI / unit (push) Successful in 58s
CI / publish (push) Successful in 54s

MCP-5 of milestone 461 "MCP server: Steward as an LLM-inspectable,
LLM-operable hub". Contributed through each plugin's register_mcp_tools
hook, so they exist only while the plugin is enabled.

docker (viewer, read-only):
- list_containers: filter by host, status, compose project, or problem
  (restarting, crash_looping, unhealthy, oom_killed, exited_nonzero);
  problems sort first and each row names what is wrong.
- get_container: state, CPU/memory history across the raw/hourly seam,
  lifecycle events. A unique part of a swarm task name is enough.
- list_swarm_services: replicas per node (agent-less placements as ghosts)
  and the swarm's nodes.
- search_container_logs: text (% and _ literal), stream, range, context
  lines, paging by a full-precision `before` cursor; over-long lines keep
  both ends and say how much of the middle was cut.
- get_docker_disk_usage: bytes per host, reclaimable first.
- mcp_overview: failing containers and under-replicated swarm services
  join get_overview.

host_agent: list_agents (reporting / stale / pending, version, OS), and an
overview finding for agents registered over 30 minutes ago that never
checked in.

Settings > Thresholds gains the TLS-expiry warn/crit days (were 14/3 in
code) and the crash-loop deaths-per-hour count (3), which get_overview
and the docker tools now read.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
2026-10-06 07:47:02 -04:00
co-authored by Claude Opus 5.5
parent 41c7509366
commit 21338dbcd9
13 changed files with 957 additions and 11 deletions
+10
View File
@@ -49,6 +49,16 @@ def get_blueprint():
return docker_bp
def register_mcp_tools(register) -> None:
from .mcp_tools import register_tools
register_tools(register)
async def mcp_overview(db) -> list[dict]:
from .mcp_tools import mcp_overview as findings
return await findings(db)
def get_nav() -> list[dict]:
# Surfaces in the sidebar's Infrastructure group (fleet-wide container view).
return [{"label": "Docker", "href": "/plugins/docker/"}]
+418
View File
@@ -0,0 +1,418 @@
"""The docker plugin's MCP tools and its get_overview findings.
Registered through register_mcp_tools() in __init__.py, so they exist only
while the plugin is enabled. Reads go through service.py, the same functions
the plugin's pages use.
"""
from __future__ import annotations
import json
from datetime import datetime, timedelta
from sqlalchemy import select
from steward.mcp._context import mcp_app
from steward.mcp.tools._shape import (
age_s, bounded, clamp_limit, db_session, iso, lookback, now_utc, pick, rnd, summarize,
)
from steward.mcp.tools.overview import finding
from steward.models.hosts import Host
from steward.models.users import UserRole
from . import service as docker_svc
from .dedup import dedup_by_container_id
from .models import DockerDiskUsage, DockerSwarmNode, DockerSwarmService
from .swarm_view import build_swarm_services
PROBLEMS = ("restarting", "crash_looping", "unhealthy", "oom_killed", "exited_nonzero")
CRASH_LOOP_DEATHS = 3 # default; Settings → Thresholds sets it
LOG_LINE_CHARS = 2000
# ── pure helpers ─────────────────────────────────────────────────────────────
def container_problems(c, failures_1h: int, failed_24h: bool,
crash_loop_deaths: int = CRASH_LOOP_DEATHS) -> list[str]:
"""What is wrong with one container, most severe first; [] when nothing is."""
status = (c.status or "").lower()
out = []
if status == "restarting":
out.append("restarting")
if failures_1h >= crash_loop_deaths:
out.append("crash_looping")
if status == "running" and (c.health or "") == "unhealthy":
out.append("unhealthy")
if c.oom_killed:
out.append("oom_killed")
if status in ("exited", "dead") and (c.exit_code or 0) != 0 and failed_24h:
out.append("exited_nonzero")
return out
def cut_middle(text: str, limit: int = LOG_LINE_CHARS) -> str:
"""Keep both ends of an over-long line and say how much of the middle went."""
if text is None or len(text) <= limit:
return text
keep = limit // 2
return f"{text[:keep]} [… {len(text) - 2 * keep} characters omitted …] {text[-keep:]}"
def container_row(c, host_names: dict, problems: list[str], now: datetime) -> dict:
running = (c.status or "").lower() == "running"
return {
"container_name": c.name,
"host_id": c.host_id,
"host_name": host_names.get(c.host_id),
"image": c.image,
"status": c.status,
"health": c.health,
"problems": problems,
"exit_code": c.exit_code,
"oom_killed": bool(c.oom_killed),
"restart_count": c.restart_count,
"cpu_pct": rnd(c.cpu_pct),
"mem_pct": rnd(c.mem_pct),
"mem_usage_bytes": c.mem_usage_bytes,
"mem_limit_bytes": c.mem_limit_bytes,
"started_at": iso(c.started_at),
"uptime_s": age_s(c.started_at, now) if running and c.started_at else None,
"compose_project": c.compose_project,
"swarm_service": c.service_name,
"reported_at": iso(c.scraped_at),
"reported_age_s": age_s(c.scraped_at, now),
}
_SEVERITY = {"restarting": "critical", "crash_looping": "critical", "unhealthy": "critical",
"oom_killed": "warning", "exited_nonzero": "warning"}
_SAYS = {"restarting": "is restarting", "crash_looping": "keeps dying",
"unhealthy": "fails its healthcheck", "oom_killed": "was OOM-killed",
"exited_nonzero": "exited with an error"}
def container_findings(rows: list[dict]) -> list[dict]:
out = []
for r in rows:
if not r["problems"]:
continue
worst = r["problems"][0]
where = f" on {r['host_name']}" if r["host_name"] else ""
detail = f" (exit {r['exit_code']})" if r["exit_code"] not in (None, 0) else ""
out.append(finding(
_SEVERITY[worst], f"container_{worst}", r["container_name"],
f"container {r['container_name']}{where} {' and '.join(_SAYS[p] for p in r['problems'])}{detail}",
ids={"host_id": r["host_id"], "container_name": r["container_name"]},
next=f'get_container(container="{r["container_name"]}", host="{r["host_id"]}")'))
return out
def service_findings(services: list[dict]) -> list[dict]:
out = []
for s in services:
if s["state"] == "healthy":
continue
out.append(finding(
"critical" if s["state"] == "down" else "warning", f"swarm_service_{s['state']}",
s["name"], f"swarm service {s['name']} has {s['running']} of {s['desired']} replicas running",
ids={"service_name": s["name"]}, next="list_swarm_services()"))
return out
# ── loading ──────────────────────────────────────────────────────────────────
async def _host_names(db) -> dict[str, str]:
return dict((await db.execute(select(Host.id, Host.name))).all())
def _crash_loop_deaths() -> int:
th = (mcp_app().config.get("THRESHOLDS") or {}).get("crash_loop") or {}
return int(th.get("warn") or CRASH_LOOP_DEATHS)
async def _rows(db, now: datetime) -> list[dict]:
containers = dedup_by_container_id(await docker_svc.list_containers(db))
failures = await docker_svc.recent_failures_by_container(db, now - timedelta(hours=24))
hour_ago = now - timedelta(hours=1)
host_names = await _host_names(db)
deaths = _crash_loop_deaths()
rows = []
for c in containers:
events = failures.get((c.host_id, c.name), [])
recent = sum(1 for e in events if e.at >= hour_ago)
rows.append(container_row(c, host_names, container_problems(c, recent, bool(events), deaths), now))
return rows
async def _swarm(db):
containers = dedup_by_container_id(await docker_svc.list_containers(db))
services = list((await db.execute(select(DockerSwarmService))).scalars())
nodes = list((await db.execute(select(DockerSwarmNode))).scalars())
host_names = await _host_names(db)
node_hostname = {n.node_id: n.hostname for n in nodes if n.hostname}
built = build_swarm_services(containers, services, node_hostname, host_names)
return built["services"], nodes, host_names
async def mcp_overview(db) -> list[dict]:
"""get_overview findings: failing containers and under-replicated swarm services."""
rows = await _rows(db, now_utc())
services, _, _ = await _swarm(db)
return container_findings(rows) + service_findings(services)
# ── tools ────────────────────────────────────────────────────────────────────
async def list_containers(host: str = "", status: str = "", problem: str = "",
compose_project: str = "", limit: int = 100) -> dict:
"""Docker containers across every host that runs a Steward agent with Docker.
host: a host's name or id (list_hosts); empty for all hosts.
status: Docker's state — "running", "exited", "restarting", "paused",
"created" or "dead"; empty for all.
problem: "any" for containers with something wrong, or one of
"restarting", "crash_looping" (died repeatedly within the last hour;
the count is set under Settings → Thresholds, default 3), "unhealthy"
(failing its healthcheck), "oom_killed", "exited_nonzero" (died with an
error in the last 24h); empty for no filter.
compose_project: only containers from this Compose project.
limit: default 100, max 500.
Containers with problems come first. Each row's `problems` lists what is
wrong, most severe first. `reported_age_s` is how old the agent's last
report is, so a large value means the row may be stale.
"""
if problem and problem != "any" and problem not in PROBLEMS:
raise ValueError(f"problem {problem!r} is not 'any' or one of {', '.join(PROBLEMS)}.")
limit = clamp_limit(limit, 100, 500)
now = now_utc()
async with db_session() as db:
rows = await _rows(db, now)
if host:
hosts = list((await db.execute(select(Host))).scalars())
h = pick(host, hosts, kind="host", list_tool="list_hosts")
rows = [r for r in rows if r["host_id"] == h.id]
if status:
rows = [r for r in rows if (r["status"] or "").lower() == status.lower()]
if problem == "any":
rows = [r for r in rows if r["problems"]]
elif problem:
rows = [r for r in rows if problem in r["problems"]]
if compose_project:
rows = [r for r in rows if r["compose_project"] == compose_project]
rows.sort(key=lambda r: (not r["problems"], (r["status"] or "") != "running",
(r["host_name"] or "").lower(), r["container_name"]))
return bounded(rows, limit, key="containers",
more="Pass a larger `limit` (max 500), or filter by `host`, `status` or `problem`.")
async def _pick_container(db, container: str, host: str):
containers = await docker_svc.list_containers(db)
if host:
hosts = list((await db.execute(select(Host))).scalars())
h = pick(host, hosts, kind="host", list_tool="list_hosts")
containers = [c for c in containers if c.host_id == h.id]
want = (container or "").strip()
if not want:
raise ValueError("pass a container name; list_containers lists them.")
same = [c for c in containers if c.name == want] or \
[c for c in containers if c.name.lower() == want.lower()]
if len(same) == 1:
return same[0]
if same:
raise ValueError(f"{len(same)} hosts run a container named {want!r}; pass `host` "
f"(one of: {', '.join(sorted(c.host_id for c in same))}).")
# One more chance: a unique container whose name contains it (swarm task
# names look like "stack_svc.1.k2j…").
partial = [c for c in containers if want.lower() in c.name.lower()]
if len(partial) == 1:
return partial[0]
hint = (f" Containers whose names contain it: {', '.join(sorted(c.name for c in partial)[:10])}."
if partial else "")
raise ValueError(f"no container named {want!r}{' on that host' if host else ''}.{hint} "
"list_containers lists them.")
async def get_container(container: str, host: str = "", range: str = "24h", events: int = 20) -> dict:
"""One container in depth: state, resources over time and lifecycle events.
container: the container's name, as list_containers shows. A unique part of
a name also works (swarm task names are long).
host: the host's name or id. Needed only when several hosts run a
container with this name.
range: the window for `cpu_pct_history` and `mem_pct_history` (min, avg,
max, p95, last and a ≤60-point series) — "1h", "6h", "24h" (default),
"7d" or "30d". `cpu_pct` and `mem_pct` are the latest reported values.
events: how many lifecycle events (start, stop, die, oom, health_change)
to return, newest first (default 20, max 100).
"""
since, key = lookback(range)
events = clamp_limit(events, 20, 100)
now = now_utc()
async with db_session() as db:
c = await _pick_container(db, container, host)
failures = (await docker_svc.recent_failures_by_container(
db, now - timedelta(hours=24))).get((c.host_id, c.name), [])
host_names = await _host_names(db)
cpu = await docker_svc.container_points(db, c.host_id, c.name, "cpu_pct", since)
mem = await docker_svc.container_points(db, c.host_id, c.name, "mem_pct", since)
evs = await docker_svc.container_events(db, c.host_id, c.name, events)
recent = sum(1 for e in failures if e.at >= now - timedelta(hours=1))
try:
ports = json.loads(c.ports_json or "[]")
except ValueError:
ports = []
return {
**container_row(c, host_names,
container_problems(c, recent, bool(failures), _crash_loop_deaths()), now),
"container_id": c.container_id,
"ports": ports,
"net_rx_bytes": c.net_rx_bytes, "net_tx_bytes": c.net_tx_bytes,
"blk_read_bytes": c.blk_read_bytes, "blk_write_bytes": c.blk_write_bytes,
"range": key,
"cpu_pct_history": summarize(cpu),
"mem_pct_history": summarize(mem),
"events": [{"event": e.event, "detail": e.detail, "at": iso(e.at), "age_s": age_s(e.at, now)}
for e in evs],
"next": f'search_container_logs(container="{c.name}", host="{c.host_id}")',
}
async def list_swarm_services(limit: int = 100) -> dict:
"""Docker Swarm services with their replicas per node, and the swarm's nodes.
`state` is "healthy" (all desired replicas running), "degraded" (some) or
"down" (none). Each replica names its host and container status; a replica
marked `ghost` runs on a node with no Steward agent, so Steward knows only
how many tasks are placed there. `nodes` lists every swarm node once, with
its role (manager/worker), availability and status.
limit: default 100, max 500.
"""
limit = clamp_limit(limit, 100, 500)
async with db_session() as db:
services, nodes, host_names = await _swarm(db)
seen, node_rows = set(), []
for n in sorted(nodes, key=lambda n: n.updated_at, reverse=True):
if n.node_id in seen:
continue
seen.add(n.node_id)
node_rows.append({"node_id": n.node_id, "hostname": n.hostname, "role": n.role,
"leader": n.leader, "availability": n.availability, "status": n.status,
"reported_by": host_names.get(n.host_id)})
node_rows.sort(key=lambda n: n["hostname"].lower())
order = {"down": 0, "degraded": 1, "healthy": 2}
rows = [
{
"service_name": s["name"], "mode": s["mode"], "state": s["state"],
"running": s["running"], "desired": s["desired"],
"replicas": [
{"ghost": True, "host": r["host"], "count": r["count"]} if r["ghost"] else
{"ghost": False, "host": r["host"], "host_id": r["host_id"], "container_name": r["name"],
"status": r["status"], "health": r["health"], "restart_count": r["restart_count"],
"cpu_pct": rnd(r["cpu_pct"]), "mem_pct": rnd(r["mem_pct"])}
for r in s["replicas"]
],
}
for s in sorted(services, key=lambda s: (order.get(s["state"], 3), s["name"]))
]
return {"nodes": node_rows, **bounded(rows, limit, key="services")}
async def search_container_logs(container: str = "", host: str = "", text: str = "",
range: str = "1h", stream: str = "", before: str = "",
context: int = 0, limit: int = 50) -> dict:
"""Search the container log lines Steward has retained, newest first.
container: one container's name (as list_containers shows); empty searches
every container (narrow with `host`).
host: a host's name or id; empty for all hosts.
text: case-insensitive substring to match; empty matches every line.
range: how far back — "1h" (default), "6h", "24h", "7d" or "30d". Logs are
kept per container up to a size and age limit, so older lines may be gone.
stream: "stdout" or "stderr"; empty for both.
before: an ISO time; returns only lines older than it. Pass a response's
`next_before` to page further back.
context: lines of surrounding log (0-5) to include with each match, from
the same container. Only with `container` set.
limit: matches to return, default 50, max 200.
Lines longer than 2000 characters keep both ends, and the middle is
replaced by a marker saying how much was cut.
"""
since, key = lookback(range)
limit = clamp_limit(limit, 50, 200)
context = max(0, min(int(context or 0), 5))
if context and not container:
raise ValueError("context needs `container`: it shows lines around a match within one container's log.")
if stream and stream not in ("stdout", "stderr"):
raise ValueError(f"stream {stream!r} is not 'stdout' or 'stderr'.")
before_dt = None
if before:
try:
before_dt = datetime.fromisoformat(before.replace("Z", "+00:00"))
except ValueError as exc:
raise ValueError(f"before {before!r} is not an ISO-8601 time; pass a response's next_before.") from exc
async with db_session() as db:
host_id = name = None
if container:
c = await _pick_container(db, container, host)
host_id, name = c.host_id, c.name
elif host:
hosts = list((await db.execute(select(Host))).scalars())
host_id = pick(host, hosts, kind="host", list_tool="list_hosts").id
rows = await docker_svc.search_logs(db, since=since, host_id=host_id, name=name, text=text,
stream=stream, before=before_dt, limit=limit + 1)
more = len(rows) > limit
rows = rows[:limit]
host_names = await _host_names(db)
matches = []
for r in rows:
m = {"at": iso(r.ts), "container_name": r.container_name, "host_id": r.host_id,
"host_name": host_names.get(r.host_id), "stream": r.stream, "line": cut_middle(r.line)}
if context:
pre, post = await docker_svc.log_context(db, r.host_id, r.container_name, r.ts, context)
m["before"] = [cut_middle(x.line) for x in pre]
m["after"] = [cut_middle(x.line) for x in post]
matches.append(m)
out = {"range": key, "matches": matches, "returned": len(matches), "truncated": more}
if more:
# Full precision: `at` is cut to the second for reading, and a cursor
# cut the same way would skip lines logged earlier in that second.
out["next_before"] = rows[-1].ts.isoformat()
out["more"] = "More matches are older; pass next_before as `before`, or narrow with `text`."
return out
async def get_docker_disk_usage(host: str = "") -> dict:
"""Docker disk usage per host: images, containers, volumes and build cache, in bytes.
host: a host's name or id; empty for every host. `images_reclaimable_bytes`
is what pruning unused images would free. An admin can prune from
Steward's Docker → Disk page.
"""
async with db_session() as db:
summaries = list((await db.execute(select(DockerDiskUsage))).scalars())
host_names = await _host_names(db)
if host:
hosts = list((await db.execute(select(Host))).scalars())
hid = pick(host, hosts, kind="host", list_tool="list_hosts").id
summaries = [s for s in summaries if s.host_id == hid]
rows = [
{
"host_id": s.host_id, "host_name": host_names.get(s.host_id),
"images_total": s.images_total, "images_active": s.images_active,
"images_bytes": s.images_size, "images_reclaimable_bytes": s.images_reclaimable,
"containers_count": s.containers_count, "containers_bytes": s.containers_size,
"volumes_count": s.volumes_count, "volumes_bytes": s.volumes_size,
"build_cache_bytes": s.build_cache_size,
"reported_at": iso(s.updated_at), "reported_age_s": age_s(s.updated_at),
}
for s in summaries
]
rows.sort(key=lambda r: -(r["images_reclaimable_bytes"] or 0))
return {"hosts": rows, "total": len(rows)}
def register_tools(register) -> None:
for fn in (list_containers, get_container, list_swarm_services,
search_container_logs, get_docker_disk_usage):
register(fn, role=UserRole.viewer, read_only=True)
+88 -2
View File
@@ -6,11 +6,12 @@ Quart's pipeline) call the same code.
"""
from __future__ import annotations
from sqlalchemy import select
from sqlalchemy import func, select
from steward.core.time_range import bucket_seconds
from steward.models.hosts import Host
from .models import DockerContainer, DockerEvent, DockerLog, DockerMetric
from .models import DockerContainer, DockerEvent, DockerLog, DockerMetric, DockerMetricHourly
def bucket(values: list[float], target: int = 40) -> list[float]:
@@ -97,3 +98,88 @@ async def query_logs(db, host_id: str, name: str, stream: str, query: str,
stmt = stmt.where(DockerLog.line.ilike(f"%{escape_like(query)}%", escape="\\"))
stmt = stmt.order_by(DockerLog.ts.desc(), DockerLog.id.desc()).limit(limit)
return (await db.execute(stmt)).all()
CONTAINER_SERIES = ("cpu_pct", "mem_pct", "mem_usage_bytes")
async def container_points(db, host_id: str, name: str, metric: str, since,
target: int = 120) -> list[tuple]:
"""(time, average) points of one container metric since `since`, oldest first.
Raw samples are bucket-averaged in SQL; the part of the range older than
the oldest raw sample (already rolled up and pruned by retention) comes
from the hourly table.
"""
if metric not in CONTAINER_SERIES:
raise ValueError(f"metric must be one of {', '.join(CONTAINER_SERIES)}")
raw_col = getattr(DockerMetric, metric)
width_s = bucket_seconds(since, target)
rbucket = func.date_bin(
func.make_interval(0, 0, 0, 0, 0, 0, width_s), DockerMetric.scraped_at, func.to_timestamp(0),
).label("bucket")
raw = (await db.execute(
select(rbucket, func.avg(raw_col))
.where(DockerMetric.host_id == host_id, DockerMetric.container_name == name,
DockerMetric.scraped_at >= since)
.group_by(rbucket).order_by(rbucket)
)).all()
oldest_raw = raw[0][0] if raw else None
hourly_stmt = (
select(DockerMetricHourly.bucket, getattr(DockerMetricHourly, metric))
.where(DockerMetricHourly.host_id == host_id,
DockerMetricHourly.container_name == name,
DockerMetricHourly.bucket >= since)
.order_by(DockerMetricHourly.bucket)
)
if oldest_raw is not None:
hourly_stmt = hourly_stmt.where(DockerMetricHourly.bucket < oldest_raw)
hourly = (await db.execute(hourly_stmt)).all()
return [(t, float(v)) for t, v in hourly] + [(t, float(v)) for t, v in raw]
async def search_logs(db, *, since, host_id: str | None = None, name: str | None = None,
text: str = "", stream: str = "", before=None, limit: int = 50) -> list:
"""Retained log lines matching a filter, newest first (limit + 1 fetched by callers
that need to know whether more exist)."""
stmt = (select(DockerLog.id, DockerLog.host_id, DockerLog.container_name,
DockerLog.ts, DockerLog.stream, DockerLog.line)
.where(DockerLog.ts >= since))
if host_id:
stmt = stmt.where(DockerLog.host_id == host_id)
if name:
stmt = stmt.where(DockerLog.container_name == name)
if stream in ("stdout", "stderr"):
stmt = stmt.where(DockerLog.stream == stream)
if text:
stmt = stmt.where(DockerLog.line.ilike(f"%{escape_like(text)}%", escape="\\"))
if before is not None:
stmt = stmt.where(DockerLog.ts < before)
stmt = stmt.order_by(DockerLog.ts.desc(), DockerLog.id.desc()).limit(limit)
return list((await db.execute(stmt)).all())
async def log_context(db, host_id: str, name: str, ts, n: int) -> tuple[list, list]:
"""Up to n lines either side of `ts` in one container's log, each oldest first."""
cols = (DockerLog.id, DockerLog.ts, DockerLog.stream, DockerLog.line)
base = select(*cols).where(DockerLog.host_id == host_id, DockerLog.container_name == name)
before = (await db.execute(
base.where(DockerLog.ts < ts).order_by(DockerLog.ts.desc(), DockerLog.id.desc()).limit(n)
)).all()
after = (await db.execute(
base.where(DockerLog.ts > ts).order_by(DockerLog.ts, DockerLog.id).limit(n)
)).all()
return list(reversed(before)), list(after)
async def recent_failures_by_container(db, since) -> dict[tuple[str, str], list]:
"""{(host_id, container_name): [die/oom events since `since`, newest first]}."""
rows = (await db.execute(
select(DockerEvent)
.where(DockerEvent.event.in_(("die", "oom")), DockerEvent.at >= since)
.order_by(DockerEvent.at.desc())
)).scalars()
out: dict[tuple[str, str], list] = {}
for e in rows:
out.setdefault((e.host_id, e.container_name), []).append(e)
return out