bvandeusenandClaude Opus 5.5
622addc09e
Each source in an anomaly's `around` was cut to its oldest 40 with nothing in the reply to say so; on a busy host the events nearest the occurrence were the ones dropped. Each source now keeps the 40 nearest the occurrence's start, and `around_left_out` counts the rest. Sources return everything their bounded query read (Docker and the host agent no longer cut at 40 and 20 themselves), so the count is honest. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
771 lines
37 KiB
Python
771 lines
37 KiB
Python
"""The docker plugin's MCP tools and its get_overview findings.
|
|
|
|
Registered through register_mcp_tools() in __init__.py, so they exist only
|
|
while the plugin is enabled. Reads go through service.py, the same functions
|
|
the plugin's pages use.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
from dataclasses import dataclass
|
|
from datetime import datetime, timedelta
|
|
|
|
from sqlalchemy import select
|
|
|
|
from steward.ansible.runner import trigger_run
|
|
from steward.ansible.sources import BUILTIN_SOURCE_NAME
|
|
from steward.mcp._context import current_caller, mcp_app
|
|
from steward.mcp.tools._audit import audit_mcp
|
|
from steward.mcp.tools._shape import (
|
|
age_s, bounded, clamp_limit, db_session, iso, lookback, now_utc, pick, rnd, summarize,
|
|
)
|
|
from steward.mcp.tools.overview import finding
|
|
from steward.models.ansible_inventory import AnsibleTarget
|
|
from steward.models.hosts import Host
|
|
from steward.models.users import UserRole
|
|
|
|
from . import service as docker_svc
|
|
from .service import PROBLEM_SAYS, PROBLEM_SEVERITY, PROBLEMS, container_problems, problem_since
|
|
from .dedup import dedup_by_container_id
|
|
from .models import DockerContainer, DockerDiskUsage, DockerEvent, DockerSwarmNode, DockerSwarmService
|
|
from .swarm_view import build_swarm_services, replaced_tasks, task_time
|
|
|
|
CRASH_LOOP_DEATHS = docker_svc.CRASH_LOOP_DEATHS
|
|
LOG_LINE_CHARS = 2000
|
|
|
|
|
|
# ── pure helpers ─────────────────────────────────────────────────────────────
|
|
|
|
def cut_middle(text: str, limit: int = LOG_LINE_CHARS) -> str:
|
|
"""Keep both ends of an over-long line and say how much of the middle went."""
|
|
if text is None or len(text) <= limit:
|
|
return text
|
|
keep = limit // 2
|
|
return f"{text[:keep]} [… {len(text) - 2 * keep} characters omitted …] {text[-keep:]}"
|
|
|
|
|
|
def container_row(c, host_names: dict, problems: list[str], now: datetime,
|
|
replaced: bool = False, since: datetime | None = None) -> dict:
|
|
running = (c.status or "").lower() == "running"
|
|
return {
|
|
"container_name": c.name,
|
|
"host_id": c.host_id,
|
|
"host_name": host_names.get(c.host_id),
|
|
"image": c.image,
|
|
"status": c.status,
|
|
"health": c.health,
|
|
"problems": problems,
|
|
# When the worst problem began, from the container's retained events.
|
|
"problem_since": iso(since),
|
|
"problem_age_s": age_s(since, now),
|
|
"exit_code": c.exit_code,
|
|
"oom_killed": bool(c.oom_killed),
|
|
"restart_count": c.restart_count,
|
|
"cpu_pct": rnd(c.cpu_pct),
|
|
"mem_pct": rnd(c.mem_pct),
|
|
"mem_usage_bytes": c.mem_usage_bytes,
|
|
"mem_limit_bytes": c.mem_limit_bytes,
|
|
"started_at": iso(c.started_at),
|
|
"uptime_s": age_s(c.started_at, now) if running and c.started_at else None,
|
|
"compose_project": c.compose_project,
|
|
"swarm_service": c.service_name,
|
|
# A swarm task Swarm has replaced: history, judged on its service.
|
|
"swarm_replaced": replaced if c.service_name else None,
|
|
"reported_at": iso(c.scraped_at),
|
|
"reported_age_s": age_s(c.scraped_at, now),
|
|
}
|
|
|
|
|
|
def _from_iso(text: str | None) -> datetime | None:
|
|
return datetime.fromisoformat(text.replace("Z", "+00:00")) if text else None
|
|
|
|
|
|
def container_findings(rows: list[dict]) -> list[dict]:
|
|
out = []
|
|
for r in rows:
|
|
if not r["problems"]:
|
|
continue
|
|
worst = r["problems"][0]
|
|
where = f" on {r['host_name']}" if r["host_name"] else ""
|
|
detail = f" (exit {r['exit_code']})" if r["exit_code"] not in (None, 0) else ""
|
|
out.append(finding(
|
|
PROBLEM_SEVERITY[worst], f"container_{worst}", r["container_name"],
|
|
f"container {r['container_name']}{where} {' and '.join(PROBLEM_SAYS[p] for p in r['problems'])}{detail}",
|
|
since=_from_iso(r["problem_since"]),
|
|
ids={"host_id": r["host_id"], "container_name": r["container_name"]},
|
|
next=f'get_container(container="{r["container_name"]}", host="{r["host_id"]}")'))
|
|
return out
|
|
|
|
|
|
def task_error_text(err: dict | None) -> str:
|
|
"""Swarm's own account of a task's end, as one clause."""
|
|
if not err:
|
|
return ""
|
|
why = err.get("error") or err.get("message") or err.get("state") or "unknown"
|
|
code = f", exit {err['exit_code']}" if err.get("exit_code") not in (None, 0) else ""
|
|
where = f" on {err['node']}" if err.get("node") else ""
|
|
return f"{why}{code}{where}"
|
|
|
|
|
|
# Services that started failing within this long of each other read as one
|
|
# finding: they are usually one node's event (#5722).
|
|
TOGETHER = timedelta(minutes=5)
|
|
|
|
|
|
def service_findings(services: list[dict]) -> list[dict]:
|
|
"""Swarm services failing now: crash-looping, or short of replicas, each
|
|
naming Swarm's last task error. One back to all its replicas is history,
|
|
kept on the Docker page, not a finding (#5722). Several short of replicas
|
|
since within TOGETHER of each other are one finding naming them all."""
|
|
out, short = [], []
|
|
for s in services:
|
|
replicas = f"{s['running']} of {s['desired']} replicas running"
|
|
last = task_error_text(s.get("last_error"))
|
|
tail = f"; last task error: {last}" if last else ""
|
|
ids = {"service_name": s["name"]}
|
|
since = s.get("trouble_since")
|
|
if s.get("crash_looping"):
|
|
out.append(finding(
|
|
"critical", "swarm_service_crash_looping", s["name"],
|
|
f"swarm service {s['name']} keeps failing: {s['failures_1h']} tasks failed in the "
|
|
f"last hour, {replicas}{tail}", since=since, ids=ids,
|
|
next="list_swarm_services()"))
|
|
elif s["state"] != "healthy":
|
|
short.append((s, replicas, tail))
|
|
for group in _together(short):
|
|
if len(group) == 1:
|
|
s, replicas, tail = group[0]
|
|
out.append(finding(
|
|
"critical" if s["state"] == "down" else "warning", f"swarm_service_{s['state']}",
|
|
s["name"], f"swarm service {s['name']} has {replicas}{tail}",
|
|
since=s.get("trouble_since"), ids={"service_name": s["name"]},
|
|
next="list_swarm_services()"))
|
|
continue
|
|
names = [s["name"] for s, _r, _t in group]
|
|
down = any(s["state"] == "down" for s, _r, _t in group)
|
|
out.append(finding(
|
|
"critical" if down else "warning", "swarm_services_short", f"{len(group)} swarm services",
|
|
f"{len(group)} swarm services started failing together and are short of replicas: "
|
|
+ ", ".join(f"{s['name']} ({r})" for s, r, _t in group),
|
|
since=min(s["trouble_since"] for s, _r, _t in group),
|
|
ids={"service_names": names}, next="list_swarm_services()"))
|
|
return out
|
|
|
|
|
|
def _together(short: list[tuple]) -> list[list[tuple]]:
|
|
"""Short services grouped by when they started failing: each group's
|
|
starts lie within TOGETHER of the one before. One without a start stands alone."""
|
|
timed = sorted((x for x in short if x[0].get("trouble_since")), key=lambda x: x[0]["trouble_since"])
|
|
groups: list[list[tuple]] = []
|
|
for x in timed:
|
|
if groups and x[0]["trouble_since"] - groups[-1][-1][0]["trouble_since"] <= TOGETHER:
|
|
groups[-1].append(x)
|
|
else:
|
|
groups.append([x])
|
|
return groups + [[x] for x in short if not x[0].get("trouble_since")]
|
|
|
|
|
|
# ── loading ──────────────────────────────────────────────────────────────────
|
|
|
|
async def _host_names(db) -> dict[str, str]:
|
|
return dict((await db.execute(select(Host.id, Host.name))).all())
|
|
|
|
|
|
def _crash_loop_deaths() -> int:
|
|
return docker_svc.crash_loop_deaths(mcp_app().config)
|
|
|
|
|
|
async def _replaced(db, containers) -> set[tuple]:
|
|
services = list((await db.execute(select(DockerSwarmService))).scalars())
|
|
return replaced_tasks(containers, services)
|
|
|
|
|
|
async def _rows(db, now: datetime) -> list[dict]:
|
|
"""Every container with its problems. A swarm task Swarm replaced has none:
|
|
its failure, if it was one, belongs to its service (service_findings)."""
|
|
host_names = await _host_names(db)
|
|
return [container_row(r["container"], host_names, r["problems"], now,
|
|
replaced=r["replaced"], since=r["since"])
|
|
for r in await docker_svc.ranked_containers(db, now, _crash_loop_deaths())]
|
|
|
|
|
|
async def _swarm(db, now: datetime):
|
|
containers = dedup_by_container_id(await docker_svc.list_containers(db))
|
|
services = list((await db.execute(select(DockerSwarmService))).scalars())
|
|
nodes = list((await db.execute(select(DockerSwarmNode))).scalars())
|
|
host_names = await _host_names(db)
|
|
node_hostname = {n.node_id: n.hostname for n in nodes if n.hostname}
|
|
built = build_swarm_services(containers, services, node_hostname, host_names,
|
|
now=now, crash_loop_deaths=_crash_loop_deaths())
|
|
return built["services"], nodes, host_names
|
|
|
|
|
|
def unique_nodes(nodes: list) -> list:
|
|
"""Each swarm node once: every manager reports them all, so keep the
|
|
newest report of each."""
|
|
seen, out = set(), []
|
|
for n in sorted(nodes, key=lambda n: n.updated_at, reverse=True):
|
|
if n.node_id not in seen:
|
|
seen.add(n.node_id)
|
|
out.append(n)
|
|
return out
|
|
|
|
|
|
async def _maintaining(db) -> dict[str, str]:
|
|
"""{host name, lower case: job state} for hosts with a maintenance job
|
|
running (#5835): a node drained or rebooting for one is expected. The
|
|
host agent is always loaded; None-safe all the same."""
|
|
from steward.core.plugin_manager import loaded_plugin_submodule
|
|
models = loaded_plugin_submodule("host_agent", "models")
|
|
maint = loaded_plugin_submodule("host_agent", "maintenance")
|
|
if models is None or maint is None:
|
|
return {}
|
|
job = models.HostMaintenanceJob
|
|
rows = await db.execute(select(Host.name, job.state).join(Host, Host.id == job.host_id)
|
|
.where(job.state.in_(maint.ACTIVE)))
|
|
return {name.lower(): state for name, state in rows}
|
|
|
|
|
|
NODE_AVAILABILITY_SAYS = {"drain": "drained", "pause": "paused"}
|
|
|
|
|
|
def node_findings(nodes: list, maintaining: dict[str, str]) -> list[dict]:
|
|
"""Swarm nodes down now (#5835): status not ready. A node whose host has
|
|
a maintenance job running is rebooting on purpose and isn't named.
|
|
|
|
Drained alone is not a finding: the operator keeps a manager drained on
|
|
purpose (#5777), and a drain a failed maintenance job left behind is that
|
|
job's finding."""
|
|
out = []
|
|
for n in unique_nodes(nodes):
|
|
name = n.hostname or n.node_id
|
|
if n.status == "ready" or name.lower() in maintaining:
|
|
continue
|
|
drained = NODE_AVAILABILITY_SAYS.get(n.availability)
|
|
out.append(finding(
|
|
"critical", "swarm_node_down", name,
|
|
f"swarm {n.role} {name} is {n.status}" + (f" and {drained}" if drained else ""),
|
|
ids={"node_id": n.node_id}, next="list_swarm_services()"))
|
|
return out
|
|
|
|
|
|
async def mcp_overview(db) -> list[dict]:
|
|
"""get_overview findings: failing containers, under-replicated swarm
|
|
services, and swarm nodes that are down."""
|
|
now = now_utc()
|
|
rows = await _rows(db, now)
|
|
services, nodes, _ = await _swarm(db, now)
|
|
return (container_findings(rows) + service_findings(services)
|
|
+ node_findings(nodes, await _maintaining(db)))
|
|
|
|
|
|
# ── tools ────────────────────────────────────────────────────────────────────
|
|
|
|
async def list_containers(host: str = "", status: str = "", problem: str = "",
|
|
compose_project: str = "", limit: int = 100) -> dict:
|
|
"""Docker containers across every host that runs a Steward agent with Docker.
|
|
|
|
host: a host's name or id (list_hosts); empty for all hosts.
|
|
status: Docker's state — "running", "exited", "restarting", "paused",
|
|
"created" or "dead"; empty for all.
|
|
problem: "any" for containers with something wrong, or one of
|
|
"restarting", "crash_looping" (died repeatedly within the last hour;
|
|
the count is set under Settings → Thresholds, default 3), "unhealthy"
|
|
(failing its healthcheck), "oom_killed", "exited_nonzero" (died with an
|
|
error in the last 24h; exit 137/143 without an OOM kill is a stop, not
|
|
an error); empty for no filter.
|
|
compose_project: only containers from this Compose project.
|
|
limit: default 100, max 500.
|
|
|
|
Containers with problems come first. Each row's `problems` lists what is
|
|
wrong, most severe first, and `problem_since` / `problem_age_s` say when
|
|
the worst of them began (null when Steward's retained events cannot say).
|
|
`reported_age_s` is how old the agent's last report is, so a large value
|
|
means the row may be stale.
|
|
|
|
A swarm task's container that Swarm has replaced (a rolling update, or a
|
|
retry after a failure) has `swarm_replaced: true` and no problems: Swarm
|
|
restarts a task as a new container, so failures are counted on the
|
|
service — list_swarm_services shows them.
|
|
"""
|
|
if problem and problem != "any" and problem not in PROBLEMS:
|
|
raise ValueError(f"problem {problem!r} is not 'any' or one of {', '.join(PROBLEMS)}.")
|
|
limit = clamp_limit(limit, 100, 500)
|
|
now = now_utc()
|
|
async with db_session() as db:
|
|
rows = await _rows(db, now)
|
|
if host:
|
|
hosts = list((await db.execute(select(Host))).scalars())
|
|
h = pick(host, hosts, kind="host", list_tool="list_hosts")
|
|
rows = [r for r in rows if r["host_id"] == h.id]
|
|
if status:
|
|
rows = [r for r in rows if (r["status"] or "").lower() == status.lower()]
|
|
if problem == "any":
|
|
rows = [r for r in rows if r["problems"]]
|
|
elif problem:
|
|
rows = [r for r in rows if problem in r["problems"]]
|
|
if compose_project:
|
|
rows = [r for r in rows if r["compose_project"] == compose_project]
|
|
rows.sort(key=lambda r: (not r["problems"], (r["status"] or "") != "running",
|
|
(r["host_name"] or "").lower(), r["container_name"]))
|
|
return bounded(rows, limit, key="containers",
|
|
more="Pass a larger `limit` (max 500), or filter by `host`, `status` or `problem`.")
|
|
|
|
|
|
@dataclass
|
|
class RemovedContainer:
|
|
"""A container Docker has removed, known only by its retained logs and events."""
|
|
host_id: str
|
|
name: str
|
|
last_seen: datetime
|
|
|
|
|
|
_REMOVED_HINT = ("It may have been removed: Swarm replaces a dead task with a new container under "
|
|
"a new name. list_containers(status=\"exited\") lists stopped containers, and "
|
|
"search_container_logs(host=...) searches a whole host.")
|
|
|
|
|
|
async def _pick_container(db, container: str, host: str, *, removed: bool = False):
|
|
"""The container named `container`: a live one, else (with `removed`) one
|
|
Docker has removed whose logs or events Steward still holds, returned as a
|
|
RemovedContainer. Exact names beat partial ones, and live beats removed."""
|
|
containers = await docker_svc.list_containers(db)
|
|
host_id = None
|
|
if host:
|
|
hosts = list((await db.execute(select(Host))).scalars())
|
|
host_id = pick(host, hosts, kind="host", list_tool="list_hosts").id
|
|
containers = [c for c in containers if c.host_id == host_id]
|
|
want = (container or "").strip()
|
|
if not want:
|
|
raise ValueError("pass a container name; list_containers lists them.")
|
|
live = {(c.host_id, c.name) for c in containers}
|
|
|
|
def _one(found, what):
|
|
if len(found) > 1:
|
|
raise ValueError(f"{len(found)} hosts have {what} named {want!r}; pass `host` "
|
|
f"(one of: {', '.join(sorted(f[0] for f in found))}).")
|
|
return found[0] if found else None
|
|
|
|
same = [c for c in containers if c.name == want] or \
|
|
[c for c in containers if c.name.lower() == want.lower()]
|
|
if same:
|
|
return _one([(c.host_id, c) for c in same], "a container")[1]
|
|
if removed:
|
|
gone = [r for r in await docker_svc.retained_names(db, want, exact=True, host_id=host_id)
|
|
if (r[0], r[1]) not in live]
|
|
if gone:
|
|
return RemovedContainer(*_one(gone, "a removed container"))
|
|
# One more chance: a unique container whose name contains it (swarm task
|
|
# names look like "stack_svc.1.k2j…").
|
|
partial = [c for c in containers if want.lower() in c.name.lower()]
|
|
if len(partial) == 1:
|
|
return partial[0]
|
|
gone = []
|
|
if removed and not partial:
|
|
gone = [r for r in await docker_svc.retained_names(db, want, exact=False, host_id=host_id)
|
|
if (r[0], r[1]) not in live]
|
|
if len(gone) == 1:
|
|
return RemovedContainer(*gone[0])
|
|
names = sorted(c.name for c in partial)[:10] + [f"{r[1]} (removed)" for r in gone[:10]]
|
|
hint = f" Containers whose names contain it: {', '.join(names)}." if names else ""
|
|
services = sorted({c.service_name for c in partial if c.service_name})
|
|
if services:
|
|
hint += (f" Swarm services among them: {', '.join(services[:10])}; search_container_logs "
|
|
"takes a service's name and searches every task it has run.")
|
|
raise ValueError(f"no container named {want!r}{' on that host' if host else ''}.{hint} "
|
|
f"{'' if names else _REMOVED_HINT + ' '}list_containers lists them.")
|
|
|
|
|
|
def _removed_note(c: RemovedContainer) -> str:
|
|
return (f"{c.name} no longer exists: Docker removed it (Swarm replaces a failed task with a new "
|
|
f"container). This is what Steward retained; it was last seen {iso(c.last_seen)}.")
|
|
|
|
|
|
async def get_container(container: str, host: str = "", range: str = "24h", events: int = 20) -> dict:
|
|
"""One container in depth: state, resources over time and lifecycle events.
|
|
|
|
container: the container's name, as list_containers shows. A unique part of
|
|
a name also works (swarm task names are long).
|
|
host: the host's name or id. Needed only when several hosts run a
|
|
container with this name.
|
|
range: the window for `cpu_pct_history` and `mem_pct_history` (min, avg,
|
|
max, p95, last and a ≤60-point series) — "1h", "6h", "24h" (default),
|
|
"7d" or "30d". `cpu_pct` and `mem_pct` are the latest reported values.
|
|
events: how many lifecycle events (start, restart, stop, die, oom, health_change)
|
|
to return, newest first (default 20, max 100).
|
|
|
|
A container Docker has since removed (a dead Swarm task, a `docker rm`)
|
|
still resolves by its exact name, or by a part of it that matches one
|
|
removed container, while Steward retains its logs or events: the reply
|
|
has `removed: true`, `last_seen_at`, its history and its events, and no
|
|
live state. A live container has `removed: false`.
|
|
"""
|
|
since, key = lookback(range)
|
|
events = clamp_limit(events, 20, 100)
|
|
now = now_utc()
|
|
async with db_session() as db:
|
|
c = await _pick_container(db, container, host, removed=True)
|
|
if isinstance(c, RemovedContainer):
|
|
return await _removed_container(db, c, since, key, events, now)
|
|
gone = (c.host_id, c.name) in await _replaced(db, await docker_svc.list_containers(db))
|
|
failures = (await docker_svc.recent_failures_by_container(
|
|
db, now - timedelta(hours=24))).get((c.host_id, c.name), [])
|
|
host_names = await _host_names(db)
|
|
cpu = await docker_svc.container_points(db, c.host_id, c.name, "cpu_pct", since)
|
|
mem = await docker_svc.container_points(db, c.host_id, c.name, "mem_pct", since)
|
|
evs = await docker_svc.container_events(db, c.host_id, c.name, events)
|
|
sick = await docker_svc.unhealthy_since(db, [(c.host_id, c.name)])
|
|
hour_ago = now - timedelta(hours=1)
|
|
recent = sum(1 for e in failures if e.at >= hour_ago)
|
|
try:
|
|
ports = json.loads(c.ports_json or "[]")
|
|
except ValueError:
|
|
ports = []
|
|
problems = [] if gone else container_problems(c, recent, bool(failures), _crash_loop_deaths())
|
|
began = problem_since(problems, failures, hour_ago, sick.get((c.host_id, c.name)))
|
|
return {
|
|
**container_row(c, host_names, problems, now, replaced=gone, since=began),
|
|
"removed": False,
|
|
"container_id": c.container_id,
|
|
"ports": ports,
|
|
"net_rx_bytes": c.net_rx_bytes, "net_tx_bytes": c.net_tx_bytes,
|
|
"blk_read_bytes": c.blk_read_bytes, "blk_write_bytes": c.blk_write_bytes,
|
|
"range": key,
|
|
"cpu_pct_history": summarize(cpu),
|
|
"mem_pct_history": summarize(mem),
|
|
"events": _event_rows(evs, now),
|
|
"next": f'search_container_logs(container="{c.name}", host="{c.host_id}")',
|
|
}
|
|
|
|
|
|
def _event_rows(evs, now: datetime) -> list[dict]:
|
|
return [{"event": e.event, "detail": e.detail, "at": iso(e.at), "age_s": age_s(e.at, now)}
|
|
for e in evs]
|
|
|
|
|
|
async def _removed_container(db, c: RemovedContainer, since, key: str, events: int,
|
|
now: datetime) -> dict:
|
|
"""get_container for a container Docker has removed: what Steward kept."""
|
|
host_names = await _host_names(db)
|
|
cpu = await docker_svc.container_points(db, c.host_id, c.name, "cpu_pct", since)
|
|
mem = await docker_svc.container_points(db, c.host_id, c.name, "mem_pct", since)
|
|
evs = await docker_svc.container_events(db, c.host_id, c.name, events)
|
|
return {
|
|
"container_name": c.name, "host_id": c.host_id, "host_name": host_names.get(c.host_id),
|
|
"removed": True, "note": _removed_note(c),
|
|
"last_seen_at": iso(c.last_seen), "last_seen_age_s": age_s(c.last_seen, now),
|
|
"range": key,
|
|
"cpu_pct_history": summarize(cpu),
|
|
"mem_pct_history": summarize(mem),
|
|
"events": _event_rows(evs, now),
|
|
"next": f'search_container_logs(container="{c.name}", host="{c.host_id}", range="24h")',
|
|
}
|
|
|
|
|
|
async def list_swarm_services(limit: int = 100) -> dict:
|
|
"""Docker Swarm services with their replicas per node, and the swarm's nodes.
|
|
|
|
`state` is "healthy" (all desired replicas running), "degraded" (some) or
|
|
"down" (none). `replicas` are the service's current tasks: each names its
|
|
host and container status, and one marked `ghost` runs on a node with no
|
|
Steward agent, so Steward knows only how many tasks are placed there.
|
|
Tasks Swarm replaced (rolling updates, retries) are left out and counted
|
|
in `previous_tasks`.
|
|
|
|
From Swarm's own task records (a manager's agent 1.8.0 or later;
|
|
`tasks_known: false` and nulls otherwise):
|
|
- `failures_1h` / `failures_24h`: tasks that failed or were rejected.
|
|
Swarm retries a failed task as a new one, so this, not any container's
|
|
restart count, is how a crash loop shows. Swarm keeps about 5 old tasks
|
|
per replica, which bounds the count.
|
|
- `crash_looping`: failures_1h reached the crash-loop count set under
|
|
Settings → Thresholds (default 3).
|
|
- `last_error`: the newest task that failed, was rejected, or carries an
|
|
error while Swarm retries it — `error` is Swarm's own text (e.g. "No
|
|
such image", "task: non-zero exit (1)"), with `exit_code`, `node`, `at`.
|
|
- `trouble_since` / `trouble_age_s`: when the current trouble began — a
|
|
crash loop's first failure in the hour, a short service's first failed
|
|
task in its failing slots, a recovered service's first failure in 24h.
|
|
Swarm keeps few old tasks, so a long outage reads as a lower bound.
|
|
|
|
Services with a problem come first. `nodes` lists every swarm node once,
|
|
with its role (manager/worker, `leader`), availability (active, drain,
|
|
pause) and status (ready, down, unknown); `host_id` is the Steward host
|
|
of that name (get_host, get_maintenance_job), and `maintenance` the state
|
|
of a maintenance job running on it, which drains and reboots it on
|
|
purpose (null when none).
|
|
limit: default 100, max 500.
|
|
"""
|
|
limit = clamp_limit(limit, 100, 500)
|
|
now = now_utc()
|
|
async with db_session() as db:
|
|
services, nodes, host_names = await _swarm(db, now)
|
|
maintaining = await _maintaining(db)
|
|
host_ids = {name.lower(): hid for hid, name in host_names.items()}
|
|
node_rows = [{"node_id": n.node_id, "hostname": n.hostname, "role": n.role,
|
|
"leader": n.leader, "availability": n.availability, "status": n.status,
|
|
"host_id": host_ids.get((n.hostname or "").lower()),
|
|
"maintenance": maintaining.get((n.hostname or "").lower()),
|
|
"reported_by": host_names.get(n.host_id)}
|
|
for n in unique_nodes(nodes)]
|
|
node_rows.sort(key=lambda n: n["hostname"].lower())
|
|
order = {"down": 0, "degraded": 1, "healthy": 2}
|
|
|
|
def _rank(s):
|
|
return (not s["crash_looping"], order.get(s["state"], 3), not s["failures_24h"], s["name"])
|
|
|
|
def _error_row(err):
|
|
if err is None:
|
|
return None
|
|
at = task_time(err)
|
|
return {"error": err["error"], "message": err["message"], "state": err["state"],
|
|
"exit_code": err["exit_code"], "node": err["node"],
|
|
"at": iso(at), "age_s": age_s(at, now) if at else None}
|
|
|
|
rows = [
|
|
{
|
|
"service_name": s["name"], "mode": s["mode"], "state": s["state"],
|
|
"running": s["running"], "desired": s["desired"],
|
|
"crash_looping": s["crash_looping"],
|
|
"failures_1h": s["failures_1h"], "failures_24h": s["failures_24h"],
|
|
"last_error": _error_row(s["last_error"]),
|
|
"trouble_since": iso(s["trouble_since"]),
|
|
"trouble_age_s": age_s(s["trouble_since"], now),
|
|
"previous_tasks": s["previous_tasks"], "tasks_known": s["tasks_known"],
|
|
"replicas": [
|
|
{"ghost": True, "host": r["host"], "count": r["count"]} if r["ghost"] else
|
|
{"ghost": False, "host": r["host"], "host_id": r["host_id"], "container_name": r["name"],
|
|
"status": r["status"], "health": r["health"], "restart_count": r["restart_count"],
|
|
"cpu_pct": rnd(r["cpu_pct"]), "mem_pct": rnd(r["mem_pct"])}
|
|
for r in s["replicas"]
|
|
],
|
|
}
|
|
for s in sorted(services, key=_rank)
|
|
]
|
|
return {"nodes": node_rows, **bounded(rows, limit, key="services")}
|
|
|
|
|
|
async def search_container_logs(container: str = "", host: str = "", text: str = "",
|
|
range: str = "1h", stream: str = "", before: str = "",
|
|
context: int = 0, limit: int = 50) -> dict:
|
|
"""Search the container log lines Steward has retained, newest first.
|
|
|
|
container: one container's name (as list_containers shows), or a Swarm
|
|
service's name (as list_swarm_services shows) to search every task
|
|
container the service has run, live and removed, so a search reaches
|
|
back across deploys; each match then has the `slot` and `task_id` that
|
|
wrote it. Empty searches every container (narrow with `host`). A
|
|
container Docker has removed still works while its lines are retained:
|
|
the reply then has `container_removed: true` and a `note` saying when it
|
|
was last seen.
|
|
host: a host's name or id; empty for all hosts.
|
|
text: case-insensitive substring to match; empty matches every line.
|
|
range: how far back — "1h" (default), "6h", "24h", "7d" or "30d". Logs are
|
|
kept per container up to a size and age limit, so older lines may be gone.
|
|
stream: "stdout" or "stderr"; empty for both.
|
|
before: an ISO time; returns only lines older than it. Pass a response's
|
|
`next_before` to page further back.
|
|
context: lines of surrounding log (0-5) to include with each match, from
|
|
the same container. Only with `container` set.
|
|
limit: matches to return, default 50, max 200.
|
|
|
|
Lines longer than 2000 characters keep both ends, and the middle is
|
|
replaced by a marker saying how much was cut.
|
|
"""
|
|
since, key = lookback(range)
|
|
limit = clamp_limit(limit, 50, 200)
|
|
context = max(0, min(int(context or 0), 5))
|
|
if context and not container:
|
|
raise ValueError("context needs `container`: it shows lines around a match within one container's log.")
|
|
if stream and stream not in ("stdout", "stderr"):
|
|
raise ValueError(f"stream {stream!r} is not 'stdout' or 'stderr'.")
|
|
before_dt = None
|
|
if before:
|
|
try:
|
|
before_dt = datetime.fromisoformat(before.replace("Z", "+00:00"))
|
|
except ValueError as exc:
|
|
raise ValueError(f"before {before!r} is not an ISO-8601 time; pass a response's next_before.") from exc
|
|
async with db_session() as db:
|
|
host_id = name = removed = service = tasks = None
|
|
if host:
|
|
hosts = list((await db.execute(select(Host))).scalars())
|
|
host_id = pick(host, hosts, kind="host", list_tool="list_hosts").id
|
|
if container:
|
|
service, tasks = await _as_service(db, container.strip(), host_id)
|
|
if service is None:
|
|
c = await _pick_container(db, container, host, removed=True)
|
|
host_id, name = c.host_id, c.name
|
|
removed = c if isinstance(c, RemovedContainer) else None
|
|
rows = await docker_svc.search_logs(db, since=since, host_id=host_id, name=name,
|
|
containers=tasks, text=text, stream=stream,
|
|
before=before_dt, limit=limit + 1)
|
|
more = len(rows) > limit
|
|
rows = rows[:limit]
|
|
host_names = await _host_names(db)
|
|
matches = []
|
|
for r in rows:
|
|
m = {"at": iso(r.ts), "container_name": r.container_name, "host_id": r.host_id,
|
|
"host_name": host_names.get(r.host_id), "stream": r.stream, "line": cut_middle(r.line)}
|
|
if service:
|
|
m["slot"], m["task_id"] = docker_svc.task_of(service, r.container_name)
|
|
if context:
|
|
pre, post = await docker_svc.log_context(db, r.host_id, r.container_name, r.ts, context)
|
|
m["before"] = [cut_middle(x.line) for x in pre]
|
|
m["after"] = [cut_middle(x.line) for x in post]
|
|
matches.append(m)
|
|
out = {"range": key, "matches": matches, "returned": len(matches), "truncated": more}
|
|
if service:
|
|
out["service"] = service
|
|
out["tasks_searched"] = len(tasks)
|
|
if removed is not None:
|
|
out["container_removed"] = True
|
|
out["note"] = _removed_note(removed)
|
|
if removed.last_seen < since:
|
|
out["note"] += " That is before this range; pass a longer `range`."
|
|
if more:
|
|
# Full precision: `at` is cut to the second for reading, and a cursor
|
|
# cut the same way would skip lines logged earlier in that second.
|
|
out["next_before"] = rows[-1].ts.isoformat()
|
|
out["more"] = "More matches are older; pass next_before as `before`, or narrow with `text`."
|
|
return out
|
|
|
|
|
|
async def _as_service(db, want: str, host_id: str | None):
|
|
"""(service, its task containers) when `want` names a Swarm service rather
|
|
than a container (#5468): every task container Steward holds anything for,
|
|
live and removed, kept to `host_id` when given (none there reads as no
|
|
lines, not as a missing name). (None, None) when it is not a service, or
|
|
a container carries that exact name, which keeps precedence."""
|
|
if not want:
|
|
return None, None
|
|
exact = (await db.execute(select(DockerContainer.name).where(
|
|
DockerContainer.name == want).limit(1))).first()
|
|
if exact is not None:
|
|
return None, None
|
|
tasks = await docker_svc.service_containers(db, want)
|
|
if not tasks:
|
|
return None, None
|
|
return want, [(h, n) for h, n in tasks if host_id is None or h == host_id]
|
|
|
|
|
|
async def get_docker_disk_usage(host: str = "") -> dict:
|
|
"""Docker disk usage per host: images, containers, volumes and build cache, in bytes.
|
|
|
|
host: a host's name or id; empty for every host. `images_reclaimable_bytes`
|
|
is what pruning unused images would free. An admin can prune from
|
|
Steward's Docker → Disk page.
|
|
"""
|
|
async with db_session() as db:
|
|
summaries = list((await db.execute(select(DockerDiskUsage))).scalars())
|
|
host_names = await _host_names(db)
|
|
if host:
|
|
hosts = list((await db.execute(select(Host))).scalars())
|
|
hid = pick(host, hosts, kind="host", list_tool="list_hosts").id
|
|
summaries = [s for s in summaries if s.host_id == hid]
|
|
rows = [
|
|
{
|
|
"host_id": s.host_id, "host_name": host_names.get(s.host_id),
|
|
"images_total": s.images_total, "images_active": s.images_active,
|
|
"images_bytes": s.images_size, "images_reclaimable_bytes": s.images_reclaimable,
|
|
"containers_count": s.containers_count, "containers_bytes": s.containers_size,
|
|
"volumes_count": s.volumes_count, "volumes_bytes": s.volumes_size,
|
|
"build_cache_bytes": s.build_cache_size,
|
|
"reported_at": iso(s.updated_at), "reported_age_s": age_s(s.updated_at),
|
|
}
|
|
for s in summaries
|
|
]
|
|
rows.sort(key=lambda r: -(r["images_reclaimable_bytes"] or 0))
|
|
return {"hosts": rows, "total": len(rows)}
|
|
|
|
|
|
PRUNE_BUILTIN_PLAYBOOK = "maintenance/docker_prune.yml"
|
|
_STOPPED = ("exited", "created", "dead")
|
|
|
|
|
|
def prune_preview(what: str, usage, stopped: int) -> dict:
|
|
"""What a prune of `what` would remove, from the host's latest disk report."""
|
|
out: dict = {}
|
|
if what in ("containers", "system"):
|
|
out["stopped_containers"] = stopped
|
|
if what == "images":
|
|
out["unused_images_bytes"] = usage.images_reclaimable if usage else None
|
|
if what == "system":
|
|
out["dangling_images"] = "yes (unused but tagged images stay)"
|
|
out["build_cache_bytes"] = usage.build_cache_size if usage else None
|
|
out["unused_networks"] = "yes"
|
|
return out
|
|
|
|
|
|
async def prune_docker(host: str, what: str, check: bool = True) -> dict:
|
|
"""Reclaim Docker disk on a host. Previews unless check=false.
|
|
|
|
host: the Steward host's name or id. It needs a linked Ansible target
|
|
(list_inventory shows the links); the prune runs through Ansible as
|
|
Steward's built-in docker_prune playbook.
|
|
what: "containers" removes stopped containers; "images" removes ALL
|
|
unused images, not only dangling ones; "system" is `docker system
|
|
prune -f`: stopped containers, dangling images, unused networks and
|
|
build cache. Volumes are never pruned.
|
|
check: true (default) starts nothing and returns `would_remove`, sized
|
|
from the host's latest Docker disk report. Ansible's own check mode
|
|
skips the prune commands, so a check-mode run would show nothing.
|
|
Pass false to run the prune.
|
|
|
|
Admin only, as on the Docker → Disk page. A real prune returns `run_id`
|
|
(follow it with wait_for_run) and is recorded in the audit log.
|
|
"""
|
|
what = (what or "").strip()
|
|
if what not in docker_svc.PRUNE_TARGETS:
|
|
raise ValueError(f"what {what!r} is not one of {', '.join(docker_svc.PRUNE_TARGETS)}.")
|
|
async with db_session() as db:
|
|
h = pick(host, (await db.execute(select(Host))).scalars(), kind="host", list_tool="list_hosts")
|
|
target = (await db.execute(
|
|
select(AnsibleTarget).where(AnsibleTarget.host_id == h.id))).scalars().first()
|
|
usage = await db.get(DockerDiskUsage, h.id)
|
|
stopped = sum(1 for c in await docker_svc.list_containers(db, h.id)
|
|
if (c.status or "").lower() in _STOPPED)
|
|
if target is None:
|
|
raise ValueError(f"host {h.name!r} has no linked Ansible target, so Steward cannot reach it "
|
|
"to prune. Link one under Ansible → Inventory. Nothing was run.")
|
|
preview = {
|
|
"host": h.name,
|
|
"what": what,
|
|
"ansible_target": target.name,
|
|
"would_remove": prune_preview(what, usage, stopped),
|
|
"disk_report_age_s": age_s(usage.updated_at) if usage else None,
|
|
}
|
|
if check:
|
|
return {**preview, "check": True,
|
|
"summary": f"Nothing was run. Pass check=false to prune {what} on {h.name}."}
|
|
caller = current_caller()
|
|
run, _src, err = await trigger_run(
|
|
mcp_app(), source_name=BUILTIN_SOURCE_NAME, playbook_path=PRUNE_BUILTIN_PLAYBOOK,
|
|
inventory_scope=f"steward:target:{target.id}",
|
|
params={"extra_vars_map": docker_svc.prune_extra_vars(what)},
|
|
triggered_by=caller.user_id, origin="mcp")
|
|
if err:
|
|
raise ValueError(f"the prune could not start: {err}.")
|
|
await audit_mcp("docker.prune.started", "host", h.name,
|
|
{"host_id": h.id, "what": what, "run_id": run.id})
|
|
return {**preview, "check": False, "run_id": run.id,
|
|
"next": f'wait_for_run(run_id="{run.id}")'}
|
|
|
|
|
|
async def mcp_events(db, host_id: str, host_name: str, since, until) -> list[dict]:
|
|
"""Container lifecycle events on a host in a window (get_anomaly): starts
|
|
(with the image, so a deploy shows), deaths with their exit, OOM kills,
|
|
restarts and health changes."""
|
|
from steward.mcp.tools._events import event
|
|
rows = (await db.execute(select(DockerEvent).where(
|
|
DockerEvent.host_id == host_id, DockerEvent.at >= since, DockerEvent.at <= until)
|
|
.order_by(DockerEvent.at).limit(500))).scalars()
|
|
return [event(e.at, "container", f"{e.container_name} {e.event}" + (f" ({e.detail})" if e.detail else ""),
|
|
subject=f"{host_name}/{e.container_name}") for e in rows]
|
|
|
|
|
|
def register_tools(register) -> None:
|
|
for fn in (list_containers, get_container, list_swarm_services,
|
|
search_container_logs, get_docker_disk_usage):
|
|
register(fn, role=UserRole.viewer, read_only=True)
|
|
register(prune_docker, role=UserRole.admin, read_only=False)
|