bvandeusenandClaude Opus 5.5
0945d78290
get_anomaly gives each occurrence against what is usual at that hour, the metric around the newest one with its normal, what else happened on the resource's host around each occurrence (alert transitions, Ansible runs, other anomalies there and a count elsewhere, plus plugins' own events through a new mcp_events hook: the host agent's maintenance jobs and downtime, Docker's container starts, deaths and OOM kills), and how earlier anomalies on the same metric were judged. get_metric_normal reads the learned slots and the level that would be flagged. review_anomaly takes one id or several, all or nothing, with a note. get_anomaly_settings and (admin) set_anomaly_settings read and change the review window and each normal's floor, as Settings → Thresholds does. metric_series.points takes an optional `until`; core.alerts gains host_of_resource; floor_setting names its floor. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
771 lines
37 KiB
Python
771 lines
37 KiB
Python
"""The docker plugin's MCP tools and its get_overview findings.
|
|
|
|
Registered through register_mcp_tools() in __init__.py, so they exist only
|
|
while the plugin is enabled. Reads go through service.py, the same functions
|
|
the plugin's pages use.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
from dataclasses import dataclass
|
|
from datetime import datetime, timedelta
|
|
|
|
from sqlalchemy import select
|
|
|
|
from steward.ansible.runner import trigger_run
|
|
from steward.ansible.sources import BUILTIN_SOURCE_NAME
|
|
from steward.mcp._context import current_caller, mcp_app
|
|
from steward.mcp.tools._audit import audit_mcp
|
|
from steward.mcp.tools._shape import (
|
|
age_s, bounded, clamp_limit, db_session, iso, lookback, now_utc, pick, rnd, summarize,
|
|
)
|
|
from steward.mcp.tools.overview import finding
|
|
from steward.models.ansible_inventory import AnsibleTarget
|
|
from steward.models.hosts import Host
|
|
from steward.models.users import UserRole
|
|
|
|
from . import service as docker_svc
|
|
from .service import PROBLEM_SAYS, PROBLEM_SEVERITY, PROBLEMS, container_problems, problem_since
|
|
from .dedup import dedup_by_container_id
|
|
from .models import DockerContainer, DockerDiskUsage, DockerEvent, DockerSwarmNode, DockerSwarmService
|
|
from .swarm_view import build_swarm_services, replaced_tasks, task_time
|
|
|
|
CRASH_LOOP_DEATHS = docker_svc.CRASH_LOOP_DEATHS
|
|
LOG_LINE_CHARS = 2000
|
|
|
|
|
|
# ── pure helpers ─────────────────────────────────────────────────────────────
|
|
|
|
def cut_middle(text: str, limit: int = LOG_LINE_CHARS) -> str:
|
|
"""Keep both ends of an over-long line and say how much of the middle went."""
|
|
if text is None or len(text) <= limit:
|
|
return text
|
|
keep = limit // 2
|
|
return f"{text[:keep]} [… {len(text) - 2 * keep} characters omitted …] {text[-keep:]}"
|
|
|
|
|
|
def container_row(c, host_names: dict, problems: list[str], now: datetime,
|
|
replaced: bool = False, since: datetime | None = None) -> dict:
|
|
running = (c.status or "").lower() == "running"
|
|
return {
|
|
"container_name": c.name,
|
|
"host_id": c.host_id,
|
|
"host_name": host_names.get(c.host_id),
|
|
"image": c.image,
|
|
"status": c.status,
|
|
"health": c.health,
|
|
"problems": problems,
|
|
# When the worst problem began, from the container's retained events.
|
|
"problem_since": iso(since),
|
|
"problem_age_s": age_s(since, now),
|
|
"exit_code": c.exit_code,
|
|
"oom_killed": bool(c.oom_killed),
|
|
"restart_count": c.restart_count,
|
|
"cpu_pct": rnd(c.cpu_pct),
|
|
"mem_pct": rnd(c.mem_pct),
|
|
"mem_usage_bytes": c.mem_usage_bytes,
|
|
"mem_limit_bytes": c.mem_limit_bytes,
|
|
"started_at": iso(c.started_at),
|
|
"uptime_s": age_s(c.started_at, now) if running and c.started_at else None,
|
|
"compose_project": c.compose_project,
|
|
"swarm_service": c.service_name,
|
|
# A swarm task Swarm has replaced: history, judged on its service.
|
|
"swarm_replaced": replaced if c.service_name else None,
|
|
"reported_at": iso(c.scraped_at),
|
|
"reported_age_s": age_s(c.scraped_at, now),
|
|
}
|
|
|
|
|
|
def _from_iso(text: str | None) -> datetime | None:
|
|
return datetime.fromisoformat(text.replace("Z", "+00:00")) if text else None
|
|
|
|
|
|
def container_findings(rows: list[dict]) -> list[dict]:
|
|
out = []
|
|
for r in rows:
|
|
if not r["problems"]:
|
|
continue
|
|
worst = r["problems"][0]
|
|
where = f" on {r['host_name']}" if r["host_name"] else ""
|
|
detail = f" (exit {r['exit_code']})" if r["exit_code"] not in (None, 0) else ""
|
|
out.append(finding(
|
|
PROBLEM_SEVERITY[worst], f"container_{worst}", r["container_name"],
|
|
f"container {r['container_name']}{where} {' and '.join(PROBLEM_SAYS[p] for p in r['problems'])}{detail}",
|
|
since=_from_iso(r["problem_since"]),
|
|
ids={"host_id": r["host_id"], "container_name": r["container_name"]},
|
|
next=f'get_container(container="{r["container_name"]}", host="{r["host_id"]}")'))
|
|
return out
|
|
|
|
|
|
def task_error_text(err: dict | None) -> str:
|
|
"""Swarm's own account of a task's end, as one clause."""
|
|
if not err:
|
|
return ""
|
|
why = err.get("error") or err.get("message") or err.get("state") or "unknown"
|
|
code = f", exit {err['exit_code']}" if err.get("exit_code") not in (None, 0) else ""
|
|
where = f" on {err['node']}" if err.get("node") else ""
|
|
return f"{why}{code}{where}"
|
|
|
|
|
|
# Services that started failing within this long of each other read as one
|
|
# finding: they are usually one node's event (#5722).
|
|
TOGETHER = timedelta(minutes=5)
|
|
|
|
|
|
def service_findings(services: list[dict]) -> list[dict]:
|
|
"""Swarm services failing now: crash-looping, or short of replicas, each
|
|
naming Swarm's last task error. One back to all its replicas is history,
|
|
kept on the Docker page, not a finding (#5722). Several short of replicas
|
|
since within TOGETHER of each other are one finding naming them all."""
|
|
out, short = [], []
|
|
for s in services:
|
|
replicas = f"{s['running']} of {s['desired']} replicas running"
|
|
last = task_error_text(s.get("last_error"))
|
|
tail = f"; last task error: {last}" if last else ""
|
|
ids = {"service_name": s["name"]}
|
|
since = s.get("trouble_since")
|
|
if s.get("crash_looping"):
|
|
out.append(finding(
|
|
"critical", "swarm_service_crash_looping", s["name"],
|
|
f"swarm service {s['name']} keeps failing: {s['failures_1h']} tasks failed in the "
|
|
f"last hour, {replicas}{tail}", since=since, ids=ids,
|
|
next="list_swarm_services()"))
|
|
elif s["state"] != "healthy":
|
|
short.append((s, replicas, tail))
|
|
for group in _together(short):
|
|
if len(group) == 1:
|
|
s, replicas, tail = group[0]
|
|
out.append(finding(
|
|
"critical" if s["state"] == "down" else "warning", f"swarm_service_{s['state']}",
|
|
s["name"], f"swarm service {s['name']} has {replicas}{tail}",
|
|
since=s.get("trouble_since"), ids={"service_name": s["name"]},
|
|
next="list_swarm_services()"))
|
|
continue
|
|
names = [s["name"] for s, _r, _t in group]
|
|
down = any(s["state"] == "down" for s, _r, _t in group)
|
|
out.append(finding(
|
|
"critical" if down else "warning", "swarm_services_short", f"{len(group)} swarm services",
|
|
f"{len(group)} swarm services started failing together and are short of replicas: "
|
|
+ ", ".join(f"{s['name']} ({r})" for s, r, _t in group),
|
|
since=min(s["trouble_since"] for s, _r, _t in group),
|
|
ids={"service_names": names}, next="list_swarm_services()"))
|
|
return out
|
|
|
|
|
|
def _together(short: list[tuple]) -> list[list[tuple]]:
|
|
"""Short services grouped by when they started failing: each group's
|
|
starts lie within TOGETHER of the one before. One without a start stands alone."""
|
|
timed = sorted((x for x in short if x[0].get("trouble_since")), key=lambda x: x[0]["trouble_since"])
|
|
groups: list[list[tuple]] = []
|
|
for x in timed:
|
|
if groups and x[0]["trouble_since"] - groups[-1][-1][0]["trouble_since"] <= TOGETHER:
|
|
groups[-1].append(x)
|
|
else:
|
|
groups.append([x])
|
|
return groups + [[x] for x in short if not x[0].get("trouble_since")]
|
|
|
|
|
|
# ── loading ──────────────────────────────────────────────────────────────────
|
|
|
|
async def _host_names(db) -> dict[str, str]:
|
|
return dict((await db.execute(select(Host.id, Host.name))).all())
|
|
|
|
|
|
def _crash_loop_deaths() -> int:
|
|
return docker_svc.crash_loop_deaths(mcp_app().config)
|
|
|
|
|
|
async def _replaced(db, containers) -> set[tuple]:
|
|
services = list((await db.execute(select(DockerSwarmService))).scalars())
|
|
return replaced_tasks(containers, services)
|
|
|
|
|
|
async def _rows(db, now: datetime) -> list[dict]:
|
|
"""Every container with its problems. A swarm task Swarm replaced has none:
|
|
its failure, if it was one, belongs to its service (service_findings)."""
|
|
host_names = await _host_names(db)
|
|
return [container_row(r["container"], host_names, r["problems"], now,
|
|
replaced=r["replaced"], since=r["since"])
|
|
for r in await docker_svc.ranked_containers(db, now, _crash_loop_deaths())]
|
|
|
|
|
|
async def _swarm(db, now: datetime):
|
|
containers = dedup_by_container_id(await docker_svc.list_containers(db))
|
|
services = list((await db.execute(select(DockerSwarmService))).scalars())
|
|
nodes = list((await db.execute(select(DockerSwarmNode))).scalars())
|
|
host_names = await _host_names(db)
|
|
node_hostname = {n.node_id: n.hostname for n in nodes if n.hostname}
|
|
built = build_swarm_services(containers, services, node_hostname, host_names,
|
|
now=now, crash_loop_deaths=_crash_loop_deaths())
|
|
return built["services"], nodes, host_names
|
|
|
|
|
|
def unique_nodes(nodes: list) -> list:
|
|
"""Each swarm node once: every manager reports them all, so keep the
|
|
newest report of each."""
|
|
seen, out = set(), []
|
|
for n in sorted(nodes, key=lambda n: n.updated_at, reverse=True):
|
|
if n.node_id not in seen:
|
|
seen.add(n.node_id)
|
|
out.append(n)
|
|
return out
|
|
|
|
|
|
async def _maintaining(db) -> dict[str, str]:
|
|
"""{host name, lower case: job state} for hosts with a maintenance job
|
|
running (#5835): a node drained or rebooting for one is expected. The
|
|
host agent is always loaded; None-safe all the same."""
|
|
from steward.core.plugin_manager import loaded_plugin_submodule
|
|
models = loaded_plugin_submodule("host_agent", "models")
|
|
maint = loaded_plugin_submodule("host_agent", "maintenance")
|
|
if models is None or maint is None:
|
|
return {}
|
|
job = models.HostMaintenanceJob
|
|
rows = await db.execute(select(Host.name, job.state).join(Host, Host.id == job.host_id)
|
|
.where(job.state.in_(maint.ACTIVE)))
|
|
return {name.lower(): state for name, state in rows}
|
|
|
|
|
|
NODE_AVAILABILITY_SAYS = {"drain": "drained", "pause": "paused"}
|
|
|
|
|
|
def node_findings(nodes: list, maintaining: dict[str, str]) -> list[dict]:
|
|
"""Swarm nodes down now (#5835): status not ready. A node whose host has
|
|
a maintenance job running is rebooting on purpose and isn't named.
|
|
|
|
Drained alone is not a finding: the operator keeps a manager drained on
|
|
purpose (#5777), and a drain a failed maintenance job left behind is that
|
|
job's finding."""
|
|
out = []
|
|
for n in unique_nodes(nodes):
|
|
name = n.hostname or n.node_id
|
|
if n.status == "ready" or name.lower() in maintaining:
|
|
continue
|
|
drained = NODE_AVAILABILITY_SAYS.get(n.availability)
|
|
out.append(finding(
|
|
"critical", "swarm_node_down", name,
|
|
f"swarm {n.role} {name} is {n.status}" + (f" and {drained}" if drained else ""),
|
|
ids={"node_id": n.node_id}, next="list_swarm_services()"))
|
|
return out
|
|
|
|
|
|
async def mcp_overview(db) -> list[dict]:
|
|
"""get_overview findings: failing containers, under-replicated swarm
|
|
services, and swarm nodes that are down."""
|
|
now = now_utc()
|
|
rows = await _rows(db, now)
|
|
services, nodes, _ = await _swarm(db, now)
|
|
return (container_findings(rows) + service_findings(services)
|
|
+ node_findings(nodes, await _maintaining(db)))
|
|
|
|
|
|
# ── tools ────────────────────────────────────────────────────────────────────
|
|
|
|
async def list_containers(host: str = "", status: str = "", problem: str = "",
|
|
compose_project: str = "", limit: int = 100) -> dict:
|
|
"""Docker containers across every host that runs a Steward agent with Docker.
|
|
|
|
host: a host's name or id (list_hosts); empty for all hosts.
|
|
status: Docker's state — "running", "exited", "restarting", "paused",
|
|
"created" or "dead"; empty for all.
|
|
problem: "any" for containers with something wrong, or one of
|
|
"restarting", "crash_looping" (died repeatedly within the last hour;
|
|
the count is set under Settings → Thresholds, default 3), "unhealthy"
|
|
(failing its healthcheck), "oom_killed", "exited_nonzero" (died with an
|
|
error in the last 24h; exit 137/143 without an OOM kill is a stop, not
|
|
an error); empty for no filter.
|
|
compose_project: only containers from this Compose project.
|
|
limit: default 100, max 500.
|
|
|
|
Containers with problems come first. Each row's `problems` lists what is
|
|
wrong, most severe first, and `problem_since` / `problem_age_s` say when
|
|
the worst of them began (null when Steward's retained events cannot say).
|
|
`reported_age_s` is how old the agent's last report is, so a large value
|
|
means the row may be stale.
|
|
|
|
A swarm task's container that Swarm has replaced (a rolling update, or a
|
|
retry after a failure) has `swarm_replaced: true` and no problems: Swarm
|
|
restarts a task as a new container, so failures are counted on the
|
|
service — list_swarm_services shows them.
|
|
"""
|
|
if problem and problem != "any" and problem not in PROBLEMS:
|
|
raise ValueError(f"problem {problem!r} is not 'any' or one of {', '.join(PROBLEMS)}.")
|
|
limit = clamp_limit(limit, 100, 500)
|
|
now = now_utc()
|
|
async with db_session() as db:
|
|
rows = await _rows(db, now)
|
|
if host:
|
|
hosts = list((await db.execute(select(Host))).scalars())
|
|
h = pick(host, hosts, kind="host", list_tool="list_hosts")
|
|
rows = [r for r in rows if r["host_id"] == h.id]
|
|
if status:
|
|
rows = [r for r in rows if (r["status"] or "").lower() == status.lower()]
|
|
if problem == "any":
|
|
rows = [r for r in rows if r["problems"]]
|
|
elif problem:
|
|
rows = [r for r in rows if problem in r["problems"]]
|
|
if compose_project:
|
|
rows = [r for r in rows if r["compose_project"] == compose_project]
|
|
rows.sort(key=lambda r: (not r["problems"], (r["status"] or "") != "running",
|
|
(r["host_name"] or "").lower(), r["container_name"]))
|
|
return bounded(rows, limit, key="containers",
|
|
more="Pass a larger `limit` (max 500), or filter by `host`, `status` or `problem`.")
|
|
|
|
|
|
@dataclass
|
|
class RemovedContainer:
|
|
"""A container Docker has removed, known only by its retained logs and events."""
|
|
host_id: str
|
|
name: str
|
|
last_seen: datetime
|
|
|
|
|
|
_REMOVED_HINT = ("It may have been removed: Swarm replaces a dead task with a new container under "
|
|
"a new name. list_containers(status=\"exited\") lists stopped containers, and "
|
|
"search_container_logs(host=...) searches a whole host.")
|
|
|
|
|
|
async def _pick_container(db, container: str, host: str, *, removed: bool = False):
|
|
"""The container named `container`: a live one, else (with `removed`) one
|
|
Docker has removed whose logs or events Steward still holds, returned as a
|
|
RemovedContainer. Exact names beat partial ones, and live beats removed."""
|
|
containers = await docker_svc.list_containers(db)
|
|
host_id = None
|
|
if host:
|
|
hosts = list((await db.execute(select(Host))).scalars())
|
|
host_id = pick(host, hosts, kind="host", list_tool="list_hosts").id
|
|
containers = [c for c in containers if c.host_id == host_id]
|
|
want = (container or "").strip()
|
|
if not want:
|
|
raise ValueError("pass a container name; list_containers lists them.")
|
|
live = {(c.host_id, c.name) for c in containers}
|
|
|
|
def _one(found, what):
|
|
if len(found) > 1:
|
|
raise ValueError(f"{len(found)} hosts have {what} named {want!r}; pass `host` "
|
|
f"(one of: {', '.join(sorted(f[0] for f in found))}).")
|
|
return found[0] if found else None
|
|
|
|
same = [c for c in containers if c.name == want] or \
|
|
[c for c in containers if c.name.lower() == want.lower()]
|
|
if same:
|
|
return _one([(c.host_id, c) for c in same], "a container")[1]
|
|
if removed:
|
|
gone = [r for r in await docker_svc.retained_names(db, want, exact=True, host_id=host_id)
|
|
if (r[0], r[1]) not in live]
|
|
if gone:
|
|
return RemovedContainer(*_one(gone, "a removed container"))
|
|
# One more chance: a unique container whose name contains it (swarm task
|
|
# names look like "stack_svc.1.k2j…").
|
|
partial = [c for c in containers if want.lower() in c.name.lower()]
|
|
if len(partial) == 1:
|
|
return partial[0]
|
|
gone = []
|
|
if removed and not partial:
|
|
gone = [r for r in await docker_svc.retained_names(db, want, exact=False, host_id=host_id)
|
|
if (r[0], r[1]) not in live]
|
|
if len(gone) == 1:
|
|
return RemovedContainer(*gone[0])
|
|
names = sorted(c.name for c in partial)[:10] + [f"{r[1]} (removed)" for r in gone[:10]]
|
|
hint = f" Containers whose names contain it: {', '.join(names)}." if names else ""
|
|
services = sorted({c.service_name for c in partial if c.service_name})
|
|
if services:
|
|
hint += (f" Swarm services among them: {', '.join(services[:10])}; search_container_logs "
|
|
"takes a service's name and searches every task it has run.")
|
|
raise ValueError(f"no container named {want!r}{' on that host' if host else ''}.{hint} "
|
|
f"{'' if names else _REMOVED_HINT + ' '}list_containers lists them.")
|
|
|
|
|
|
def _removed_note(c: RemovedContainer) -> str:
|
|
return (f"{c.name} no longer exists: Docker removed it (Swarm replaces a failed task with a new "
|
|
f"container). This is what Steward retained; it was last seen {iso(c.last_seen)}.")
|
|
|
|
|
|
async def get_container(container: str, host: str = "", range: str = "24h", events: int = 20) -> dict:
|
|
"""One container in depth: state, resources over time and lifecycle events.
|
|
|
|
container: the container's name, as list_containers shows. A unique part of
|
|
a name also works (swarm task names are long).
|
|
host: the host's name or id. Needed only when several hosts run a
|
|
container with this name.
|
|
range: the window for `cpu_pct_history` and `mem_pct_history` (min, avg,
|
|
max, p95, last and a ≤60-point series) — "1h", "6h", "24h" (default),
|
|
"7d" or "30d". `cpu_pct` and `mem_pct` are the latest reported values.
|
|
events: how many lifecycle events (start, restart, stop, die, oom, health_change)
|
|
to return, newest first (default 20, max 100).
|
|
|
|
A container Docker has since removed (a dead Swarm task, a `docker rm`)
|
|
still resolves by its exact name, or by a part of it that matches one
|
|
removed container, while Steward retains its logs or events: the reply
|
|
has `removed: true`, `last_seen_at`, its history and its events, and no
|
|
live state. A live container has `removed: false`.
|
|
"""
|
|
since, key = lookback(range)
|
|
events = clamp_limit(events, 20, 100)
|
|
now = now_utc()
|
|
async with db_session() as db:
|
|
c = await _pick_container(db, container, host, removed=True)
|
|
if isinstance(c, RemovedContainer):
|
|
return await _removed_container(db, c, since, key, events, now)
|
|
gone = (c.host_id, c.name) in await _replaced(db, await docker_svc.list_containers(db))
|
|
failures = (await docker_svc.recent_failures_by_container(
|
|
db, now - timedelta(hours=24))).get((c.host_id, c.name), [])
|
|
host_names = await _host_names(db)
|
|
cpu = await docker_svc.container_points(db, c.host_id, c.name, "cpu_pct", since)
|
|
mem = await docker_svc.container_points(db, c.host_id, c.name, "mem_pct", since)
|
|
evs = await docker_svc.container_events(db, c.host_id, c.name, events)
|
|
sick = await docker_svc.unhealthy_since(db, [(c.host_id, c.name)])
|
|
hour_ago = now - timedelta(hours=1)
|
|
recent = sum(1 for e in failures if e.at >= hour_ago)
|
|
try:
|
|
ports = json.loads(c.ports_json or "[]")
|
|
except ValueError:
|
|
ports = []
|
|
problems = [] if gone else container_problems(c, recent, bool(failures), _crash_loop_deaths())
|
|
began = problem_since(problems, failures, hour_ago, sick.get((c.host_id, c.name)))
|
|
return {
|
|
**container_row(c, host_names, problems, now, replaced=gone, since=began),
|
|
"removed": False,
|
|
"container_id": c.container_id,
|
|
"ports": ports,
|
|
"net_rx_bytes": c.net_rx_bytes, "net_tx_bytes": c.net_tx_bytes,
|
|
"blk_read_bytes": c.blk_read_bytes, "blk_write_bytes": c.blk_write_bytes,
|
|
"range": key,
|
|
"cpu_pct_history": summarize(cpu),
|
|
"mem_pct_history": summarize(mem),
|
|
"events": _event_rows(evs, now),
|
|
"next": f'search_container_logs(container="{c.name}", host="{c.host_id}")',
|
|
}
|
|
|
|
|
|
def _event_rows(evs, now: datetime) -> list[dict]:
|
|
return [{"event": e.event, "detail": e.detail, "at": iso(e.at), "age_s": age_s(e.at, now)}
|
|
for e in evs]
|
|
|
|
|
|
async def _removed_container(db, c: RemovedContainer, since, key: str, events: int,
|
|
now: datetime) -> dict:
|
|
"""get_container for a container Docker has removed: what Steward kept."""
|
|
host_names = await _host_names(db)
|
|
cpu = await docker_svc.container_points(db, c.host_id, c.name, "cpu_pct", since)
|
|
mem = await docker_svc.container_points(db, c.host_id, c.name, "mem_pct", since)
|
|
evs = await docker_svc.container_events(db, c.host_id, c.name, events)
|
|
return {
|
|
"container_name": c.name, "host_id": c.host_id, "host_name": host_names.get(c.host_id),
|
|
"removed": True, "note": _removed_note(c),
|
|
"last_seen_at": iso(c.last_seen), "last_seen_age_s": age_s(c.last_seen, now),
|
|
"range": key,
|
|
"cpu_pct_history": summarize(cpu),
|
|
"mem_pct_history": summarize(mem),
|
|
"events": _event_rows(evs, now),
|
|
"next": f'search_container_logs(container="{c.name}", host="{c.host_id}", range="24h")',
|
|
}
|
|
|
|
|
|
async def list_swarm_services(limit: int = 100) -> dict:
|
|
"""Docker Swarm services with their replicas per node, and the swarm's nodes.
|
|
|
|
`state` is "healthy" (all desired replicas running), "degraded" (some) or
|
|
"down" (none). `replicas` are the service's current tasks: each names its
|
|
host and container status, and one marked `ghost` runs on a node with no
|
|
Steward agent, so Steward knows only how many tasks are placed there.
|
|
Tasks Swarm replaced (rolling updates, retries) are left out and counted
|
|
in `previous_tasks`.
|
|
|
|
From Swarm's own task records (a manager's agent 1.8.0 or later;
|
|
`tasks_known: false` and nulls otherwise):
|
|
- `failures_1h` / `failures_24h`: tasks that failed or were rejected.
|
|
Swarm retries a failed task as a new one, so this, not any container's
|
|
restart count, is how a crash loop shows. Swarm keeps about 5 old tasks
|
|
per replica, which bounds the count.
|
|
- `crash_looping`: failures_1h reached the crash-loop count set under
|
|
Settings → Thresholds (default 3).
|
|
- `last_error`: the newest task that failed, was rejected, or carries an
|
|
error while Swarm retries it — `error` is Swarm's own text (e.g. "No
|
|
such image", "task: non-zero exit (1)"), with `exit_code`, `node`, `at`.
|
|
- `trouble_since` / `trouble_age_s`: when the current trouble began — a
|
|
crash loop's first failure in the hour, a short service's first failed
|
|
task in its failing slots, a recovered service's first failure in 24h.
|
|
Swarm keeps few old tasks, so a long outage reads as a lower bound.
|
|
|
|
Services with a problem come first. `nodes` lists every swarm node once,
|
|
with its role (manager/worker, `leader`), availability (active, drain,
|
|
pause) and status (ready, down, unknown); `host_id` is the Steward host
|
|
of that name (get_host, get_maintenance_job), and `maintenance` the state
|
|
of a maintenance job running on it, which drains and reboots it on
|
|
purpose (null when none).
|
|
limit: default 100, max 500.
|
|
"""
|
|
limit = clamp_limit(limit, 100, 500)
|
|
now = now_utc()
|
|
async with db_session() as db:
|
|
services, nodes, host_names = await _swarm(db, now)
|
|
maintaining = await _maintaining(db)
|
|
host_ids = {name.lower(): hid for hid, name in host_names.items()}
|
|
node_rows = [{"node_id": n.node_id, "hostname": n.hostname, "role": n.role,
|
|
"leader": n.leader, "availability": n.availability, "status": n.status,
|
|
"host_id": host_ids.get((n.hostname or "").lower()),
|
|
"maintenance": maintaining.get((n.hostname or "").lower()),
|
|
"reported_by": host_names.get(n.host_id)}
|
|
for n in unique_nodes(nodes)]
|
|
node_rows.sort(key=lambda n: n["hostname"].lower())
|
|
order = {"down": 0, "degraded": 1, "healthy": 2}
|
|
|
|
def _rank(s):
|
|
return (not s["crash_looping"], order.get(s["state"], 3), not s["failures_24h"], s["name"])
|
|
|
|
def _error_row(err):
|
|
if err is None:
|
|
return None
|
|
at = task_time(err)
|
|
return {"error": err["error"], "message": err["message"], "state": err["state"],
|
|
"exit_code": err["exit_code"], "node": err["node"],
|
|
"at": iso(at), "age_s": age_s(at, now) if at else None}
|
|
|
|
rows = [
|
|
{
|
|
"service_name": s["name"], "mode": s["mode"], "state": s["state"],
|
|
"running": s["running"], "desired": s["desired"],
|
|
"crash_looping": s["crash_looping"],
|
|
"failures_1h": s["failures_1h"], "failures_24h": s["failures_24h"],
|
|
"last_error": _error_row(s["last_error"]),
|
|
"trouble_since": iso(s["trouble_since"]),
|
|
"trouble_age_s": age_s(s["trouble_since"], now),
|
|
"previous_tasks": s["previous_tasks"], "tasks_known": s["tasks_known"],
|
|
"replicas": [
|
|
{"ghost": True, "host": r["host"], "count": r["count"]} if r["ghost"] else
|
|
{"ghost": False, "host": r["host"], "host_id": r["host_id"], "container_name": r["name"],
|
|
"status": r["status"], "health": r["health"], "restart_count": r["restart_count"],
|
|
"cpu_pct": rnd(r["cpu_pct"]), "mem_pct": rnd(r["mem_pct"])}
|
|
for r in s["replicas"]
|
|
],
|
|
}
|
|
for s in sorted(services, key=_rank)
|
|
]
|
|
return {"nodes": node_rows, **bounded(rows, limit, key="services")}
|
|
|
|
|
|
async def search_container_logs(container: str = "", host: str = "", text: str = "",
|
|
range: str = "1h", stream: str = "", before: str = "",
|
|
context: int = 0, limit: int = 50) -> dict:
|
|
"""Search the container log lines Steward has retained, newest first.
|
|
|
|
container: one container's name (as list_containers shows), or a Swarm
|
|
service's name (as list_swarm_services shows) to search every task
|
|
container the service has run, live and removed, so a search reaches
|
|
back across deploys; each match then has the `slot` and `task_id` that
|
|
wrote it. Empty searches every container (narrow with `host`). A
|
|
container Docker has removed still works while its lines are retained:
|
|
the reply then has `container_removed: true` and a `note` saying when it
|
|
was last seen.
|
|
host: a host's name or id; empty for all hosts.
|
|
text: case-insensitive substring to match; empty matches every line.
|
|
range: how far back — "1h" (default), "6h", "24h", "7d" or "30d". Logs are
|
|
kept per container up to a size and age limit, so older lines may be gone.
|
|
stream: "stdout" or "stderr"; empty for both.
|
|
before: an ISO time; returns only lines older than it. Pass a response's
|
|
`next_before` to page further back.
|
|
context: lines of surrounding log (0-5) to include with each match, from
|
|
the same container. Only with `container` set.
|
|
limit: matches to return, default 50, max 200.
|
|
|
|
Lines longer than 2000 characters keep both ends, and the middle is
|
|
replaced by a marker saying how much was cut.
|
|
"""
|
|
since, key = lookback(range)
|
|
limit = clamp_limit(limit, 50, 200)
|
|
context = max(0, min(int(context or 0), 5))
|
|
if context and not container:
|
|
raise ValueError("context needs `container`: it shows lines around a match within one container's log.")
|
|
if stream and stream not in ("stdout", "stderr"):
|
|
raise ValueError(f"stream {stream!r} is not 'stdout' or 'stderr'.")
|
|
before_dt = None
|
|
if before:
|
|
try:
|
|
before_dt = datetime.fromisoformat(before.replace("Z", "+00:00"))
|
|
except ValueError as exc:
|
|
raise ValueError(f"before {before!r} is not an ISO-8601 time; pass a response's next_before.") from exc
|
|
async with db_session() as db:
|
|
host_id = name = removed = service = tasks = None
|
|
if host:
|
|
hosts = list((await db.execute(select(Host))).scalars())
|
|
host_id = pick(host, hosts, kind="host", list_tool="list_hosts").id
|
|
if container:
|
|
service, tasks = await _as_service(db, container.strip(), host_id)
|
|
if service is None:
|
|
c = await _pick_container(db, container, host, removed=True)
|
|
host_id, name = c.host_id, c.name
|
|
removed = c if isinstance(c, RemovedContainer) else None
|
|
rows = await docker_svc.search_logs(db, since=since, host_id=host_id, name=name,
|
|
containers=tasks, text=text, stream=stream,
|
|
before=before_dt, limit=limit + 1)
|
|
more = len(rows) > limit
|
|
rows = rows[:limit]
|
|
host_names = await _host_names(db)
|
|
matches = []
|
|
for r in rows:
|
|
m = {"at": iso(r.ts), "container_name": r.container_name, "host_id": r.host_id,
|
|
"host_name": host_names.get(r.host_id), "stream": r.stream, "line": cut_middle(r.line)}
|
|
if service:
|
|
m["slot"], m["task_id"] = docker_svc.task_of(service, r.container_name)
|
|
if context:
|
|
pre, post = await docker_svc.log_context(db, r.host_id, r.container_name, r.ts, context)
|
|
m["before"] = [cut_middle(x.line) for x in pre]
|
|
m["after"] = [cut_middle(x.line) for x in post]
|
|
matches.append(m)
|
|
out = {"range": key, "matches": matches, "returned": len(matches), "truncated": more}
|
|
if service:
|
|
out["service"] = service
|
|
out["tasks_searched"] = len(tasks)
|
|
if removed is not None:
|
|
out["container_removed"] = True
|
|
out["note"] = _removed_note(removed)
|
|
if removed.last_seen < since:
|
|
out["note"] += " That is before this range; pass a longer `range`."
|
|
if more:
|
|
# Full precision: `at` is cut to the second for reading, and a cursor
|
|
# cut the same way would skip lines logged earlier in that second.
|
|
out["next_before"] = rows[-1].ts.isoformat()
|
|
out["more"] = "More matches are older; pass next_before as `before`, or narrow with `text`."
|
|
return out
|
|
|
|
|
|
async def _as_service(db, want: str, host_id: str | None):
|
|
"""(service, its task containers) when `want` names a Swarm service rather
|
|
than a container (#5468): every task container Steward holds anything for,
|
|
live and removed, kept to `host_id` when given (none there reads as no
|
|
lines, not as a missing name). (None, None) when it is not a service, or
|
|
a container carries that exact name, which keeps precedence."""
|
|
if not want:
|
|
return None, None
|
|
exact = (await db.execute(select(DockerContainer.name).where(
|
|
DockerContainer.name == want).limit(1))).first()
|
|
if exact is not None:
|
|
return None, None
|
|
tasks = await docker_svc.service_containers(db, want)
|
|
if not tasks:
|
|
return None, None
|
|
return want, [(h, n) for h, n in tasks if host_id is None or h == host_id]
|
|
|
|
|
|
async def get_docker_disk_usage(host: str = "") -> dict:
|
|
"""Docker disk usage per host: images, containers, volumes and build cache, in bytes.
|
|
|
|
host: a host's name or id; empty for every host. `images_reclaimable_bytes`
|
|
is what pruning unused images would free. An admin can prune from
|
|
Steward's Docker → Disk page.
|
|
"""
|
|
async with db_session() as db:
|
|
summaries = list((await db.execute(select(DockerDiskUsage))).scalars())
|
|
host_names = await _host_names(db)
|
|
if host:
|
|
hosts = list((await db.execute(select(Host))).scalars())
|
|
hid = pick(host, hosts, kind="host", list_tool="list_hosts").id
|
|
summaries = [s for s in summaries if s.host_id == hid]
|
|
rows = [
|
|
{
|
|
"host_id": s.host_id, "host_name": host_names.get(s.host_id),
|
|
"images_total": s.images_total, "images_active": s.images_active,
|
|
"images_bytes": s.images_size, "images_reclaimable_bytes": s.images_reclaimable,
|
|
"containers_count": s.containers_count, "containers_bytes": s.containers_size,
|
|
"volumes_count": s.volumes_count, "volumes_bytes": s.volumes_size,
|
|
"build_cache_bytes": s.build_cache_size,
|
|
"reported_at": iso(s.updated_at), "reported_age_s": age_s(s.updated_at),
|
|
}
|
|
for s in summaries
|
|
]
|
|
rows.sort(key=lambda r: -(r["images_reclaimable_bytes"] or 0))
|
|
return {"hosts": rows, "total": len(rows)}
|
|
|
|
|
|
PRUNE_BUILTIN_PLAYBOOK = "maintenance/docker_prune.yml"
|
|
_STOPPED = ("exited", "created", "dead")
|
|
|
|
|
|
def prune_preview(what: str, usage, stopped: int) -> dict:
|
|
"""What a prune of `what` would remove, from the host's latest disk report."""
|
|
out: dict = {}
|
|
if what in ("containers", "system"):
|
|
out["stopped_containers"] = stopped
|
|
if what == "images":
|
|
out["unused_images_bytes"] = usage.images_reclaimable if usage else None
|
|
if what == "system":
|
|
out["dangling_images"] = "yes (unused but tagged images stay)"
|
|
out["build_cache_bytes"] = usage.build_cache_size if usage else None
|
|
out["unused_networks"] = "yes"
|
|
return out
|
|
|
|
|
|
async def prune_docker(host: str, what: str, check: bool = True) -> dict:
|
|
"""Reclaim Docker disk on a host. Previews unless check=false.
|
|
|
|
host: the Steward host's name or id. It needs a linked Ansible target
|
|
(list_inventory shows the links); the prune runs through Ansible as
|
|
Steward's built-in docker_prune playbook.
|
|
what: "containers" removes stopped containers; "images" removes ALL
|
|
unused images, not only dangling ones; "system" is `docker system
|
|
prune -f`: stopped containers, dangling images, unused networks and
|
|
build cache. Volumes are never pruned.
|
|
check: true (default) starts nothing and returns `would_remove`, sized
|
|
from the host's latest Docker disk report. Ansible's own check mode
|
|
skips the prune commands, so a check-mode run would show nothing.
|
|
Pass false to run the prune.
|
|
|
|
Admin only, as on the Docker → Disk page. A real prune returns `run_id`
|
|
(follow it with wait_for_run) and is recorded in the audit log.
|
|
"""
|
|
what = (what or "").strip()
|
|
if what not in docker_svc.PRUNE_TARGETS:
|
|
raise ValueError(f"what {what!r} is not one of {', '.join(docker_svc.PRUNE_TARGETS)}.")
|
|
async with db_session() as db:
|
|
h = pick(host, (await db.execute(select(Host))).scalars(), kind="host", list_tool="list_hosts")
|
|
target = (await db.execute(
|
|
select(AnsibleTarget).where(AnsibleTarget.host_id == h.id))).scalars().first()
|
|
usage = await db.get(DockerDiskUsage, h.id)
|
|
stopped = sum(1 for c in await docker_svc.list_containers(db, h.id)
|
|
if (c.status or "").lower() in _STOPPED)
|
|
if target is None:
|
|
raise ValueError(f"host {h.name!r} has no linked Ansible target, so Steward cannot reach it "
|
|
"to prune. Link one under Ansible → Inventory. Nothing was run.")
|
|
preview = {
|
|
"host": h.name,
|
|
"what": what,
|
|
"ansible_target": target.name,
|
|
"would_remove": prune_preview(what, usage, stopped),
|
|
"disk_report_age_s": age_s(usage.updated_at) if usage else None,
|
|
}
|
|
if check:
|
|
return {**preview, "check": True,
|
|
"summary": f"Nothing was run. Pass check=false to prune {what} on {h.name}."}
|
|
caller = current_caller()
|
|
run, _src, err = await trigger_run(
|
|
mcp_app(), source_name=BUILTIN_SOURCE_NAME, playbook_path=PRUNE_BUILTIN_PLAYBOOK,
|
|
inventory_scope=f"steward:target:{target.id}",
|
|
params={"extra_vars_map": docker_svc.prune_extra_vars(what)},
|
|
triggered_by=caller.user_id, origin="mcp")
|
|
if err:
|
|
raise ValueError(f"the prune could not start: {err}.")
|
|
await audit_mcp("docker.prune.started", "host", h.name,
|
|
{"host_id": h.id, "what": what, "run_id": run.id})
|
|
return {**preview, "check": False, "run_id": run.id,
|
|
"next": f'wait_for_run(run_id="{run.id}")'}
|
|
|
|
|
|
async def mcp_events(db, host_id: str, host_name: str, since, until) -> list[dict]:
|
|
"""Container lifecycle events on a host in a window (get_anomaly): starts
|
|
(with the image, so a deploy shows), deaths with their exit, OOM kills,
|
|
restarts and health changes."""
|
|
from steward.mcp.tools._events import event
|
|
rows = (await db.execute(select(DockerEvent).where(
|
|
DockerEvent.host_id == host_id, DockerEvent.at >= since, DockerEvent.at <= until)
|
|
.order_by(DockerEvent.at).limit(40))).scalars()
|
|
return [event(e.at, "container", f"{e.container_name} {e.event}" + (f" ({e.detail})" if e.detail else ""),
|
|
subject=f"{host_name}/{e.container_name}") for e in rows]
|
|
|
|
|
|
def register_tools(register) -> None:
|
|
for fn in (list_containers, get_container, list_swarm_services,
|
|
search_container_logs, get_docker_disk_usage):
|
|
register(fn, role=UserRole.viewer, read_only=True)
|
|
register(prune_docker, role=UserRole.admin, read_only=False)
|