bvandeusenandClaude Opus 5.5
0945d78290
get_anomaly gives each occurrence against what is usual at that hour, the metric around the newest one with its normal, what else happened on the resource's host around each occurrence (alert transitions, Ansible runs, other anomalies there and a count elsewhere, plus plugins' own events through a new mcp_events hook: the host agent's maintenance jobs and downtime, Docker's container starts, deaths and OOM kills), and how earlier anomalies on the same metric were judged. get_metric_normal reads the learned slots and the level that would be flagged. review_anomaly takes one id or several, all or nothing, with a note. get_anomaly_settings and (admin) set_anomaly_settings read and change the review window and each normal's floor, as Settings → Thresholds does. metric_series.points takes an optional `until`; core.alerts gains host_of_resource; floor_setting names its floor. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
328 lines
15 KiB
Python
328 lines
15 KiB
Python
"""The host_agent plugin's MCP tools and its get_overview findings."""
|
|
from __future__ import annotations
|
|
|
|
from datetime import timedelta
|
|
|
|
from sqlalchemy import select
|
|
|
|
import json
|
|
|
|
from steward.hosts import service as host_svc
|
|
from steward.mcp._context import current_caller, mcp_app
|
|
from steward.mcp.tools._audit import audit_mcp
|
|
from steward.mcp.tools._shape import age_s, clamp_limit, db_session, iso, now_utc, pick
|
|
from steward.mcp.tools.overview import finding
|
|
from steward.models.hosts import Host
|
|
from steward.models.users import UserRole
|
|
|
|
from . import cadence, downtime, maintenance
|
|
from .models import (HostAgentRegistration, HostDowntime, HostMaintenanceJob, HostMaintenancePlan,
|
|
HostMaintenanceRun)
|
|
|
|
# A registration this old that has never checked in is a failed install, not one in progress.
|
|
PENDING_GRACE = timedelta(minutes=30)
|
|
|
|
|
|
def agent_state(reg, now, stale_after_s: int) -> str:
|
|
"""The agent's state: reporting, stale (stopped checking in) or pending (never has)."""
|
|
if reg.last_seen_at is None:
|
|
return "pending"
|
|
return "reporting" if (now - reg.last_seen_at).total_seconds() <= stale_after_s else "stale"
|
|
|
|
|
|
async def _agents(db):
|
|
return (await db.execute(
|
|
select(HostAgentRegistration, Host)
|
|
.join(Host, Host.id == HostAgentRegistration.host_id)
|
|
.order_by(Host.name)
|
|
)).all()
|
|
|
|
|
|
def _updates(reg) -> dict | None:
|
|
"""A registration's pending updates and reboot, as list_agents says them (#5765)."""
|
|
if reg.updates_checked_at is None and reg.reboot_required is None:
|
|
return None
|
|
return {
|
|
"manager": reg.updates_manager, "pending": reg.updates_pending,
|
|
"security": reg.updates_security, "checked_at": iso(reg.updates_checked_at),
|
|
"error": reg.updates_error, "reboot_required": reg.reboot_required,
|
|
"reboot_required_since": iso(reg.reboot_required_since),
|
|
"reboot_packages": reg.reboot_packages.split(", ") if reg.reboot_packages else [],
|
|
}
|
|
|
|
|
|
async def list_agents() -> dict:
|
|
"""Every host agent Steward has registered, with its version and whether it is reporting.
|
|
|
|
`state` is "reporting", "stale" (it checked in before but its last
|
|
check-in is older than the stale window, `stale_after_s`) or "pending"
|
|
(registered, never checked in, usually an install that is still running
|
|
or that failed). Each row carries the agent version, kernel, distro, arch
|
|
and the IP the agent reports. get_host shows an agent's latest metrics.
|
|
|
|
`updates` is what the host's package manager has waiting, from the
|
|
agent's hourly check (agent ≥ 1.16.0): manager, pending, security (null
|
|
when the manager can't say), checked_at, error, and reboot_required with
|
|
since when and the packages that asked for it. null before the first check.
|
|
"""
|
|
now = now_utc()
|
|
stale_after = host_svc.agent_stale_after_seconds(mcp_app().config)
|
|
async with db_session() as db:
|
|
rows = await _agents(db)
|
|
agents = [
|
|
{
|
|
"host_id": h.id, "host_name": h.name,
|
|
"state": agent_state(reg, now, stale_after),
|
|
"last_seen_at": iso(reg.last_seen_at), "last_seen_age_s": age_s(reg.last_seen_at, now),
|
|
"agent_version": reg.agent_version, "kernel": reg.kernel, "distro": reg.distro,
|
|
"arch": reg.arch, "reported_ip": reg.host_ip,
|
|
"registered_at": iso(reg.created_at),
|
|
"updates": _updates(reg),
|
|
}
|
|
for reg, h in rows
|
|
]
|
|
counts = {s: sum(1 for a in agents if a["state"] == s) for s in ("reporting", "stale", "pending")}
|
|
return {"stale_after_s": stale_after, "counts": counts, "agents": agents, "total": len(agents)}
|
|
|
|
|
|
# ── maintenance (#5834): plans, runs, jobs, and starting one ─────────────────
|
|
|
|
JOBS_MAX = 50
|
|
# A failed job stays on the overview this long, unless a later one succeeded.
|
|
FAILED_JOB_DAYS = 7
|
|
|
|
|
|
async def _names(db, ids) -> dict[str, str]:
|
|
from steward.models.users import User
|
|
ids = {i for i in ids if i}
|
|
if not ids:
|
|
return {}
|
|
hosts = dict((await db.execute(select(Host.id, Host.name).where(Host.id.in_(ids)))).all())
|
|
users = dict((await db.execute(select(User.id, User.username).where(User.id.in_(ids)))).all())
|
|
return {**users, **hosts}
|
|
|
|
|
|
def _job_row(job, names: dict, now) -> dict:
|
|
deadline = maintenance.DEADLINES.get(job.state)
|
|
started = job.created_at if job.state == "queued" else job.state_at
|
|
return {
|
|
"job_id": job.id,
|
|
"host": names.get(job.host_id, job.host_id),
|
|
"state": job.state,
|
|
"label": maintenance.LABELS.get(job.state, job.state),
|
|
"summary": maintenance.summary(job),
|
|
"active": job.state in maintenance.ACTIVE,
|
|
"reboot_policy": job.reboot_policy,
|
|
"packages_updated": job.packages,
|
|
"reboot_needed": job.reboot_needed,
|
|
"rebooted_at": iso(job.back_at),
|
|
"swarm_node": job.node_id,
|
|
"drained_by": names.get(job.drain_host_id, job.drain_host_id),
|
|
"detail": job.detail,
|
|
"state_since": iso(job.state_at), "state_age_s": age_s(job.state_at, now),
|
|
"gives_up_at": iso(started + deadline) if deadline and started else None,
|
|
"started_by": names.get(job.created_by) if job.created_by else
|
|
("maintenance plan" if job.run_id else None),
|
|
"run_id": job.run_id,
|
|
"created_at": iso(job.created_at),
|
|
"finished_at": iso(job.finished_at),
|
|
}
|
|
|
|
|
|
async def list_maintenance_plans() -> dict:
|
|
"""Each maintenance plan: which inventory group it updates, when (weekly
|
|
slot, time zone, how long a run may keep starting hosts), its reboot
|
|
policy, whether it is enabled, its next slot, and its latest run.
|
|
|
|
A run updates the group's hosts one at a time; `latest_run` says how far
|
|
it got (`hosts_done` of `hosts`), which hosts it passed over and why
|
|
(`skipped`), and how it ended. get_maintenance_job shows one host's job.
|
|
Plans are created and edited on the Maintenance page.
|
|
"""
|
|
from steward.models.ansible_inventory import AnsibleGroup
|
|
now = now_utc()
|
|
async with db_session() as db:
|
|
plans = list((await db.execute(select(HostMaintenancePlan)
|
|
.order_by(HostMaintenancePlan.name))).scalars())
|
|
groups = dict((await db.execute(select(AnsibleGroup.id, AnsibleGroup.name))).all())
|
|
latest = {}
|
|
for plan in plans:
|
|
latest[plan.id] = (await db.execute(
|
|
select(HostMaintenanceRun).where(HostMaintenanceRun.plan_id == plan.id)
|
|
.order_by(HostMaintenanceRun.started_at.desc()).limit(1))).scalar_one_or_none()
|
|
names = await _names(db, [h for r in latest.values() if r
|
|
for h in json.loads(r.host_ids or "[]")
|
|
+ [x["host_id"] for x in json.loads(r.skipped or "[]")]])
|
|
out = []
|
|
for plan in plans:
|
|
run = latest[plan.id]
|
|
nxt = cadence.next_slot(plan, now) if plan.enabled else None
|
|
row = {
|
|
"plan_id": plan.id, "name": plan.name,
|
|
"group": groups.get(plan.group_id, plan.group_id),
|
|
"schedule": cadence.describe(plan),
|
|
"weekday": cadence.WEEKDAYS[plan.weekday], "at_time": plan.at_time,
|
|
"time_zone": plan.time_zone, "window_minutes": plan.window_minutes,
|
|
"reboot_policy": plan.reboot_policy, "enabled": plan.enabled,
|
|
"next_slot_at": iso(nxt), "last_slot_at": iso(plan.last_slot_at),
|
|
"latest_run": None,
|
|
}
|
|
if run:
|
|
hosts = json.loads(run.host_ids or "[]")
|
|
row["latest_run"] = {
|
|
"run_id": run.id, "state": run.state, "manual": run.manual,
|
|
"hosts": [names.get(h, h) for h in hosts],
|
|
"hosts_done": min(run.position, len(hosts)),
|
|
"skipped": [{"host": names.get(x["host_id"], x["host_id"]), "reason": x["reason"]}
|
|
for x in json.loads(run.skipped or "[]")],
|
|
"detail": run.detail,
|
|
"started_at": iso(run.started_at), "finished_at": iso(run.finished_at),
|
|
}
|
|
out.append(row)
|
|
return {"plans": out, "total": len(out)}
|
|
|
|
|
|
async def get_maintenance_job(host: str = "", job: str = "", limit: int = 10) -> dict:
|
|
"""A host's maintenance jobs, newest first (its running one first), or one
|
|
job by id.
|
|
|
|
host: a host's name or id (list_hosts); job: a job id (from here, or a
|
|
plan's run). One of the two.
|
|
limit: how many of the host's jobs (default 10, max 50).
|
|
|
|
A job is one host's update and, only when the update needs it, a reboot,
|
|
with its swarm node drained first and made active again after. `state`
|
|
moves queued → updating → [draining →] rebooting → [undraining →] done,
|
|
or failed / timed_out; `detail` says why a job failed. `summary` is the
|
|
host page's line ("Done: 12 packages updated, rebooted"). `gives_up_at`
|
|
is when an active job's current state times out.
|
|
"""
|
|
if bool(host.strip()) == bool(job.strip()):
|
|
raise ValueError("pass either `host` (a name or id) or `job` (a job id).")
|
|
now = now_utc()
|
|
limit = clamp_limit(limit, 10, JOBS_MAX)
|
|
async with db_session() as db:
|
|
if job.strip():
|
|
row = await db.get(HostMaintenanceJob, job.strip())
|
|
if row is None:
|
|
raise ValueError(f"no maintenance job {job.strip()!r}; "
|
|
"get_maintenance_job(host=...) lists a host's jobs.")
|
|
jobs = [row]
|
|
else:
|
|
h = pick(host, (await db.execute(select(Host))).scalars(),
|
|
kind="host", list_tool="list_hosts")
|
|
jobs = list((await db.execute(
|
|
select(HostMaintenanceJob).where(HostMaintenanceJob.host_id == h.id)
|
|
.order_by(HostMaintenanceJob.state.in_(maintenance.ACTIVE).desc(),
|
|
HostMaintenanceJob.created_at.desc())
|
|
.limit(limit + 1))).scalars())
|
|
names = await _names(db, [j.host_id for j in jobs] + [j.drain_host_id for j in jobs]
|
|
+ [j.created_by for j in jobs])
|
|
rows = [_job_row(j, names, now) for j in jobs[:limit]]
|
|
return {"jobs": rows, "total": len(rows), "truncated": len(jobs) > limit}
|
|
|
|
|
|
async def run_maintenance(host: str, reboot: str = "if_required") -> dict:
|
|
"""Update a host now, as the host page's Update now does: the job starts
|
|
with the agent's next post and get_maintenance_job follows it.
|
|
|
|
host: the host's name or id.
|
|
reboot: "if_required" (default) reboots only when the update says the
|
|
host needs it, draining its swarm node first and making it active after;
|
|
"never" updates and leaves any reboot for later.
|
|
|
|
Refused while the host already has a job running, or when its agent has
|
|
no maintenance helper yet (Update agent installs it). Admin only, as on
|
|
the page. Recorded in the audit log.
|
|
"""
|
|
if reboot not in ("if_required", "never"):
|
|
raise ValueError('reboot must be "if_required" or "never". Nothing was started.')
|
|
caller = current_caller()
|
|
async with db_session() as db, db.begin():
|
|
h = pick(host, (await db.execute(select(Host))).scalars(),
|
|
kind="host", list_tool="list_hosts")
|
|
reg = (await db.execute(select(HostAgentRegistration).where(
|
|
HostAgentRegistration.host_id == h.id))).scalar_one_or_none()
|
|
try:
|
|
created = await maintenance.create_job(db, h, reg, caller.user_id, reboot)
|
|
except maintenance.CannotStart as exc:
|
|
raise ValueError(f"{h.name} can't be updated now: {exc}. Nothing was started.") from None
|
|
job_id, host_id, name = created.id, h.id, h.name
|
|
await audit_mcp("host_agent.maintenance", "host", host_id,
|
|
{"job_id": job_id, "reboot_policy": reboot})
|
|
return {"job_id": job_id, "host": name, "state": "queued", "reboot_policy": reboot,
|
|
"next": f'get_maintenance_job(job="{job_id}") follows it; it starts with the '
|
|
"agent's next post."}
|
|
|
|
|
|
async def _failed_jobs(db, now) -> list:
|
|
"""Each host's latest finished job, when it failed or timed out within
|
|
FAILED_JOB_DAYS (a later success clears it)."""
|
|
rows = (await db.execute(
|
|
select(HostMaintenanceJob, Host.name).join(Host, Host.id == HostMaintenanceJob.host_id)
|
|
.where(HostMaintenanceJob.state.in_(maintenance.FINISHED),
|
|
HostMaintenanceJob.finished_at >= now - timedelta(days=FAILED_JOB_DAYS))
|
|
.order_by(HostMaintenanceJob.finished_at.desc()))).all()
|
|
latest = {}
|
|
for j, name in rows:
|
|
latest.setdefault(j.host_id, (j, name))
|
|
return [(j, name) for j, name in latest.values() if j.state != "done"]
|
|
|
|
|
|
async def mcp_overview(db) -> list[dict]:
|
|
"""Agents registered more than PENDING_GRACE ago that have never checked
|
|
in, and hosts whose latest maintenance job failed or timed out.
|
|
|
|
Agents that stopped reporting are already flagged by the core overview
|
|
from their metrics, so they are not repeated here.
|
|
"""
|
|
now = now_utc()
|
|
out = [
|
|
finding("warning", "agent_never_reported", h.name,
|
|
"host agent was registered but has never checked in; the install may have failed",
|
|
since=reg.created_at, ids={"host_id": h.id}, next="list_agents()")
|
|
for reg, h in await _agents(db)
|
|
if reg.last_seen_at is None and now - reg.created_at > PENDING_GRACE
|
|
]
|
|
out += [
|
|
finding("warning", "maintenance_failed", name,
|
|
f"maintenance {maintenance.LABELS[j.state].lower()}"
|
|
+ (f": {j.detail}" if j.detail else ""),
|
|
since=j.finished_at, ids={"host_id": j.host_id, "job_id": j.id},
|
|
next=f'get_maintenance_job(job="{j.id}")')
|
|
for j, name in await _failed_jobs(db, now)
|
|
]
|
|
return out
|
|
|
|
|
|
async def mcp_events(db, host_id: str, host_name: str, since, until) -> list[dict]:
|
|
"""What the host agent knows happened on a host in a window (get_anomaly):
|
|
maintenance jobs, and downtime (reboots, agent stops, lost contact)."""
|
|
from steward.mcp.tools._events import event
|
|
out = []
|
|
jobs = (await db.execute(select(HostMaintenanceJob).where(
|
|
HostMaintenanceJob.host_id == host_id, HostMaintenanceJob.created_at <= until,
|
|
(HostMaintenanceJob.finished_at.is_(None)) | (HostMaintenanceJob.finished_at >= since))
|
|
.order_by(HostMaintenanceJob.created_at).limit(20))).scalars()
|
|
for job in jobs:
|
|
out.append(event(job.created_at, "maintenance", f"Maintenance: {maintenance.summary(job)}",
|
|
until=job.finished_at, ids={"job_id": job.id}))
|
|
now = now_utc()
|
|
rows = (await db.execute(select(HostDowntime).where(
|
|
HostDowntime.host_id == host_id, HostDowntime.started_at <= until,
|
|
(HostDowntime.ended_at.is_(None)) | (HostDowntime.ended_at >= since))
|
|
.order_by(HostDowntime.started_at).limit(20))).scalars()
|
|
for row in rows:
|
|
seen = downtime.describe(row, now)
|
|
out.append(event(row.started_at, "downtime",
|
|
f"{seen['label']} ({int(seen['seconds'])} s{', still going' if seen['ongoing'] else ''})",
|
|
until=row.ended_at))
|
|
return out
|
|
|
|
|
|
def register_tools(register) -> None:
|
|
register(list_agents, role=UserRole.viewer, read_only=True)
|
|
register(list_maintenance_plans, role=UserRole.viewer, read_only=True)
|
|
register(get_maintenance_job, role=UserRole.viewer, read_only=True)
|
|
register(run_maintenance, role=UserRole.admin, read_only=False)
|