feat: the host agent is always on; the MCP sees and starts host maintenance (#5834)

#5834 "The host agent is always on, and the MCP sees a host's
maintenance", step 1 of milestone 520 "The MCP sees what Steward has
grown".

The host agent is a core system (operator, 2026-10-15: "it's very core
to the functionality of Steward as a whole and shouldn't be considered
something that can be toggled off"). ALWAYS_ON_PLUGINS in core settings
makes to_plugins_cfg report it enabled whatever its stored row says, so
it always loads and its MCP tools are always listed. Settings -> Plugins
shows it as "Built in: always on" with no switch, and a save keeps it
enabled. Its code and URLs stay under plugins/host_agent: every
installed agent posts to /plugins/host_agent/ingest.

Maintenance through the MCP:
- list_maintenance_plans: each plan's group, weekly slot, reboot policy,
  next slot, and its latest run (hosts, how far it got, who it skipped
  and why, how it ended).
- get_maintenance_job(host | job): a host's jobs, newest and running
  first, or one job: state, packages updated, reboot, drained by, why it
  failed, when its current state times out.
- run_maintenance(host, reboot="if_required" | "never"): admin, as the
  host page's Update now; refuses a busy host or one with no helper;
  audited "<user> via MCP".
- The overview names a host whose latest job in the last week failed or
  timed out.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

plugins/host_agent/mcp_tools.py

@@ -5,14 +5,19 @@ from datetime import timedelta
 
 from sqlalchemy import select
 
+import json
+
 from steward.hosts import service as host_svc
-from steward.mcp._context import mcp_app
-from steward.mcp.tools._shape import age_s, db_session, iso, now_utc
+from steward.mcp._context import current_caller, mcp_app
+from steward.mcp.tools._audit import audit_mcp
+from steward.mcp.tools._shape import age_s, clamp_limit, db_session, iso, now_utc, pick
 from steward.mcp.tools.overview import finding
 from steward.models.hosts import Host
 from steward.models.users import UserRole
 
-from .models import HostAgentRegistration
+from . import cadence, maintenance
+from .models import (HostAgentRegistration, HostMaintenanceJob, HostMaintenancePlan,
+                     HostMaintenanceRun)
 
 # A registration this old that has never checked in is a failed install, not one in progress.
 PENDING_GRACE = timedelta(minutes=30)
@@ -80,21 +85,218 @@ async def list_agents() -> dict:
     return {"stale_after_s": stale_after, "counts": counts, "agents": agents, "total": len(agents)}
 
 
+# ── maintenance (#5834): plans, runs, jobs, and starting one ─────────────────
+
+JOBS_MAX = 50
+# A failed job stays on the overview this long, unless a later one succeeded.
+FAILED_JOB_DAYS = 7
+
+
+async def _names(db, ids) -> dict[str, str]:
+    from steward.models.users import User
+    ids = {i for i in ids if i}
+    if not ids:
+        return {}
+    hosts = dict((await db.execute(select(Host.id, Host.name).where(Host.id.in_(ids)))).all())
+    users = dict((await db.execute(select(User.id, User.username).where(User.id.in_(ids)))).all())
+    return {**users, **hosts}
+
+
+def _job_row(job, names: dict, now) -> dict:
+    deadline = maintenance.DEADLINES.get(job.state)
+    started = job.created_at if job.state == "queued" else job.state_at
+    return {
+        "job_id": job.id,
+        "host": names.get(job.host_id, job.host_id),
+        "state": job.state,
+        "label": maintenance.LABELS.get(job.state, job.state),
+        "summary": maintenance.summary(job),
+        "active": job.state in maintenance.ACTIVE,
+        "reboot_policy": job.reboot_policy,
+        "packages_updated": job.packages,
+        "reboot_needed": job.reboot_needed,
+        "rebooted_at": iso(job.back_at),
+        "swarm_node": job.node_id,
+        "drained_by": names.get(job.drain_host_id, job.drain_host_id),
+        "detail": job.detail,
+        "state_since": iso(job.state_at), "state_age_s": age_s(job.state_at, now),
+        "gives_up_at": iso(started + deadline) if deadline and started else None,
+        "started_by": names.get(job.created_by) if job.created_by else
+                      ("maintenance plan" if job.run_id else None),
+        "run_id": job.run_id,
+        "created_at": iso(job.created_at),
+        "finished_at": iso(job.finished_at),
+    }
+
+
+async def list_maintenance_plans() -> dict:
+    """Each maintenance plan: which inventory group it updates, when (weekly
+    slot, time zone, how long a run may keep starting hosts), its reboot
+    policy, whether it is enabled, its next slot, and its latest run.
+
+    A run updates the group's hosts one at a time; `latest_run` says how far
+    it got (`hosts_done` of `hosts`), which hosts it passed over and why
+    (`skipped`), and how it ended. get_maintenance_job shows one host's job.
+    Plans are created and edited on the Maintenance page.
+    """
+    from steward.models.ansible_inventory import AnsibleGroup
+    now = now_utc()
+    async with db_session() as db:
+        plans = list((await db.execute(select(HostMaintenancePlan)
+                                       .order_by(HostMaintenancePlan.name))).scalars())
+        groups = dict((await db.execute(select(AnsibleGroup.id, AnsibleGroup.name))).all())
+        latest = {}
+        for plan in plans:
+            latest[plan.id] = (await db.execute(
+                select(HostMaintenanceRun).where(HostMaintenanceRun.plan_id == plan.id)
+                .order_by(HostMaintenanceRun.started_at.desc()).limit(1))).scalar_one_or_none()
+        names = await _names(db, [h for r in latest.values() if r
+                                  for h in json.loads(r.host_ids or "[]")
+                                  + [x["host_id"] for x in json.loads(r.skipped or "[]")]])
+    out = []
+    for plan in plans:
+        run = latest[plan.id]
+        nxt = cadence.next_slot(plan, now) if plan.enabled else None
+        row = {
+            "plan_id": plan.id, "name": plan.name,
+            "group": groups.get(plan.group_id, plan.group_id),
+            "schedule": cadence.describe(plan),
+            "weekday": cadence.WEEKDAYS[plan.weekday], "at_time": plan.at_time,
+            "time_zone": plan.time_zone, "window_minutes": plan.window_minutes,
+            "reboot_policy": plan.reboot_policy, "enabled": plan.enabled,
+            "next_slot_at": iso(nxt), "last_slot_at": iso(plan.last_slot_at),
+            "latest_run": None,
+        }
+        if run:
+            hosts = json.loads(run.host_ids or "[]")
+            row["latest_run"] = {
+                "run_id": run.id, "state": run.state, "manual": run.manual,
+                "hosts": [names.get(h, h) for h in hosts],
+                "hosts_done": min(run.position, len(hosts)),
+                "skipped": [{"host": names.get(x["host_id"], x["host_id"]), "reason": x["reason"]}
+                            for x in json.loads(run.skipped or "[]")],
+                "detail": run.detail,
+                "started_at": iso(run.started_at), "finished_at": iso(run.finished_at),
+            }
+        out.append(row)
+    return {"plans": out, "total": len(out)}
+
+
+async def get_maintenance_job(host: str = "", job: str = "", limit: int = 10) -> dict:
+    """A host's maintenance jobs, newest first (its running one first), or one
+    job by id.
+
+    host: a host's name or id (list_hosts); job: a job id (from here, or a
+    plan's run). One of the two.
+    limit: how many of the host's jobs (default 10, max 50).
+
+    A job is one host's update and, only when the update needs it, a reboot,
+    with its swarm node drained first and made active again after. `state`
+    moves queued → updating → [draining →] rebooting → [undraining →] done,
+    or failed / timed_out; `detail` says why a job failed. `summary` is the
+    host page's line ("Done: 12 packages updated, rebooted"). `gives_up_at`
+    is when an active job's current state times out.
+    """
+    if bool(host.strip()) == bool(job.strip()):
+        raise ValueError("pass either `host` (a name or id) or `job` (a job id).")
+    now = now_utc()
+    limit = clamp_limit(limit, 10, JOBS_MAX)
+    async with db_session() as db:
+        if job.strip():
+            row = await db.get(HostMaintenanceJob, job.strip())
+            if row is None:
+                raise ValueError(f"no maintenance job {job.strip()!r}; "
+                                 "get_maintenance_job(host=...) lists a host's jobs.")
+            jobs = [row]
+        else:
+            h = pick(host, (await db.execute(select(Host))).scalars(),
+                     kind="host", list_tool="list_hosts")
+            jobs = list((await db.execute(
+                select(HostMaintenanceJob).where(HostMaintenanceJob.host_id == h.id)
+                .order_by(HostMaintenanceJob.state.in_(maintenance.ACTIVE).desc(),
+                          HostMaintenanceJob.created_at.desc())
+                .limit(limit + 1))).scalars())
+        names = await _names(db, [j.host_id for j in jobs] + [j.drain_host_id for j in jobs]
+                             + [j.created_by for j in jobs])
+    rows = [_job_row(j, names, now) for j in jobs[:limit]]
+    return {"jobs": rows, "total": len(rows), "truncated": len(jobs) > limit}
+
+
+async def run_maintenance(host: str, reboot: str = "if_required") -> dict:
+    """Update a host now, as the host page's Update now does: the job starts
+    with the agent's next post and get_maintenance_job follows it.
+
+    host: the host's name or id.
+    reboot: "if_required" (default) reboots only when the update says the
+      host needs it, draining its swarm node first and making it active after;
+      "never" updates and leaves any reboot for later.
+
+    Refused while the host already has a job running, or when its agent has
+    no maintenance helper yet (Update agent installs it). Admin only, as on
+    the page. Recorded in the audit log.
+    """
+    if reboot not in ("if_required", "never"):
+        raise ValueError('reboot must be "if_required" or "never". Nothing was started.')
+    caller = current_caller()
+    async with db_session() as db, db.begin():
+        h = pick(host, (await db.execute(select(Host))).scalars(),
+                 kind="host", list_tool="list_hosts")
+        reg = (await db.execute(select(HostAgentRegistration).where(
+            HostAgentRegistration.host_id == h.id))).scalar_one_or_none()
+        try:
+            created = await maintenance.create_job(db, h, reg, caller.user_id, reboot)
+        except maintenance.CannotStart as exc:
+            raise ValueError(f"{h.name} can't be updated now: {exc}. Nothing was started.") from None
+        job_id, host_id, name = created.id, h.id, h.name
+    await audit_mcp("host_agent.maintenance", "host", host_id,
+                    {"job_id": job_id, "reboot_policy": reboot})
+    return {"job_id": job_id, "host": name, "state": "queued", "reboot_policy": reboot,
+            "next": f'get_maintenance_job(job="{job_id}") follows it; it starts with the '
+                    "agent's next post."}
+
+
+async def _failed_jobs(db, now) -> list:
+    """Each host's latest finished job, when it failed or timed out within
+    FAILED_JOB_DAYS (a later success clears it)."""
+    rows = (await db.execute(
+        select(HostMaintenanceJob, Host.name).join(Host, Host.id == HostMaintenanceJob.host_id)
+        .where(HostMaintenanceJob.state.in_(maintenance.FINISHED),
+               HostMaintenanceJob.finished_at >= now - timedelta(days=FAILED_JOB_DAYS))
+        .order_by(HostMaintenanceJob.finished_at.desc()))).all()
+    latest = {}
+    for j, name in rows:
+        latest.setdefault(j.host_id, (j, name))
+    return [(j, name) for j, name in latest.values() if j.state != "done"]
+
+
 async def mcp_overview(db) -> list[dict]:
-    """Agents registered more than PENDING_GRACE ago that have never checked in.
+    """Agents registered more than PENDING_GRACE ago that have never checked
+    in, and hosts whose latest maintenance job failed or timed out.
 
     Agents that stopped reporting are already flagged by the core overview
     from their metrics, so they are not repeated here.
     """
     now = now_utc()
-    return [
+    out = [
         finding("warning", "agent_never_reported", h.name,
                 "host agent was registered but has never checked in; the install may have failed",
                 since=reg.created_at, ids={"host_id": h.id}, next="list_agents()")
         for reg, h in await _agents(db)
         if reg.last_seen_at is None and now - reg.created_at > PENDING_GRACE
     ]
+    out += [
+        finding("warning", "maintenance_failed", name,
+                f"maintenance {maintenance.LABELS[j.state].lower()}"
+                + (f": {j.detail}" if j.detail else ""),
+                since=j.finished_at, ids={"host_id": j.host_id, "job_id": j.id},
+                next=f'get_maintenance_job(job="{j.id}")')
+        for j, name in await _failed_jobs(db, now)
+    ]
+    return out
 
 
 def register_tools(register) -> None:
     register(list_agents, role=UserRole.viewer, read_only=True)
+    register(list_maintenance_plans, role=UserRole.viewer, read_only=True)
+    register(get_maintenance_job, role=UserRole.viewer, read_only=True)
+    register(run_maintenance, role=UserRole.admin, read_only=False)