feat(host_agent): a group's updates and reboots on a weekly cadence, one host at a time (#5768)

#5768 "A maintenance cadence: a group's updates and reboots, one host at a
time, on a schedule". A plan names an inventory group, a weekday and time
in a time zone, a window and a reboot policy (only when required by
default). At its slot a run takes the group's hosts in order and gives each
a #5767 maintenance job. The next host starts only when the last is back in
service: its agent is posting and, for a swarm node, the node is active and
every replicated service is at its replicas. A failure stops the run, and
no host is started once the window has passed. A host's latest failure
shows as host_agent/<host>/maintenance_failed, for an ordinary alert rule.
The page is Hosts → Maintenance.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

plugins/host_agent/routes.py

@@ -1343,3 +1343,172 @@ async def update_via_ansible():
     if err:
         return _error(400, "update_failed", err)
     return redirect(url_for("ansible.run_detail", run_id=run.id))
+
+
+# ── Maintenance cadence (#5768) ───────────────────────────────────────────────
+
+def _run_rows(run, jobs: dict, names: dict) -> list[dict]:
+    """A run's hosts in order, each with where it stands, then those passed over."""
+    import json as _json
+    rows = []
+    for i, host_id in enumerate(_json.loads(run.host_ids)):
+        job = jobs.get((run.id, host_id))
+        if job is not None:
+            text = maintenance.summary(job)
+            bad = job.state in ("failed", "timed_out")
+            busy = job.state in maintenance.ACTIVE
+        elif run.state == "running" and i == run.position:
+            text, bad, busy = "Starting", False, True
+        else:
+            text, bad, busy = ("Not reached" if run.state != "running" else "Waiting"), False, False
+        if run.state == "running" and i == run.position and job is not None and job.state == "done":
+            text, busy = "Coming back into service", True
+        rows.append({"host_id": host_id, "name": names.get(host_id, host_id), "text": text,
+                     "bad": bad, "busy": busy, "detail": job.detail if job is not None else None})
+    for s in _json.loads(run.skipped):
+        rows.append({"host_id": s["host_id"], "name": names.get(s["host_id"], s["host_id"]),
+                     "text": f"Skipped: {s['reason']}", "bad": False, "busy": False, "detail": None})
+    return rows
+
+
+@host_agent_bp.get("/maintenance")
+@require_role(UserRole.viewer)
+async def maintenance_page():
+    """Maintenance plans: a group's updates and reboots on a weekly cadence,
+    one host at a time. A running run first, then a failed one, then the plans
+    and their recent runs."""
+    from steward.models.ansible_inventory import AnsibleGroup
+    from steward.models.users import User
+    from . import cadence
+    from .models import HostMaintenanceJob, HostMaintenancePlan, HostMaintenanceRun
+    now = datetime.now(timezone.utc)
+    async with current_app.db_sessionmaker() as db:
+        plans = (await db.execute(select(HostMaintenancePlan)
+                                  .order_by(HostMaintenancePlan.name))).scalars().all()
+        groups = (await db.execute(select(AnsibleGroup).order_by(AnsibleGroup.name))).scalars().all()
+        runs = (await db.execute(select(HostMaintenanceRun)
+                                 .order_by(HostMaintenanceRun.started_at.desc()).limit(60))).scalars().all()
+        jobs = {}
+        if runs:
+            for j in (await db.execute(select(HostMaintenanceJob).where(
+                    HostMaintenanceJob.run_id.in_([r.id for r in runs]))
+                    .order_by(HostMaintenanceJob.created_at))).scalars():
+                jobs[(j.run_id, j.host_id)] = j
+        names = dict((await db.execute(select(Host.id, Host.name))).all())
+        me = await db.get(User, session.get("user_id")) if session.get("user_id") else None
+        sizes = {g.id: len(await cadence.group_hosts(db, g.id)) for g in groups}
+    group_name = {g.id: g.name for g in groups}
+    by_plan: dict = {}
+    for r in runs:
+        by_plan.setdefault(r.plan_id, []).append(r)
+    view = []
+    for p in plans:
+        prs = by_plan.get(p.id, [])[:5]
+        view.append({"plan": p, "group": group_name.get(p.group_id, "—"),
+                     "hosts": sizes.get(p.group_id, 0), "when": cadence.describe(p),
+                     "next": cadence.next_slot(p, now) if p.enabled else None,
+                     "runs": [{"run": r, "rows": _run_rows(r, jobs, names)} for r in prs]})
+    # What needs attention first: a run going now, then a run that failed lately.
+    plan_name = {p.id: p.name for p in plans}
+    live = [{"run": r, "plan": plan_name.get(r.plan_id, "—"), "rows": _run_rows(r, jobs, names)}
+            for r in runs if r.state == "running"]
+    failed = [{"run": r, "plan": plan_name.get(r.plan_id, "—")} for r in runs
+              if r.state == "failed" and r.finished_at and now - r.finished_at < timedelta(days=7)]
+    return await render_template(
+        "maintenance.html", plans=view, groups=groups, live=live, failed=failed,
+        weekdays=cadence.WEEKDAYS, policies=maintenance.REBOOT_POLICIES,
+        default_zone=(me.time_zone if me and me.time_zone else "UTC"))
+
+
+@host_agent_bp.post("/maintenance/plans")
+@require_role(UserRole.admin)
+async def maintenance_plan_create():
+    from steward.core.formatting import zone
+    from steward.models.ansible_inventory import AnsibleGroup
+    from . import cadence
+    from .models import HostMaintenancePlan
+    form = await request.form
+    name = (form.get("name") or "").strip()[:128]
+    try:
+        weekday = int(form.get("weekday", ""))
+        at_time = (form.get("at_time") or "").strip()
+        cadence.parse_at(at_time)
+        window = int(form.get("window_minutes") or 240)
+    except ValueError:
+        return _error(400, "invalid", "weekday, time (HH:MM) and window must be numbers")
+    policy = form.get("reboot_policy") or "if_required"
+    tz = (form.get("time_zone") or "UTC").strip()
+    if not name or not 0 <= weekday <= 6 or not 30 <= window <= 24 * 60 \
+            or policy not in maintenance.REBOOT_POLICIES:
+        return _error(400, "invalid", "name, a weekday, a window of 30 min to 24 h and a reboot policy are needed")
+    if zone(tz) is timezone.utc and tz.upper() != "UTC":
+        return _error(400, "invalid", f"unknown time zone {tz!r}")
+    async with current_app.db_sessionmaker() as db:
+        if await db.get(AnsibleGroup, form.get("group_id") or "") is None:
+            return _error(400, "invalid", "pick an inventory group")
+        plan = HostMaintenancePlan(name=name, group_id=form["group_id"], weekday=weekday,
+                                   at_time=f"{cadence.parse_at(at_time):%H:%M}", time_zone=tz,
+                                   window_minutes=window, reboot_policy=policy, enabled=True,
+                                   created_by=session.get("user_id"),
+                                   # the first slot is the next one, not one already past
+                                   last_slot_at=datetime.now(timezone.utc))
+        db.add(plan)
+        await db.commit()
+    await audit_request("host_agent.maintenance_plan_created", "maintenance_plan", plan.id,
+                        {"name": name})
+    return redirect(url_for("host_agent.maintenance_page"))
+
+
+async def _plan_action(plan_id: str, act):
+    from .models import HostMaintenancePlan
+    async with current_app.db_sessionmaker() as db:
+        plan = await db.get(HostMaintenancePlan, plan_id)
+        if plan is None:
+            return _error(404, "not_found")
+        out = await act(db, plan)
+        if out is not None:
+            return out
+        await db.commit()
+    return redirect(url_for("host_agent.maintenance_page"))
+
+
+@host_agent_bp.post("/maintenance/plans/<plan_id>/toggle")
+@require_role(UserRole.admin)
+async def maintenance_plan_toggle(plan_id: str):
+    async def act(db, plan):
+        plan.enabled = not plan.enabled
+        if plan.enabled:
+            plan.last_slot_at = datetime.now(timezone.utc)   # no catching up on a missed slot
+    resp = await _plan_action(plan_id, act)
+    await audit_request("host_agent.maintenance_plan_toggled", "maintenance_plan", plan_id)
+    return resp
+
+
+@host_agent_bp.post("/maintenance/plans/<plan_id>/delete")
+@require_role(UserRole.admin)
+async def maintenance_plan_delete(plan_id: str):
+    async def act(db, plan):
+        await db.delete(plan)
+    resp = await _plan_action(plan_id, act)
+    await audit_request("host_agent.maintenance_plan_deleted", "maintenance_plan", plan_id)
+    return resp
+
+
+@host_agent_bp.post("/maintenance/plans/<plan_id>/run")
+@require_role(UserRole.admin)
+async def maintenance_plan_run_now(plan_id: str):
+    """Run the plan now. Its schedule keeps its time (Ansible ruling: Run now
+    leaves the schedule as it was)."""
+    from . import cadence
+    from .models import HostMaintenanceRun
+
+    async def act(db, plan):
+        busy = (await db.execute(select(HostMaintenanceRun.id).where(
+            HostMaintenanceRun.plan_id == plan.id,
+            HostMaintenanceRun.state == "running"))).first()
+        if busy is not None:
+            return _error(409, "busy", "this plan already has a run going")
+        await cadence.start_run(db, plan, datetime.now(timezone.utc), manual=True)
+    resp = await _plan_action(plan_id, act)
+    await audit_request("host_agent.maintenance_plan_run", "maintenance_plan", plan_id)
+    return resp