feat(host_agent): a group's updates and reboots on a weekly cadence, one host at a time (#5768)

#5768 "A maintenance cadence: a group's updates and reboots, one host at a
time, on a schedule". A plan names an inventory group, a weekday and time
in a time zone, a window and a reboot policy (only when required by
default). At its slot a run takes the group's hosts in order and gives each
a #5767 maintenance job. The next host starts only when the last is back in
service: its agent is posting and, for a swarm node, the node is active and
every replicated service is at its replicas. A failure stops the run, and
no host is started once the window has passed. A host's latest failure
shows as host_agent/<host>/maintenance_failed, for an ordinary alert rule.
The page is Hosts → Maintenance.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

plugins/host_agent/__init__.py

@@ -36,6 +36,12 @@ def setup(app: "Quart") -> None:
     prune_by_age(HostDowntime.started_at)
 
 
+def get_audit_entities() -> dict:
+    """Audit rows about a maintenance plan name it and link to the plans page."""
+    from .models import HostMaintenancePlan
+    return {"maintenance_plan": (HostMaintenancePlan, "/plugins/host_agent/maintenance")}
+
+
 def get_scheduled_tasks() -> list:
     from .scheduler import make_maintenance_task, make_task
     return [make_task(_app), make_maintenance_task(_app)]
@@ -70,12 +76,14 @@ def get_alert_metrics() -> dict[str, list[str]]:
     healthy agent sits in tens of MB and a few %. updates_pending and
     updates_security are what the package manager has waiting, and
     reboot_required is 1 when the host needs a reboot (agent ≥ 1.16.0):
-    "security updates waiting" is updates_security > 0."""
+    "security updates waiting" is updates_security > 0. maintenance_failed
+    comes from the scheduler (#5768): 1 when the host's latest maintenance job
+    failed or timed out, 0 once one succeeds: "maintenance failed" is > 0."""
     return {"host_agent": [
         "cpu_pct", "mem_used_pct", "mem_available_bytes", "swap_used_bytes",
         "disk_used_pct_worst", "disk_used_pct", "inodes_used_pct", "load_1m", "load_5m", "load_15m",
         "load_pct",
         "uptime_secs", "seconds_since_seen", "clock_off_secs", "mounts_stalled",
         "agent_rss_bytes", "agent_cpu_pct",
-        "updates_pending", "updates_security", "reboot_required",
+        "updates_pending", "updates_security", "reboot_required", "maintenance_failed",
     ]}

plugins/host_agent/cadence.py

@@ -0,0 +1,302 @@
+# plugins/host_agent/cadence.py
+"""A group's maintenance cadence (#5768): every week, its hosts updated one at
+a time, each rebooting only when its update needs it (or as the plan says).
+
+A plan names an inventory group, a weekday and a time in its time zone, and a
+window. At that time a run starts with the group's hosts in a fixed order. Each
+host gets a maintenance job (maintenance.py); the next host starts only when
+the last one is back in service: its agent posting, its swarm node ready and
+active, every replicated service at its replicas. A job that fails, or a host
+that doesn't come back into service, stops the run, and the host's
+maintenance_failed figure goes to 1 for an alert rule to catch. No host is
+started after the window; the run then stops and says which hosts it didn't
+reach.
+
+The decisions are model-free here (plans and runs are anything with the
+attributes); the database side is below them, and the plugin's minute tick
+drives it. Everything is on the rows, so a restart of Steward, or the reboot of
+the host it runs on, picks up where it was.
+"""
+from __future__ import annotations
+
+import json
+from datetime import datetime, time, timedelta, timezone
+
+from . import maintenance
+
+WEEKDAYS = ("Monday", "Tuesday", "Wednesday", "Thursday", "Friday", "Saturday", "Sunday")
+# How long a host may take to be back in service after its job finished.
+SETTLE_LIMIT = timedelta(minutes=30)
+# An agent heard from this recently is posting.
+POSTING_WITHIN = timedelta(minutes=3)
+RUN_STATES = ("running", "done", "failed", "stopped")
+
+
+def _zone(name: str):
+    from steward.core.formatting import zone
+    return zone(name)
+
+
+def parse_at(at_time: str) -> time:
+    h, m = (int(x) for x in at_time.split(":", 1))
+    if not (0 <= h < 24 and 0 <= m < 60):
+        raise ValueError(at_time)
+    return time(h, m)
+
+
+def last_slot(plan, now: datetime) -> datetime:
+    """The most recent scheduled time at or before `now`, in UTC."""
+    tz = _zone(plan.time_zone)
+    local = now.astimezone(tz)
+    day = local.date() - timedelta(days=(local.weekday() - plan.weekday) % 7)
+    slot = datetime.combine(day, parse_at(plan.at_time), tzinfo=tz)
+    if slot > local:
+        slot -= timedelta(days=7)
+    return slot.astimezone(timezone.utc)
+
+
+def next_slot(plan, now: datetime) -> datetime:
+    return last_slot(plan, now) + timedelta(days=7)
+
+
+def is_due(plan, now: datetime) -> datetime | None:
+    """The slot to fire now, or None. A slot fires once (last_slot_at), and
+    only while its window is still open: a Steward that was down over the slot
+    starts late, not a day late."""
+    if not plan.enabled:
+        return None
+    slot = last_slot(plan, now)
+    since = plan.last_slot_at or plan.created_at
+    if since is not None and slot <= since:
+        return None
+    if now - slot >= timedelta(minutes=plan.window_minutes):
+        return None
+    return slot
+
+
+def back_in_service(*, last_seen_at, node, services, job_finished_at, now: datetime) -> str | None:
+    """None when the host is back in service, else why not yet. `node` is the
+    freshest swarm-node row for the job's node (None outside a swarm, or when
+    the job has no node); `services` the freshest row per swarm service. Only
+    rows reported after the job finished count, so a stale "all fine" from
+    before the reboot never passes."""
+    if last_seen_at is None or now - last_seen_at > POSTING_WITHIN:
+        return "its agent isn't posting yet"
+    if node is None:
+        return None
+    if node.updated_at < job_finished_at:
+        return "no swarm report since it came back"
+    if node.status != "ready" or node.availability != "active":
+        return f"its swarm node is {node.status}/{node.availability}"
+    fresh = [s for s in services if s.updated_at >= job_finished_at]
+    if not fresh:
+        return "no swarm report since it came back"
+    short = sorted(s.service_name for s in fresh
+                   if s.mode == "replicated" and s.running < s.desired)
+    if short:
+        return "services below their replicas: " + ", ".join(short[:5])
+    return None
+
+
+def next_move(run, plan, job, back: str | None, now: datetime) -> tuple[str, str | None]:
+    """What a running run does now: ("start", None) the current host's job;
+    ("wait", None); ("next", None) on to the next host; ("settling", why);
+    ("fail", why); ("stop", why) the window ended; ("done", None)."""
+    hosts = json.loads(run.host_ids)
+    if run.position >= len(hosts):
+        return "done", None
+    if job is None:
+        if now >= run.started_at + timedelta(minutes=plan.window_minutes):
+            left = len(hosts) - run.position
+            return "stop", f"the window ended with {left} host{'s' if left != 1 else ''} not started"
+        return "start", None
+    if job.state in maintenance.ACTIVE:
+        return "wait", None
+    if job.state != "done":
+        return "fail", job.detail or maintenance.LABELS[job.state]
+    if back is None:
+        return "next", None
+    if run.settle_since is not None and now - run.settle_since > SETTLE_LIMIT:
+        return "fail", f"not back in service within {int(SETTLE_LIMIT.total_seconds() // 60)} min: {back}"
+    return "settling", back
+
+
+def describe(plan) -> str:
+    """"Sundays at 03:00 (Europe/London), up to 4 h" for the page."""
+    hours = plan.window_minutes / 60
+    span = f"{hours:g} h" if plan.window_minutes % 60 == 0 else f"{plan.window_minutes} min"
+    return f"{WEEKDAYS[plan.weekday]}s at {plan.at_time} ({plan.time_zone}), up to {span}"
+
+
+# ── applied to the database: the minute tick, Run now, the page ──────────────
+
+async def group_hosts(session, group_id: str) -> list:
+    """The group's hosts (through their linked Ansible targets), by name."""
+    from sqlalchemy import select
+    from steward.models.ansible_inventory import AnsibleTarget, ansible_target_groups
+    from steward.models.hosts import Host
+    rows = (await session.execute(
+        select(Host).join(AnsibleTarget, AnsibleTarget.host_id == Host.id)
+        .join(ansible_target_groups, ansible_target_groups.c.target_id == AnsibleTarget.id)
+        .where(ansible_target_groups.c.group_id == group_id))).scalars().all()
+    return sorted({h.id: h for h in rows}.values(), key=lambda h: h.name.lower())
+
+
+async def start_run(session, plan, now: datetime, manual: bool = False):
+    """A run of the plan with the group's hosts in order. A host whose agent
+    has no maintenance helper is passed over, and the run says so."""
+    from sqlalchemy import select
+    from .models import HostAgentRegistration, HostMaintenanceRun
+    hosts = await group_hosts(session, plan.group_id)
+    regs = {r.host_id: r for r in (await session.execute(select(HostAgentRegistration).where(
+        HostAgentRegistration.host_id.in_([h.id for h in hosts])))).scalars()} if hosts else {}
+    order, skipped = [], []
+    for h in hosts:
+        reg = regs.get(h.id)
+        if reg is None:
+            skipped.append({"host_id": h.id, "reason": "no agent"})
+        elif not reg.maint_helper:
+            skipped.append({"host_id": h.id, "reason": "no maintenance helper: run Update agent"})
+        else:
+            order.append(h.id)
+    run = HostMaintenanceRun(plan_id=plan.id, state="running", host_ids=json.dumps(order),
+                             position=0, skipped=json.dumps(skipped), manual=manual,
+                             started_at=now)
+    session.add(run)
+    await session.flush()
+    return run
+
+
+async def _back(session, host, job, reg, now: datetime) -> str | None:
+    from sqlalchemy import select
+    node = None
+    services: list = []
+    if job.node_id:
+        try:
+            from plugins.docker.models import DockerSwarmNode, DockerSwarmService
+        except ImportError:
+            DockerSwarmNode = DockerSwarmService = None
+        if DockerSwarmNode is not None:
+            node = (await session.execute(select(DockerSwarmNode).where(
+                DockerSwarmNode.node_id == job.node_id)
+                .order_by(DockerSwarmNode.updated_at.desc()).limit(1))).scalar_one_or_none()
+            freshest: dict = {}
+            for s in (await session.execute(select(DockerSwarmService))).scalars():
+                if s.service_name not in freshest or s.updated_at > freshest[s.service_name].updated_at:
+                    freshest[s.service_name] = s
+            services = list(freshest.values())
+    return back_in_service(last_seen_at=reg.last_seen_at if reg else None, node=node,
+                           services=services, job_finished_at=job.finished_at or now, now=now)
+
+
+async def advance(session, run, plan, now: datetime) -> str:
+    """Move one running run on as far as it can go now; returns its state."""
+    from sqlalchemy import select
+    from steward.models.hosts import Host
+    from .models import HostAgentRegistration, HostMaintenanceJob
+    for _ in range(64):                             # a host per pass; bounded
+        hosts = json.loads(run.host_ids)
+        host_id = hosts[run.position] if run.position < len(hosts) else None
+        job = host = reg = None
+        if host_id is not None:
+            host = await session.get(Host, host_id)
+            reg = (await session.execute(select(HostAgentRegistration).where(
+                HostAgentRegistration.host_id == host_id))).scalar_one_or_none()
+            job = (await session.execute(select(HostMaintenanceJob).where(
+                HostMaintenanceJob.run_id == run.id, HostMaintenanceJob.host_id == host_id)
+                .order_by(HostMaintenanceJob.created_at.desc()).limit(1))).scalar_one_or_none()
+        back = None
+        if job is not None and job.state == "done":
+            back = await _back(session, host, job, reg, now)
+        move, why = next_move(run, plan, job, back, now)
+        if move == "start":
+            if host is None:
+                _skip(run, host_id, "the host is gone")
+                continue
+            try:
+                await maintenance.create_job(session, host, reg, plan.created_by,
+                                             plan.reboot_policy, now, run_id=run.id)
+            except maintenance.CannotStart as exc:
+                _skip(run, host_id, str(exc))
+                continue
+            return run.state
+        if move == "next":
+            run.position += 1
+            run.settle_since = None
+            continue
+        if move == "settling":
+            run.settle_since = run.settle_since or now
+            return run.state
+        if move == "fail":
+            if job is not None and job.state == "done":
+                maintenance._finish(job, "failed", now, why)     # the host never came back into service
+            name = host.name if host else host_id
+            _end(run, "failed", now, f"{name}: {why}")
+        elif move == "stop":
+            _end(run, "stopped", now, why)
+        elif move == "done":
+            _end(run, "done", now, None)
+        return run.state
+    return run.state
+
+
+def _skip(run, host_id: str, reason: str) -> None:
+    skipped = json.loads(run.skipped)
+    skipped.append({"host_id": host_id, "reason": reason})
+    run.skipped = json.dumps(skipped)
+    hosts = json.loads(run.host_ids)
+    hosts.pop(run.position)
+    run.host_ids = json.dumps(hosts)
+
+
+def _end(run, state: str, now: datetime, detail: str | None) -> None:
+    run.state, run.finished_at, run.settle_since = state, now, None
+    if detail:
+        run.detail = detail[:maintenance.DETAIL_MAX]
+
+
+async def tick(session, now: datetime) -> list[str]:
+    """Start due plans and move running runs on; returns lines worth logging."""
+    from sqlalchemy import select
+    from .models import HostMaintenancePlan, HostMaintenanceRun
+    said = []
+    running = {r.plan_id: r for r in (await session.execute(select(HostMaintenanceRun).where(
+        HostMaintenanceRun.state == "running"))).scalars()}
+    for plan in (await session.execute(select(HostMaintenancePlan))).scalars():
+        slot = is_due(plan, now)
+        if slot is None:
+            continue
+        plan.last_slot_at = slot
+        if plan.id in running:
+            said.append(f"plan {plan.name!r}: the {slot:%Y-%m-%d %H:%M}Z slot came with a run still going")
+            continue
+        running[plan.id] = await start_run(session, plan, now)
+        said.append(f"plan {plan.name!r}: run started")
+    for run in running.values():
+        plan = await session.get(HostMaintenancePlan, run.plan_id)
+        if plan is None:
+            continue
+        before = run.state
+        after = await advance(session, run, plan, now)
+        if after != before:
+            said.append(f"plan {plan.name!r}: run {after}" + (f" ({run.detail})" if run.detail else ""))
+    return said
+
+
+async def failed_hosts(session, now: datetime) -> dict[str, float]:
+    """{host name: 1 or 0}: whether each host's latest finished maintenance job
+    failed or timed out, for the maintenance_failed figure an alert rule
+    watches. A later job that succeeds puts it back to 0."""
+    from sqlalchemy import select
+    from steward.models.hosts import Host
+    from .models import HostMaintenanceJob
+    rows = (await session.execute(
+        select(Host.name, HostMaintenanceJob.state, HostMaintenanceJob.finished_at)
+        .join(Host, Host.id == HostMaintenanceJob.host_id)
+        .where(HostMaintenanceJob.state.in_(maintenance.FINISHED),
+               HostMaintenanceJob.finished_at >= now - timedelta(days=30)))).all()
+    latest: dict = {}
+    for name, state, finished in rows:
+        if name not in latest or finished > latest[name][1]:
+            latest[name] = (state, finished)
+    return {name: 0.0 if state == "done" else 1.0 for name, (state, _) in latest.items()}

plugins/host_agent/maintenance.py

@@ -86,8 +86,19 @@ def handoff(job) -> dict | None:
     return out
 
 
+REBOOT_POLICIES = ("if_required", "always", "never")
+
+
+def wants_reboot(job) -> bool:
+    """Reboot only when the update says one is needed (the operator's
+    default), always (a plan may ask), or never."""
+    if job.reboot_policy == "always":
+        return True
+    return bool(job.reboot_needed) and job.reboot_policy == "if_required"
+
+
 def _after_update(job, now: datetime) -> None:
-    if not (job.reboot_needed and job.reboot_policy != "never"):
+    if not wants_reboot(job):
         _enter(job, "done", now)
     elif job.node_id and job.drain_host_id:
         _enter(job, "draining", now)
@@ -296,7 +307,7 @@ async def plan_job(session, host, reg, reboot_policy: str = "if_required") -> tu
     from .models import HostAgentRegistration, HostMaintenanceJob
     if reg is None or not reg.maint_helper:
         raise CannotStart("its agent has no maintenance helper: run Update agent first")
-    if reboot_policy not in ("if_required", "never"):
+    if reboot_policy not in REBOOT_POLICIES:
         raise CannotStart(f"unknown reboot policy {reboot_policy!r}")
     busy = (await session.execute(select(HostMaintenanceJob.id).where(
         HostMaintenanceJob.host_id == host.id,
@@ -312,21 +323,23 @@ async def plan_job(session, host, reg, reboot_policy: str = "if_required") -> tu
 
 
 async def create_job(session, host, reg, user_id: str | None,
-                     reboot_policy: str = "if_required", now: datetime | None = None):
+                     reboot_policy: str = "if_required", now: datetime | None = None,
+                     run_id: str | None = None):
     """Queue a maintenance job for `host`; it starts with the agent's next post."""
     from .models import HostMaintenanceJob
     now = now or utcnow()
     node_id, drain_host_id = await plan_job(session, host, reg, reboot_policy)
     job = HostMaintenanceJob(host_id=host.id, state="queued", state_at=now,
                              reboot_policy=reboot_policy, node_id=node_id,
-                             drain_host_id=drain_host_id, created_by=user_id, created_at=now)
+                             drain_host_id=drain_host_id, created_by=user_id, created_at=now,
+                             run_id=run_id)
     session.add(job)
     await session.flush()
     return job
 
 
-async def latest_job(session, host_id: str, now: datetime, days: int = 7):
-    """The host's running job, else its last one in `days`, for the panel."""
+async def latest_job(session, host_id: str, now: datetime, days: int = 366):
+    """The host's running job, else its last one (within a year), for the panel."""
     from sqlalchemy import select
     from .models import HostMaintenanceJob
     return (await session.execute(select(HostMaintenanceJob).where(

plugins/host_agent/migrations/versions/host_agent_008_maint_plans.py

@@ -0,0 +1,61 @@
+"""host_agent: a group's maintenance cadence and its runs (#5768)
+
+Revision ID: host_agent_008_maint_plans
+"""
+from typing import Sequence, Union
+from alembic import op
+import sqlalchemy as sa
+
+revision: str = "host_agent_008_maint_plans"
+down_revision: Union[str, None] = "host_agent_007_maint_jobs"
+branch_labels: Union[str, Sequence[str], None] = None
+# ansible_groups (core 0016) must exist before the group_id key.
+depends_on: Union[str, Sequence[str], None] = ("0016_ansible_inventory_groups",)
+
+
+def upgrade() -> None:
+    op.create_table(
+        "host_maintenance_plans",
+        sa.Column("id", sa.String(36), primary_key=True),
+        sa.Column("name", sa.String(128), nullable=False),
+        sa.Column("group_id", sa.String(36), sa.ForeignKey("ansible_groups.id", ondelete="CASCADE"),
+                  nullable=False),
+        sa.Column("weekday", sa.Integer(), nullable=False),
+        sa.Column("at_time", sa.String(5), nullable=False),
+        sa.Column("time_zone", sa.String(64), nullable=False, server_default="UTC"),
+        sa.Column("window_minutes", sa.Integer(), nullable=False, server_default="240"),
+        sa.Column("reboot_policy", sa.String(16), nullable=False, server_default="if_required"),
+        sa.Column("enabled", sa.Boolean(), nullable=False, server_default=sa.true()),
+        sa.Column("last_slot_at", sa.DateTime(timezone=True), nullable=True),
+        sa.Column("created_by", sa.String(36), sa.ForeignKey("users.id", ondelete="SET NULL"),
+                  nullable=True),
+        sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
+    )
+    op.create_table(
+        "host_maintenance_runs",
+        sa.Column("id", sa.String(36), primary_key=True),
+        sa.Column("plan_id", sa.String(36),
+                  sa.ForeignKey("host_maintenance_plans.id", ondelete="CASCADE"), nullable=False),
+        sa.Column("state", sa.String(16), nullable=False),
+        sa.Column("host_ids", sa.Text(), nullable=False, server_default="[]"),
+        sa.Column("position", sa.Integer(), nullable=False, server_default="0"),
+        sa.Column("skipped", sa.Text(), nullable=False, server_default="[]"),
+        sa.Column("settle_since", sa.DateTime(timezone=True), nullable=True),
+        sa.Column("detail", sa.Text(), nullable=True),
+        sa.Column("manual", sa.Boolean(), nullable=False, server_default=sa.false()),
+        sa.Column("started_at", sa.DateTime(timezone=True), nullable=False),
+        sa.Column("finished_at", sa.DateTime(timezone=True), nullable=True),
+    )
+    op.create_index("ix_host_maintenance_runs_plan_id", "host_maintenance_runs", ["plan_id"])
+    op.create_index("ix_host_maintenance_runs_state", "host_maintenance_runs", ["state"])
+    op.add_column("host_maintenance_jobs", sa.Column(
+        "run_id", sa.String(36), sa.ForeignKey("host_maintenance_runs.id", ondelete="SET NULL"),
+        nullable=True))
+    op.create_index("ix_host_maintenance_jobs_run_id", "host_maintenance_jobs", ["run_id"])
+
+
+def downgrade() -> None:
+    op.drop_index("ix_host_maintenance_jobs_run_id", "host_maintenance_jobs")
+    op.drop_column("host_maintenance_jobs", "run_id")
+    op.drop_table("host_maintenance_runs")
+    op.drop_table("host_maintenance_plans")

plugins/host_agent/models.py

@@ -106,3 +106,52 @@ class HostMaintenanceJob(Base):
     created_by = Column(String(36), ForeignKey("users.id", ondelete="SET NULL"), nullable=True)
     created_at = Column(DateTime(timezone=True), nullable=False, default=_utcnow)
     finished_at = Column(DateTime(timezone=True), nullable=True)
+    # The cadence run this job belongs to (#5768); None for Update now.
+    run_id = Column(String(36), ForeignKey("host_maintenance_runs.id", ondelete="SET NULL"),
+                    nullable=True, index=True)
+
+
+class HostMaintenancePlan(Base):
+    """A group's maintenance cadence (#5768): every week on `weekday` at
+    `at_time` in `time_zone`, its hosts are updated one at a time, each
+    rebooting per `reboot_policy`. No host is started after `window_minutes`
+    from the run's start. last_slot_at is the last scheduled slot that fired;
+    Run now leaves it alone, so the schedule keeps its time."""
+    __tablename__ = "host_maintenance_plans"
+
+    id = Column(String(36), primary_key=True, default=_uuid)
+    name = Column(String(128), nullable=False)
+    group_id = Column(String(36), ForeignKey("ansible_groups.id", ondelete="CASCADE"),
+                      nullable=False)
+    weekday = Column(Integer, nullable=False)              # 0 = Monday
+    at_time = Column(String(5), nullable=False)            # "HH:MM"
+    time_zone = Column(String(64), nullable=False, default="UTC")
+    window_minutes = Column(Integer, nullable=False, default=240)
+    reboot_policy = Column(String(16), nullable=False, default="if_required")
+    enabled = Column(Boolean, nullable=False, default=True)
+    last_slot_at = Column(DateTime(timezone=True), nullable=True)
+    created_by = Column(String(36), ForeignKey("users.id", ondelete="SET NULL"), nullable=True)
+    created_at = Column(DateTime(timezone=True), nullable=False, default=_utcnow)
+
+
+class HostMaintenanceRun(Base):
+    """One run of a plan: its hosts in order, one at a time (#5768).
+
+    host_ids is the order fixed at the start (JSON list); position is the host
+    being worked on; skipped is [{"host_id", "reason"}] for hosts the run
+    passed over. settle_since is when the current host's job finished and the
+    run began waiting for the host to be back in service."""
+    __tablename__ = "host_maintenance_runs"
+
+    id = Column(String(36), primary_key=True, default=_uuid)
+    plan_id = Column(String(36), ForeignKey("host_maintenance_plans.id", ondelete="CASCADE"),
+                     nullable=False, index=True)
+    state = Column(String(16), nullable=False, default="running", index=True)
+    host_ids = Column(Text, nullable=False, default="[]")
+    position = Column(Integer, nullable=False, default=0)
+    skipped = Column(Text, nullable=False, default="[]")
+    settle_since = Column(DateTime(timezone=True), nullable=True)
+    detail = Column(Text, nullable=True)
+    manual = Column(Boolean, nullable=False, default=False)
+    started_at = Column(DateTime(timezone=True), nullable=False, default=_utcnow)
+    finished_at = Column(DateTime(timezone=True), nullable=True)

plugins/host_agent/routes.py

@@ -1343,3 +1343,172 @@ async def update_via_ansible():
     if err:
         return _error(400, "update_failed", err)
     return redirect(url_for("ansible.run_detail", run_id=run.id))
+
+
+# ── Maintenance cadence (#5768) ───────────────────────────────────────────────
+
+def _run_rows(run, jobs: dict, names: dict) -> list[dict]:
+    """A run's hosts in order, each with where it stands, then those passed over."""
+    import json as _json
+    rows = []
+    for i, host_id in enumerate(_json.loads(run.host_ids)):
+        job = jobs.get((run.id, host_id))
+        if job is not None:
+            text = maintenance.summary(job)
+            bad = job.state in ("failed", "timed_out")
+            busy = job.state in maintenance.ACTIVE
+        elif run.state == "running" and i == run.position:
+            text, bad, busy = "Starting", False, True
+        else:
+            text, bad, busy = ("Not reached" if run.state != "running" else "Waiting"), False, False
+        if run.state == "running" and i == run.position and job is not None and job.state == "done":
+            text, busy = "Coming back into service", True
+        rows.append({"host_id": host_id, "name": names.get(host_id, host_id), "text": text,
+                     "bad": bad, "busy": busy, "detail": job.detail if job is not None else None})
+    for s in _json.loads(run.skipped):
+        rows.append({"host_id": s["host_id"], "name": names.get(s["host_id"], s["host_id"]),
+                     "text": f"Skipped: {s['reason']}", "bad": False, "busy": False, "detail": None})
+    return rows
+
+
+@host_agent_bp.get("/maintenance")
+@require_role(UserRole.viewer)
+async def maintenance_page():
+    """Maintenance plans: a group's updates and reboots on a weekly cadence,
+    one host at a time. A running run first, then a failed one, then the plans
+    and their recent runs."""
+    from steward.models.ansible_inventory import AnsibleGroup
+    from steward.models.users import User
+    from . import cadence
+    from .models import HostMaintenanceJob, HostMaintenancePlan, HostMaintenanceRun
+    now = datetime.now(timezone.utc)
+    async with current_app.db_sessionmaker() as db:
+        plans = (await db.execute(select(HostMaintenancePlan)
+                                  .order_by(HostMaintenancePlan.name))).scalars().all()
+        groups = (await db.execute(select(AnsibleGroup).order_by(AnsibleGroup.name))).scalars().all()
+        runs = (await db.execute(select(HostMaintenanceRun)
+                                 .order_by(HostMaintenanceRun.started_at.desc()).limit(60))).scalars().all()
+        jobs = {}
+        if runs:
+            for j in (await db.execute(select(HostMaintenanceJob).where(
+                    HostMaintenanceJob.run_id.in_([r.id for r in runs]))
+                    .order_by(HostMaintenanceJob.created_at))).scalars():
+                jobs[(j.run_id, j.host_id)] = j
+        names = dict((await db.execute(select(Host.id, Host.name))).all())
+        me = await db.get(User, session.get("user_id")) if session.get("user_id") else None
+        sizes = {g.id: len(await cadence.group_hosts(db, g.id)) for g in groups}
+    group_name = {g.id: g.name for g in groups}
+    by_plan: dict = {}
+    for r in runs:
+        by_plan.setdefault(r.plan_id, []).append(r)
+    view = []
+    for p in plans:
+        prs = by_plan.get(p.id, [])[:5]
+        view.append({"plan": p, "group": group_name.get(p.group_id, "—"),
+                     "hosts": sizes.get(p.group_id, 0), "when": cadence.describe(p),
+                     "next": cadence.next_slot(p, now) if p.enabled else None,
+                     "runs": [{"run": r, "rows": _run_rows(r, jobs, names)} for r in prs]})
+    # What needs attention first: a run going now, then a run that failed lately.
+    plan_name = {p.id: p.name for p in plans}
+    live = [{"run": r, "plan": plan_name.get(r.plan_id, "—"), "rows": _run_rows(r, jobs, names)}
+            for r in runs if r.state == "running"]
+    failed = [{"run": r, "plan": plan_name.get(r.plan_id, "—")} for r in runs
+              if r.state == "failed" and r.finished_at and now - r.finished_at < timedelta(days=7)]
+    return await render_template(
+        "maintenance.html", plans=view, groups=groups, live=live, failed=failed,
+        weekdays=cadence.WEEKDAYS, policies=maintenance.REBOOT_POLICIES,
+        default_zone=(me.time_zone if me and me.time_zone else "UTC"))
+
+
+@host_agent_bp.post("/maintenance/plans")
+@require_role(UserRole.admin)
+async def maintenance_plan_create():
+    from steward.core.formatting import zone
+    from steward.models.ansible_inventory import AnsibleGroup
+    from . import cadence
+    from .models import HostMaintenancePlan
+    form = await request.form
+    name = (form.get("name") or "").strip()[:128]
+    try:
+        weekday = int(form.get("weekday", ""))
+        at_time = (form.get("at_time") or "").strip()
+        cadence.parse_at(at_time)
+        window = int(form.get("window_minutes") or 240)
+    except ValueError:
+        return _error(400, "invalid", "weekday, time (HH:MM) and window must be numbers")
+    policy = form.get("reboot_policy") or "if_required"
+    tz = (form.get("time_zone") or "UTC").strip()
+    if not name or not 0 <= weekday <= 6 or not 30 <= window <= 24 * 60 \
+            or policy not in maintenance.REBOOT_POLICIES:
+        return _error(400, "invalid", "name, a weekday, a window of 30 min to 24 h and a reboot policy are needed")
+    if zone(tz) is timezone.utc and tz.upper() != "UTC":
+        return _error(400, "invalid", f"unknown time zone {tz!r}")
+    async with current_app.db_sessionmaker() as db:
+        if await db.get(AnsibleGroup, form.get("group_id") or "") is None:
+            return _error(400, "invalid", "pick an inventory group")
+        plan = HostMaintenancePlan(name=name, group_id=form["group_id"], weekday=weekday,
+                                   at_time=f"{cadence.parse_at(at_time):%H:%M}", time_zone=tz,
+                                   window_minutes=window, reboot_policy=policy, enabled=True,
+                                   created_by=session.get("user_id"),
+                                   # the first slot is the next one, not one already past
+                                   last_slot_at=datetime.now(timezone.utc))
+        db.add(plan)
+        await db.commit()
+    await audit_request("host_agent.maintenance_plan_created", "maintenance_plan", plan.id,
+                        {"name": name})
+    return redirect(url_for("host_agent.maintenance_page"))
+
+
+async def _plan_action(plan_id: str, act):
+    from .models import HostMaintenancePlan
+    async with current_app.db_sessionmaker() as db:
+        plan = await db.get(HostMaintenancePlan, plan_id)
+        if plan is None:
+            return _error(404, "not_found")
+        out = await act(db, plan)
+        if out is not None:
+            return out
+        await db.commit()
+    return redirect(url_for("host_agent.maintenance_page"))
+
+
+@host_agent_bp.post("/maintenance/plans/<plan_id>/toggle")
+@require_role(UserRole.admin)
+async def maintenance_plan_toggle(plan_id: str):
+    async def act(db, plan):
+        plan.enabled = not plan.enabled
+        if plan.enabled:
+            plan.last_slot_at = datetime.now(timezone.utc)   # no catching up on a missed slot
+    resp = await _plan_action(plan_id, act)
+    await audit_request("host_agent.maintenance_plan_toggled", "maintenance_plan", plan_id)
+    return resp
+
+
+@host_agent_bp.post("/maintenance/plans/<plan_id>/delete")
+@require_role(UserRole.admin)
+async def maintenance_plan_delete(plan_id: str):
+    async def act(db, plan):
+        await db.delete(plan)
+    resp = await _plan_action(plan_id, act)
+    await audit_request("host_agent.maintenance_plan_deleted", "maintenance_plan", plan_id)
+    return resp
+
+
+@host_agent_bp.post("/maintenance/plans/<plan_id>/run")
+@require_role(UserRole.admin)
+async def maintenance_plan_run_now(plan_id: str):
+    """Run the plan now. Its schedule keeps its time (Ansible ruling: Run now
+    leaves the schedule as it was)."""
+    from . import cadence
+    from .models import HostMaintenanceRun
+
+    async def act(db, plan):
+        busy = (await db.execute(select(HostMaintenanceRun.id).where(
+            HostMaintenanceRun.plan_id == plan.id,
+            HostMaintenanceRun.state == "running"))).first()
+        if busy is not None:
+            return _error(409, "busy", "this plan already has a run going")
+        await cadence.start_run(db, plan, datetime.now(timezone.utc), manual=True)
+    resp = await _plan_action(plan_id, act)
+    await audit_request("host_agent.maintenance_plan_run", "maintenance_plan", plan_id)
+    return resp

plugins/host_agent/scheduler.py

@@ -122,16 +122,27 @@ def make_task(app) -> ScheduledTask:
 
 
 def make_maintenance_task(app) -> ScheduledTask:
-    """Gives up on maintenance jobs past their deadline (#5767). Every other
-    move a job makes comes from its agent's posts."""
+    """Gives up on maintenance jobs past their deadline (#5767), starts due
+    maintenance plans and moves their runs on one host at a time (#5768), and
+    records each host's maintenance_failed for an alert rule. Every move a job
+    itself makes comes from its agent's posts."""
     async def _tick():
-        from . import maintenance
+        from . import cadence, maintenance
+        now = datetime.now(timezone.utc)
         try:
             async with app.db_sessionmaker() as session:
                 async with session.begin():
-                    gave_up = await maintenance.tick(session, datetime.now(timezone.utc))
+                    gave_up = await maintenance.tick(session, now)
+                    said = await cadence.tick(session, now)
+                    batch = MetricBatch()
+                    for name, value in (await cadence.failed_hosts(session, now)).items():
+                        batch.add(source_module="host_agent", resource_name=name,
+                                  metric_name="maintenance_failed", value=value, recorded_at=now)
+                    await batch.record(session)
             if gave_up:
                 logger.warning("host_agent: %d maintenance job(s) timed out", gave_up)
+            for line in said:
+                logger.info("host_agent maintenance: %s", line)
         except Exception:
             logger.exception("host_agent: maintenance tick failed")
 

plugins/host_agent/templates/maintenance.html

@@ -0,0 +1,143 @@
+{% extends "base.html" %}
+{% from "_macros.html" import crumbs, empty_state, page_header %}
+{% block title %}Maintenance — Steward{% endblock %}
+{% block breadcrumb %}{{ crumbs([("Hosts", "/hosts/"), ("Maintenance", "")]) }}{% endblock %}
+{% block content %}
+{# A group's updates and reboots on a weekly cadence, one host at a time
+   (#5768). What needs attention first: a run going now, a run that failed;
+   then the plans with their recent runs. #}
+{% macro run_hosts(rows) -%}
+<div style="display:flex;flex-direction:column;gap:0.15rem;font-size:0.82rem;">
+  {% for r in rows %}
+  <div style="display:flex;gap:0.6rem;align-items:baseline;" {% if r.detail %}title="{{ r.detail }}"{% endif %}>
+    <a href="/hosts/{{ r.host_id }}" style="min-width:10rem;">{{ r.name }}</a>
+    <span class="{{ 'tone-warn' if r.bad else '' }}" style="{{ '' if r.bad or r.busy else 'color:var(--text-muted);' }}">{{ r.text }}</span>
+  </div>
+  {% endfor %}
+</div>
+{%- endmacro %}
+
+{{ page_header("Maintenance") }}
+<p style="color:var(--text-muted);font-size:0.84rem;margin:-0.4rem 0 1.25rem;">
+  Each plan updates an inventory group's hosts once a week, one host at a time. A host reboots
+  only when its update needs it, unless the plan says otherwise; a swarm node is drained first
+  and made active again after. The next host starts once the last is back in service. A failure
+  stops the run, and the host's <em>Maintenance failed</em> figure can drive an alert rule.
+</p>
+
+{% if live %}
+{% for l in live %}
+<div class="card">
+  <h3 style="margin-bottom:0.5rem;">Running: {{ l.plan }} <span style="font-size:0.78rem;color:var(--text-muted);font-weight:normal;">· started {{ l.run.started_at|ago }}{% if l.run.manual %} · run now{% endif %}</span></h3>
+  {{ run_hosts(l.rows) }}
+</div>
+{% endfor %}
+{% endif %}
+
+{% for f in failed %}
+<div class="card" style="border-color:color-mix(in srgb,var(--yellow) 40%,var(--border));">
+  <span class="tone-warn">Failed: {{ f.plan }}</span>
+  <span style="font-size:0.82rem;color:var(--text-muted);">· {{ f.run.finished_at|ago }} · {{ f.run.detail or "no detail" }}</span>
+</div>
+{% endfor %}
+
+{% if plans %}
+{% for p in plans %}
+<div class="card">
+  <div style="display:flex;align-items:baseline;justify-content:space-between;gap:1rem;flex-wrap:wrap;">
+    <h3 style="margin-bottom:0.2rem;">{{ p.plan.name }}{% if not p.plan.enabled %} <span style="font-size:0.78rem;color:var(--text-muted);font-weight:normal;">· paused</span>{% endif %}</h3>
+    {% if session.user_role == 'admin' %}
+    <div style="display:flex;gap:0.4rem;">
+      <form method="post" action="/plugins/host_agent/maintenance/plans/{{ p.plan.id }}/run" style="margin:0;"
+            data-confirm="Run {{ p.plan.name }} now? Its {{ p.hosts }} host{{ 's' if p.hosts != 1 }} are updated one at a time; the schedule keeps its time.">{{ csrf_field() }}
+        <button type="submit" class="btn btn-sm">Run now</button>
+      </form>
+      <form method="post" action="/plugins/host_agent/maintenance/plans/{{ p.plan.id }}/toggle" style="margin:0;">{{ csrf_field() }}
+        <button type="submit" class="btn btn-sm btn-ghost">{{ 'Pause' if p.plan.enabled else 'Resume' }}</button>
+      </form>
+      <form method="post" action="/plugins/host_agent/maintenance/plans/{{ p.plan.id }}/delete" style="margin:0;"
+            data-confirm="Delete {{ p.plan.name }} and its run history? Hosts' own job history stays.">{{ csrf_field() }}
+        <button type="submit" class="btn btn-sm btn-danger">Delete</button>
+      </form>
+    </div>
+    {% endif %}
+  </div>
+  <div style="font-size:0.82rem;color:var(--text-muted);margin-bottom:0.6rem;">
+    Group <a href="/hosts/?group={{ p.group|urlencode }}">{{ p.group }}</a> ({{ p.hosts }} host{{ 's' if p.hosts != 1 }})
+    · {{ p.when }}
+    · reboot {{ {'if_required': 'only when required', 'always': 'always', 'never': 'never'}[p.plan.reboot_policy] }}
+    {% if p.next %}· next {{ p.next|when }}{% endif %}
+  </div>
+  {% for r in p.runs %}
+  <details {% if loop.first %}open{% endif %} style="margin-top:0.3rem;">
+    <summary style="cursor:pointer;font-size:0.82rem;">
+      <span class="{{ 'tone-warn' if r.run.state == 'failed' else '' }}">{{ {'running': 'Running', 'done': 'Done', 'failed': 'Failed', 'stopped': 'Stopped'}[r.run.state] }}</span>
+      <span style="color:var(--text-muted);">· {{ r.run.started_at|when }}{% if r.run.manual %} · run now{% endif %}{% if r.run.detail %} · {{ r.run.detail|truncate(100) }}{% endif %}</span>
+    </summary>
+    <div style="margin:0.3rem 0 0.4rem 1rem;">{{ run_hosts(r.rows) }}</div>
+  </details>
+  {% else %}
+  <p style="font-size:0.82rem;color:var(--text-dim);margin:0;">No runs yet.</p>
+  {% endfor %}
+</div>
+{% endfor %}
+{% else %}
+<div class="card">{{ empty_state("No maintenance plans yet." ~ (" Add one below." if session.user_role == 'admin' else "")) }}</div>
+{% endif %}
+
+{% if session.user_role == 'admin' %}
+<div class="card">
+  <h3 style="margin-bottom:0.6rem;">Add a plan</h3>
+  {% if groups %}
+  <form method="post" action="/plugins/host_agent/maintenance/plans"
+        style="display:flex;gap:0.75rem;align-items:flex-end;flex-wrap:wrap;">{{ csrf_field() }}
+    <div class="form-group" style="margin-bottom:0;">
+      <label>Name</label>
+      <input type="text" name="name" required maxlength="128" placeholder="Docker hosts, weekly" style="width:13rem;">
+    </div>
+    <div class="form-group" style="margin-bottom:0;">
+      <label>Group</label>
+      <select name="group_id" required>
+        {% for g in groups %}<option value="{{ g.id }}">{{ g.name }}</option>{% endfor %}
+      </select>
+    </div>
+    <div class="form-group" style="margin-bottom:0;">
+      <label>Day</label>
+      <select name="weekday">
+        {% for d in weekdays %}<option value="{{ loop.index0 }}" {% if loop.index0 == 6 %}selected{% endif %}>{{ d }}</option>{% endfor %}
+      </select>
+    </div>
+    <div class="form-group" style="margin-bottom:0;">
+      <label>Time</label>
+      <input type="time" name="at_time" value="03:00" required style="width:7rem;">
+    </div>
+    <div class="form-group" style="margin-bottom:0;">
+      <label>Time zone</label>
+      <input type="text" name="time_zone" value="{{ default_zone }}" required style="width:11rem;">
+    </div>
+    <div class="form-group" style="margin-bottom:0;">
+      <label>Window (min)</label>
+      <input type="number" name="window_minutes" value="240" min="30" max="1440" style="width:6rem;">
+    </div>
+    <div class="form-group" style="margin-bottom:0;">
+      <label>Reboot</label>
+      <select name="reboot_policy">
+        <option value="if_required" selected>Only when required</option>
+        <option value="always">Always</option>
+        <option value="never">Never</option>
+      </select>
+    </div>
+    <button type="submit" class="btn">Add plan</button>
+  </form>
+  <p style="font-size:0.78rem;color:var(--text-dim);margin:0.6rem 0 0;">
+    Hosts whose agent has no maintenance helper are skipped and listed; run Update agent on them.
+    No host is started once the window has passed.
+  </p>
+  {% else %}
+  <p style="font-size:0.85rem;color:var(--text-dim);margin:0;">
+    Plans work on inventory groups. Add one under <a href="/ansible/inventory/">Ansible → Inventory</a>.
+  </p>
+  {% endif %}
+</div>
+{% endif %}
+{% endblock %}

plugins/host_agent/templates/panel.html

@@ -27,7 +27,7 @@
     {% if maint_job %}
     {% set running = maint_job.state not in ('done', 'failed', 'timed_out') %}
     <div class="{{ 'tone-warn' if running or maint_job.state != 'done' else '' }}" title="{{ maint_job.detail or '' }}">
-      Maintenance: {{ maint_summary }}
+      <a href="/plugins/host_agent/maintenance" style="color:inherit;">Maintenance</a>: {{ maint_summary }}
       <span style="color:var(--text-muted);">· {% if running %}since {{ maint_job.state_at|ago }}{% else %}{{ maint_job.finished_at|ago }}{% endif %}{% if maint_job.state in ('failed', 'timed_out') and maint_job.detail %} · {{ maint_job.detail|truncate(90) }}{% endif %}</span>
     </div>
     {% endif %}