feat(host_agent): a group's updates and reboots on a weekly cadence, one host at a time (#5768)

#5768 "A maintenance cadence: a group's updates and reboots, one host at a
time, on a schedule". A plan names an inventory group, a weekday and time
in a time zone, a window and a reboot policy (only when required by
default). At its slot a run takes the group's hosts in order and gives each
a #5767 maintenance job. The next host starts only when the last is back in
service: its agent is posting and, for a swarm node, the node is active and
every replicated service is at its replicas. A failure stops the run, and
no host is started once the window has passed. A host's latest failure
shows as host_agent/<host>/maintenance_failed, for an ordinary alert rule.
The page is Hosts → Maintenance.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

docs/plugins/host-agent-design.md

@@ -221,6 +221,37 @@ since-when; a finished one says what it did ("Done: 12 packages updated,
 rebooted"). Under Manage agent, **Update now** (with "never reboot") starts a
 job, or the panel says why one can't start.
 
+### A maintenance cadence: a group, one host at a time (#5768)
+
+`plugins/host_agent/cadence.py`, driven by the same minute tick as the jobs.
+A **plan** (`host_maintenance_plans`) names an inventory group, a weekday, a
+time in a time zone (the creator's, by default), a window and a reboot policy:
+`if_required` (the default), `always` or `never`. The page is **Hosts →
+Maintenance** (`/plugins/host_agent/maintenance`).
+
+- **When.** A plan's slot is the latest weekday-and-time at or before now, in
+  its zone. A slot fires once (`last_slot_at`) and only while its window is
+  open, so a Steward that was down over the slot starts late, not a day late.
+  A new or resumed plan starts from the next slot. **Run now** starts a run
+  without touching the schedule (the Ansible ruling on Run now).
+- **A run** (`host_maintenance_runs`) fixes its hosts at the start, by name:
+  the group's hosts through their linked Ansible targets. A host with no agent,
+  or no maintenance helper, is skipped and listed.
+- **One at a time.** Each host gets a #5767 job (`run_id` on the job). The next
+  host starts only when the last is **back in service**: its agent posting in
+  the last 3 min and, for a swarm node, the node `ready`/`active` and every
+  replicated service at its replicas, judged only from swarm reports made
+  after the job finished. A host not back within 30 min fails the run, and
+  its job is marked failed with the reason.
+- **Stops.** A failed or timed-out job fails the run. No host is started after
+  the window; the run then stops and says how many weren't reached.
+- **Alerting** is an ordinary alert rule on `host_agent/<host>/maintenance_failed`:
+  1 when the host's latest job (in 30 days) failed, timed out or didn't come
+  back into service, 0 once a later one succeeds.
+- The page leads with a running run and its hosts, then failed runs from the
+  last week, then each plan with its schedule, next slot and last five runs.
+  Each host's Agent panel shows its last maintenance job.
+
 ### What happened to the host (1.14.0, #5756)
 
 A post may carry `events`, said once each, with the next post that gets through:

plugins/host_agent/__init__.py

@@ -36,6 +36,12 @@ def setup(app: "Quart") -> None:
     prune_by_age(HostDowntime.started_at)
 
 
+def get_audit_entities() -> dict:
+    """Audit rows about a maintenance plan name it and link to the plans page."""
+    from .models import HostMaintenancePlan
+    return {"maintenance_plan": (HostMaintenancePlan, "/plugins/host_agent/maintenance")}
+
+
 def get_scheduled_tasks() -> list:
     from .scheduler import make_maintenance_task, make_task
     return [make_task(_app), make_maintenance_task(_app)]
@@ -70,12 +76,14 @@ def get_alert_metrics() -> dict[str, list[str]]:
     healthy agent sits in tens of MB and a few %. updates_pending and
     updates_security are what the package manager has waiting, and
     reboot_required is 1 when the host needs a reboot (agent ≥ 1.16.0):
-    "security updates waiting" is updates_security > 0."""
+    "security updates waiting" is updates_security > 0. maintenance_failed
+    comes from the scheduler (#5768): 1 when the host's latest maintenance job
+    failed or timed out, 0 once one succeeds: "maintenance failed" is > 0."""
     return {"host_agent": [
         "cpu_pct", "mem_used_pct", "mem_available_bytes", "swap_used_bytes",
         "disk_used_pct_worst", "disk_used_pct", "inodes_used_pct", "load_1m", "load_5m", "load_15m",
         "load_pct",
         "uptime_secs", "seconds_since_seen", "clock_off_secs", "mounts_stalled",
         "agent_rss_bytes", "agent_cpu_pct",
-        "updates_pending", "updates_security", "reboot_required",
+        "updates_pending", "updates_security", "reboot_required", "maintenance_failed",
     ]}

plugins/host_agent/cadence.py

@@ -0,0 +1,302 @@
+# plugins/host_agent/cadence.py
+"""A group's maintenance cadence (#5768): every week, its hosts updated one at
+a time, each rebooting only when its update needs it (or as the plan says).
+
+A plan names an inventory group, a weekday and a time in its time zone, and a
+window. At that time a run starts with the group's hosts in a fixed order. Each
+host gets a maintenance job (maintenance.py); the next host starts only when
+the last one is back in service: its agent posting, its swarm node ready and
+active, every replicated service at its replicas. A job that fails, or a host
+that doesn't come back into service, stops the run, and the host's
+maintenance_failed figure goes to 1 for an alert rule to catch. No host is
+started after the window; the run then stops and says which hosts it didn't
+reach.
+
+The decisions are model-free here (plans and runs are anything with the
+attributes); the database side is below them, and the plugin's minute tick
+drives it. Everything is on the rows, so a restart of Steward, or the reboot of
+the host it runs on, picks up where it was.
+"""
+from __future__ import annotations
+
+import json
+from datetime import datetime, time, timedelta, timezone
+
+from . import maintenance
+
+WEEKDAYS = ("Monday", "Tuesday", "Wednesday", "Thursday", "Friday", "Saturday", "Sunday")
+# How long a host may take to be back in service after its job finished.
+SETTLE_LIMIT = timedelta(minutes=30)
+# An agent heard from this recently is posting.
+POSTING_WITHIN = timedelta(minutes=3)
+RUN_STATES = ("running", "done", "failed", "stopped")
+
+
+def _zone(name: str):
+    from steward.core.formatting import zone
+    return zone(name)
+
+
+def parse_at(at_time: str) -> time:
+    h, m = (int(x) for x in at_time.split(":", 1))
+    if not (0 <= h < 24 and 0 <= m < 60):
+        raise ValueError(at_time)
+    return time(h, m)
+
+
+def last_slot(plan, now: datetime) -> datetime:
+    """The most recent scheduled time at or before `now`, in UTC."""
+    tz = _zone(plan.time_zone)
+    local = now.astimezone(tz)
+    day = local.date() - timedelta(days=(local.weekday() - plan.weekday) % 7)
+    slot = datetime.combine(day, parse_at(plan.at_time), tzinfo=tz)
+    if slot > local:
+        slot -= timedelta(days=7)
+    return slot.astimezone(timezone.utc)
+
+
+def next_slot(plan, now: datetime) -> datetime:
+    return last_slot(plan, now) + timedelta(days=7)
+
+
+def is_due(plan, now: datetime) -> datetime | None:
+    """The slot to fire now, or None. A slot fires once (last_slot_at), and
+    only while its window is still open: a Steward that was down over the slot
+    starts late, not a day late."""
+    if not plan.enabled:
+        return None
+    slot = last_slot(plan, now)
+    since = plan.last_slot_at or plan.created_at
+    if since is not None and slot <= since:
+        return None
+    if now - slot >= timedelta(minutes=plan.window_minutes):
+        return None
+    return slot
+
+
+def back_in_service(*, last_seen_at, node, services, job_finished_at, now: datetime) -> str | None:
+    """None when the host is back in service, else why not yet. `node` is the
+    freshest swarm-node row for the job's node (None outside a swarm, or when
+    the job has no node); `services` the freshest row per swarm service. Only
+    rows reported after the job finished count, so a stale "all fine" from
+    before the reboot never passes."""
+    if last_seen_at is None or now - last_seen_at > POSTING_WITHIN:
+        return "its agent isn't posting yet"
+    if node is None:
+        return None
+    if node.updated_at < job_finished_at:
+        return "no swarm report since it came back"
+    if node.status != "ready" or node.availability != "active":
+        return f"its swarm node is {node.status}/{node.availability}"
+    fresh = [s for s in services if s.updated_at >= job_finished_at]
+    if not fresh:
+        return "no swarm report since it came back"
+    short = sorted(s.service_name for s in fresh
+                   if s.mode == "replicated" and s.running < s.desired)
+    if short:
+        return "services below their replicas: " + ", ".join(short[:5])
+    return None
+
+
+def next_move(run, plan, job, back: str | None, now: datetime) -> tuple[str, str | None]:
+    """What a running run does now: ("start", None) the current host's job;
+    ("wait", None); ("next", None) on to the next host; ("settling", why);
+    ("fail", why); ("stop", why) the window ended; ("done", None)."""
+    hosts = json.loads(run.host_ids)
+    if run.position >= len(hosts):
+        return "done", None
+    if job is None:
+        if now >= run.started_at + timedelta(minutes=plan.window_minutes):
+            left = len(hosts) - run.position
+            return "stop", f"the window ended with {left} host{'s' if left != 1 else ''} not started"
+        return "start", None
+    if job.state in maintenance.ACTIVE:
+        return "wait", None
+    if job.state != "done":
+        return "fail", job.detail or maintenance.LABELS[job.state]
+    if back is None:
+        return "next", None
+    if run.settle_since is not None and now - run.settle_since > SETTLE_LIMIT:
+        return "fail", f"not back in service within {int(SETTLE_LIMIT.total_seconds() // 60)} min: {back}"
+    return "settling", back
+
+
+def describe(plan) -> str:
+    """"Sundays at 03:00 (Europe/London), up to 4 h" for the page."""
+    hours = plan.window_minutes / 60
+    span = f"{hours:g} h" if plan.window_minutes % 60 == 0 else f"{plan.window_minutes} min"
+    return f"{WEEKDAYS[plan.weekday]}s at {plan.at_time} ({plan.time_zone}), up to {span}"
+
+
+# ── applied to the database: the minute tick, Run now, the page ──────────────
+
+async def group_hosts(session, group_id: str) -> list:
+    """The group's hosts (through their linked Ansible targets), by name."""
+    from sqlalchemy import select
+    from steward.models.ansible_inventory import AnsibleTarget, ansible_target_groups
+    from steward.models.hosts import Host
+    rows = (await session.execute(
+        select(Host).join(AnsibleTarget, AnsibleTarget.host_id == Host.id)
+        .join(ansible_target_groups, ansible_target_groups.c.target_id == AnsibleTarget.id)
+        .where(ansible_target_groups.c.group_id == group_id))).scalars().all()
+    return sorted({h.id: h for h in rows}.values(), key=lambda h: h.name.lower())
+
+
+async def start_run(session, plan, now: datetime, manual: bool = False):
+    """A run of the plan with the group's hosts in order. A host whose agent
+    has no maintenance helper is passed over, and the run says so."""
+    from sqlalchemy import select
+    from .models import HostAgentRegistration, HostMaintenanceRun
+    hosts = await group_hosts(session, plan.group_id)
+    regs = {r.host_id: r for r in (await session.execute(select(HostAgentRegistration).where(
+        HostAgentRegistration.host_id.in_([h.id for h in hosts])))).scalars()} if hosts else {}
+    order, skipped = [], []
+    for h in hosts:
+        reg = regs.get(h.id)
+        if reg is None:
+            skipped.append({"host_id": h.id, "reason": "no agent"})
+        elif not reg.maint_helper:
+            skipped.append({"host_id": h.id, "reason": "no maintenance helper: run Update agent"})
+        else:
+            order.append(h.id)
+    run = HostMaintenanceRun(plan_id=plan.id, state="running", host_ids=json.dumps(order),
+                             position=0, skipped=json.dumps(skipped), manual=manual,
+                             started_at=now)
+    session.add(run)
+    await session.flush()
+    return run
+
+
+async def _back(session, host, job, reg, now: datetime) -> str | None:
+    from sqlalchemy import select
+    node = None
+    services: list = []
+    if job.node_id:
+        try:
+            from plugins.docker.models import DockerSwarmNode, DockerSwarmService
+        except ImportError:
+            DockerSwarmNode = DockerSwarmService = None
+        if DockerSwarmNode is not None:
+            node = (await session.execute(select(DockerSwarmNode).where(
+                DockerSwarmNode.node_id == job.node_id)
+                .order_by(DockerSwarmNode.updated_at.desc()).limit(1))).scalar_one_or_none()
+            freshest: dict = {}
+            for s in (await session.execute(select(DockerSwarmService))).scalars():
+                if s.service_name not in freshest or s.updated_at > freshest[s.service_name].updated_at:
+                    freshest[s.service_name] = s
+            services = list(freshest.values())
+    return back_in_service(last_seen_at=reg.last_seen_at if reg else None, node=node,
+                           services=services, job_finished_at=job.finished_at or now, now=now)
+
+
+async def advance(session, run, plan, now: datetime) -> str:
+    """Move one running run on as far as it can go now; returns its state."""
+    from sqlalchemy import select
+    from steward.models.hosts import Host
+    from .models import HostAgentRegistration, HostMaintenanceJob
+    for _ in range(64):                             # a host per pass; bounded
+        hosts = json.loads(run.host_ids)
+        host_id = hosts[run.position] if run.position < len(hosts) else None
+        job = host = reg = None
+        if host_id is not None:
+            host = await session.get(Host, host_id)
+            reg = (await session.execute(select(HostAgentRegistration).where(
+                HostAgentRegistration.host_id == host_id))).scalar_one_or_none()
+            job = (await session.execute(select(HostMaintenanceJob).where(
+                HostMaintenanceJob.run_id == run.id, HostMaintenanceJob.host_id == host_id)
+                .order_by(HostMaintenanceJob.created_at.desc()).limit(1))).scalar_one_or_none()
+        back = None
+        if job is not None and job.state == "done":
+            back = await _back(session, host, job, reg, now)
+        move, why = next_move(run, plan, job, back, now)
+        if move == "start":
+            if host is None:
+                _skip(run, host_id, "the host is gone")
+                continue
+            try:
+                await maintenance.create_job(session, host, reg, plan.created_by,
+                                             plan.reboot_policy, now, run_id=run.id)
+            except maintenance.CannotStart as exc:
+                _skip(run, host_id, str(exc))
+                continue
+            return run.state
+        if move == "next":
+            run.position += 1
+            run.settle_since = None
+            continue
+        if move == "settling":
+            run.settle_since = run.settle_since or now
+            return run.state
+        if move == "fail":
+            if job is not None and job.state == "done":
+                maintenance._finish(job, "failed", now, why)     # the host never came back into service
+            name = host.name if host else host_id
+            _end(run, "failed", now, f"{name}: {why}")
+        elif move == "stop":
+            _end(run, "stopped", now, why)
+        elif move == "done":
+            _end(run, "done", now, None)
+        return run.state
+    return run.state
+
+
+def _skip(run, host_id: str, reason: str) -> None:
+    skipped = json.loads(run.skipped)
+    skipped.append({"host_id": host_id, "reason": reason})
+    run.skipped = json.dumps(skipped)
+    hosts = json.loads(run.host_ids)
+    hosts.pop(run.position)
+    run.host_ids = json.dumps(hosts)
+
+
+def _end(run, state: str, now: datetime, detail: str | None) -> None:
+    run.state, run.finished_at, run.settle_since = state, now, None
+    if detail:
+        run.detail = detail[:maintenance.DETAIL_MAX]
+
+
+async def tick(session, now: datetime) -> list[str]:
+    """Start due plans and move running runs on; returns lines worth logging."""
+    from sqlalchemy import select
+    from .models import HostMaintenancePlan, HostMaintenanceRun
+    said = []
+    running = {r.plan_id: r for r in (await session.execute(select(HostMaintenanceRun).where(
+        HostMaintenanceRun.state == "running"))).scalars()}
+    for plan in (await session.execute(select(HostMaintenancePlan))).scalars():
+        slot = is_due(plan, now)
+        if slot is None:
+            continue
+        plan.last_slot_at = slot
+        if plan.id in running:
+            said.append(f"plan {plan.name!r}: the {slot:%Y-%m-%d %H:%M}Z slot came with a run still going")
+            continue
+        running[plan.id] = await start_run(session, plan, now)
+        said.append(f"plan {plan.name!r}: run started")
+    for run in running.values():
+        plan = await session.get(HostMaintenancePlan, run.plan_id)
+        if plan is None:
+            continue
+        before = run.state
+        after = await advance(session, run, plan, now)
+        if after != before:
+            said.append(f"plan {plan.name!r}: run {after}" + (f" ({run.detail})" if run.detail else ""))
+    return said
+
+
+async def failed_hosts(session, now: datetime) -> dict[str, float]:
+    """{host name: 1 or 0}: whether each host's latest finished maintenance job
+    failed or timed out, for the maintenance_failed figure an alert rule
+    watches. A later job that succeeds puts it back to 0."""
+    from sqlalchemy import select
+    from steward.models.hosts import Host
+    from .models import HostMaintenanceJob
+    rows = (await session.execute(
+        select(Host.name, HostMaintenanceJob.state, HostMaintenanceJob.finished_at)
+        .join(Host, Host.id == HostMaintenanceJob.host_id)
+        .where(HostMaintenanceJob.state.in_(maintenance.FINISHED),
+               HostMaintenanceJob.finished_at >= now - timedelta(days=30)))).all()
+    latest: dict = {}
+    for name, state, finished in rows:
+        if name not in latest or finished > latest[name][1]:
+            latest[name] = (state, finished)
+    return {name: 0.0 if state == "done" else 1.0 for name, (state, _) in latest.items()}

plugins/host_agent/maintenance.py

@@ -86,8 +86,19 @@ def handoff(job) -> dict | None:
     return out
 
 
+REBOOT_POLICIES = ("if_required", "always", "never")
+
+
+def wants_reboot(job) -> bool:
+    """Reboot only when the update says one is needed (the operator's
+    default), always (a plan may ask), or never."""
+    if job.reboot_policy == "always":
+        return True
+    return bool(job.reboot_needed) and job.reboot_policy == "if_required"
+
+
 def _after_update(job, now: datetime) -> None:
-    if not (job.reboot_needed and job.reboot_policy != "never"):
+    if not wants_reboot(job):
         _enter(job, "done", now)
     elif job.node_id and job.drain_host_id:
         _enter(job, "draining", now)
@@ -296,7 +307,7 @@ async def plan_job(session, host, reg, reboot_policy: str = "if_required") -> tu
     from .models import HostAgentRegistration, HostMaintenanceJob
     if reg is None or not reg.maint_helper:
         raise CannotStart("its agent has no maintenance helper: run Update agent first")
-    if reboot_policy not in ("if_required", "never"):
+    if reboot_policy not in REBOOT_POLICIES:
         raise CannotStart(f"unknown reboot policy {reboot_policy!r}")
     busy = (await session.execute(select(HostMaintenanceJob.id).where(
         HostMaintenanceJob.host_id == host.id,
@@ -312,21 +323,23 @@ async def plan_job(session, host, reg, reboot_policy: str = "if_required") -> tu
 
 
 async def create_job(session, host, reg, user_id: str | None,
-                     reboot_policy: str = "if_required", now: datetime | None = None):
+                     reboot_policy: str = "if_required", now: datetime | None = None,
+                     run_id: str | None = None):
     """Queue a maintenance job for `host`; it starts with the agent's next post."""
     from .models import HostMaintenanceJob
     now = now or utcnow()
     node_id, drain_host_id = await plan_job(session, host, reg, reboot_policy)
     job = HostMaintenanceJob(host_id=host.id, state="queued", state_at=now,
                              reboot_policy=reboot_policy, node_id=node_id,
-                             drain_host_id=drain_host_id, created_by=user_id, created_at=now)
+                             drain_host_id=drain_host_id, created_by=user_id, created_at=now,
+                             run_id=run_id)
     session.add(job)
     await session.flush()
     return job
 
 
-async def latest_job(session, host_id: str, now: datetime, days: int = 7):
-    """The host's running job, else its last one in `days`, for the panel."""
+async def latest_job(session, host_id: str, now: datetime, days: int = 366):
+    """The host's running job, else its last one (within a year), for the panel."""
     from sqlalchemy import select
     from .models import HostMaintenanceJob
     return (await session.execute(select(HostMaintenanceJob).where(

plugins/host_agent/migrations/versions/host_agent_008_maint_plans.py

@@ -0,0 +1,61 @@
+"""host_agent: a group's maintenance cadence and its runs (#5768)
+
+Revision ID: host_agent_008_maint_plans
+"""
+from typing import Sequence, Union
+from alembic import op
+import sqlalchemy as sa
+
+revision: str = "host_agent_008_maint_plans"
+down_revision: Union[str, None] = "host_agent_007_maint_jobs"
+branch_labels: Union[str, Sequence[str], None] = None
+# ansible_groups (core 0016) must exist before the group_id key.
+depends_on: Union[str, Sequence[str], None] = ("0016_ansible_inventory_groups",)
+
+
+def upgrade() -> None:
+    op.create_table(
+        "host_maintenance_plans",
+        sa.Column("id", sa.String(36), primary_key=True),
+        sa.Column("name", sa.String(128), nullable=False),
+        sa.Column("group_id", sa.String(36), sa.ForeignKey("ansible_groups.id", ondelete="CASCADE"),
+                  nullable=False),
+        sa.Column("weekday", sa.Integer(), nullable=False),
+        sa.Column("at_time", sa.String(5), nullable=False),
+        sa.Column("time_zone", sa.String(64), nullable=False, server_default="UTC"),
+        sa.Column("window_minutes", sa.Integer(), nullable=False, server_default="240"),
+        sa.Column("reboot_policy", sa.String(16), nullable=False, server_default="if_required"),
+        sa.Column("enabled", sa.Boolean(), nullable=False, server_default=sa.true()),
+        sa.Column("last_slot_at", sa.DateTime(timezone=True), nullable=True),
+        sa.Column("created_by", sa.String(36), sa.ForeignKey("users.id", ondelete="SET NULL"),
+                  nullable=True),
+        sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
+    )
+    op.create_table(
+        "host_maintenance_runs",
+        sa.Column("id", sa.String(36), primary_key=True),
+        sa.Column("plan_id", sa.String(36),
+                  sa.ForeignKey("host_maintenance_plans.id", ondelete="CASCADE"), nullable=False),
+        sa.Column("state", sa.String(16), nullable=False),
+        sa.Column("host_ids", sa.Text(), nullable=False, server_default="[]"),
+        sa.Column("position", sa.Integer(), nullable=False, server_default="0"),
+        sa.Column("skipped", sa.Text(), nullable=False, server_default="[]"),
+        sa.Column("settle_since", sa.DateTime(timezone=True), nullable=True),
+        sa.Column("detail", sa.Text(), nullable=True),
+        sa.Column("manual", sa.Boolean(), nullable=False, server_default=sa.false()),
+        sa.Column("started_at", sa.DateTime(timezone=True), nullable=False),
+        sa.Column("finished_at", sa.DateTime(timezone=True), nullable=True),
+    )
+    op.create_index("ix_host_maintenance_runs_plan_id", "host_maintenance_runs", ["plan_id"])
+    op.create_index("ix_host_maintenance_runs_state", "host_maintenance_runs", ["state"])
+    op.add_column("host_maintenance_jobs", sa.Column(
+        "run_id", sa.String(36), sa.ForeignKey("host_maintenance_runs.id", ondelete="SET NULL"),
+        nullable=True))
+    op.create_index("ix_host_maintenance_jobs_run_id", "host_maintenance_jobs", ["run_id"])
+
+
+def downgrade() -> None:
+    op.drop_index("ix_host_maintenance_jobs_run_id", "host_maintenance_jobs")
+    op.drop_column("host_maintenance_jobs", "run_id")
+    op.drop_table("host_maintenance_runs")
+    op.drop_table("host_maintenance_plans")

plugins/host_agent/models.py

@@ -106,3 +106,52 @@ class HostMaintenanceJob(Base):
     created_by = Column(String(36), ForeignKey("users.id", ondelete="SET NULL"), nullable=True)
     created_at = Column(DateTime(timezone=True), nullable=False, default=_utcnow)
     finished_at = Column(DateTime(timezone=True), nullable=True)
+    # The cadence run this job belongs to (#5768); None for Update now.
+    run_id = Column(String(36), ForeignKey("host_maintenance_runs.id", ondelete="SET NULL"),
+                    nullable=True, index=True)
+
+
+class HostMaintenancePlan(Base):
+    """A group's maintenance cadence (#5768): every week on `weekday` at
+    `at_time` in `time_zone`, its hosts are updated one at a time, each
+    rebooting per `reboot_policy`. No host is started after `window_minutes`
+    from the run's start. last_slot_at is the last scheduled slot that fired;
+    Run now leaves it alone, so the schedule keeps its time."""
+    __tablename__ = "host_maintenance_plans"
+
+    id = Column(String(36), primary_key=True, default=_uuid)
+    name = Column(String(128), nullable=False)
+    group_id = Column(String(36), ForeignKey("ansible_groups.id", ondelete="CASCADE"),
+                      nullable=False)
+    weekday = Column(Integer, nullable=False)              # 0 = Monday
+    at_time = Column(String(5), nullable=False)            # "HH:MM"
+    time_zone = Column(String(64), nullable=False, default="UTC")
+    window_minutes = Column(Integer, nullable=False, default=240)
+    reboot_policy = Column(String(16), nullable=False, default="if_required")
+    enabled = Column(Boolean, nullable=False, default=True)
+    last_slot_at = Column(DateTime(timezone=True), nullable=True)
+    created_by = Column(String(36), ForeignKey("users.id", ondelete="SET NULL"), nullable=True)
+    created_at = Column(DateTime(timezone=True), nullable=False, default=_utcnow)
+
+
+class HostMaintenanceRun(Base):
+    """One run of a plan: its hosts in order, one at a time (#5768).
+
+    host_ids is the order fixed at the start (JSON list); position is the host
+    being worked on; skipped is [{"host_id", "reason"}] for hosts the run
+    passed over. settle_since is when the current host's job finished and the
+    run began waiting for the host to be back in service."""
+    __tablename__ = "host_maintenance_runs"
+
+    id = Column(String(36), primary_key=True, default=_uuid)
+    plan_id = Column(String(36), ForeignKey("host_maintenance_plans.id", ondelete="CASCADE"),
+                     nullable=False, index=True)
+    state = Column(String(16), nullable=False, default="running", index=True)
+    host_ids = Column(Text, nullable=False, default="[]")
+    position = Column(Integer, nullable=False, default=0)
+    skipped = Column(Text, nullable=False, default="[]")
+    settle_since = Column(DateTime(timezone=True), nullable=True)
+    detail = Column(Text, nullable=True)
+    manual = Column(Boolean, nullable=False, default=False)
+    started_at = Column(DateTime(timezone=True), nullable=False, default=_utcnow)
+    finished_at = Column(DateTime(timezone=True), nullable=True)

plugins/host_agent/routes.py

@@ -1343,3 +1343,172 @@ async def update_via_ansible():
     if err:
         return _error(400, "update_failed", err)
     return redirect(url_for("ansible.run_detail", run_id=run.id))
+
+
+# ── Maintenance cadence (#5768) ───────────────────────────────────────────────
+
+def _run_rows(run, jobs: dict, names: dict) -> list[dict]:
+    """A run's hosts in order, each with where it stands, then those passed over."""
+    import json as _json
+    rows = []
+    for i, host_id in enumerate(_json.loads(run.host_ids)):
+        job = jobs.get((run.id, host_id))
+        if job is not None:
+            text = maintenance.summary(job)
+            bad = job.state in ("failed", "timed_out")
+            busy = job.state in maintenance.ACTIVE
+        elif run.state == "running" and i == run.position:
+            text, bad, busy = "Starting", False, True
+        else:
+            text, bad, busy = ("Not reached" if run.state != "running" else "Waiting"), False, False
+        if run.state == "running" and i == run.position and job is not None and job.state == "done":
+            text, busy = "Coming back into service", True
+        rows.append({"host_id": host_id, "name": names.get(host_id, host_id), "text": text,
+                     "bad": bad, "busy": busy, "detail": job.detail if job is not None else None})
+    for s in _json.loads(run.skipped):
+        rows.append({"host_id": s["host_id"], "name": names.get(s["host_id"], s["host_id"]),
+                     "text": f"Skipped: {s['reason']}", "bad": False, "busy": False, "detail": None})
+    return rows
+
+
+@host_agent_bp.get("/maintenance")
+@require_role(UserRole.viewer)
+async def maintenance_page():
+    """Maintenance plans: a group's updates and reboots on a weekly cadence,
+    one host at a time. A running run first, then a failed one, then the plans
+    and their recent runs."""
+    from steward.models.ansible_inventory import AnsibleGroup
+    from steward.models.users import User
+    from . import cadence
+    from .models import HostMaintenanceJob, HostMaintenancePlan, HostMaintenanceRun
+    now = datetime.now(timezone.utc)
+    async with current_app.db_sessionmaker() as db:
+        plans = (await db.execute(select(HostMaintenancePlan)
+                                  .order_by(HostMaintenancePlan.name))).scalars().all()
+        groups = (await db.execute(select(AnsibleGroup).order_by(AnsibleGroup.name))).scalars().all()
+        runs = (await db.execute(select(HostMaintenanceRun)
+                                 .order_by(HostMaintenanceRun.started_at.desc()).limit(60))).scalars().all()
+        jobs = {}
+        if runs:
+            for j in (await db.execute(select(HostMaintenanceJob).where(
+                    HostMaintenanceJob.run_id.in_([r.id for r in runs]))
+                    .order_by(HostMaintenanceJob.created_at))).scalars():
+                jobs[(j.run_id, j.host_id)] = j
+        names = dict((await db.execute(select(Host.id, Host.name))).all())
+        me = await db.get(User, session.get("user_id")) if session.get("user_id") else None
+        sizes = {g.id: len(await cadence.group_hosts(db, g.id)) for g in groups}
+    group_name = {g.id: g.name for g in groups}
+    by_plan: dict = {}
+    for r in runs:
+        by_plan.setdefault(r.plan_id, []).append(r)
+    view = []
+    for p in plans:
+        prs = by_plan.get(p.id, [])[:5]
+        view.append({"plan": p, "group": group_name.get(p.group_id, "—"),
+                     "hosts": sizes.get(p.group_id, 0), "when": cadence.describe(p),
+                     "next": cadence.next_slot(p, now) if p.enabled else None,
+                     "runs": [{"run": r, "rows": _run_rows(r, jobs, names)} for r in prs]})
+    # What needs attention first: a run going now, then a run that failed lately.
+    plan_name = {p.id: p.name for p in plans}
+    live = [{"run": r, "plan": plan_name.get(r.plan_id, "—"), "rows": _run_rows(r, jobs, names)}
+            for r in runs if r.state == "running"]
+    failed = [{"run": r, "plan": plan_name.get(r.plan_id, "—")} for r in runs
+              if r.state == "failed" and r.finished_at and now - r.finished_at < timedelta(days=7)]
+    return await render_template(
+        "maintenance.html", plans=view, groups=groups, live=live, failed=failed,
+        weekdays=cadence.WEEKDAYS, policies=maintenance.REBOOT_POLICIES,
+        default_zone=(me.time_zone if me and me.time_zone else "UTC"))
+
+
+@host_agent_bp.post("/maintenance/plans")
+@require_role(UserRole.admin)
+async def maintenance_plan_create():
+    from steward.core.formatting import zone
+    from steward.models.ansible_inventory import AnsibleGroup
+    from . import cadence
+    from .models import HostMaintenancePlan
+    form = await request.form
+    name = (form.get("name") or "").strip()[:128]
+    try:
+        weekday = int(form.get("weekday", ""))
+        at_time = (form.get("at_time") or "").strip()
+        cadence.parse_at(at_time)
+        window = int(form.get("window_minutes") or 240)
+    except ValueError:
+        return _error(400, "invalid", "weekday, time (HH:MM) and window must be numbers")
+    policy = form.get("reboot_policy") or "if_required"
+    tz = (form.get("time_zone") or "UTC").strip()
+    if not name or not 0 <= weekday <= 6 or not 30 <= window <= 24 * 60 \
+            or policy not in maintenance.REBOOT_POLICIES:
+        return _error(400, "invalid", "name, a weekday, a window of 30 min to 24 h and a reboot policy are needed")
+    if zone(tz) is timezone.utc and tz.upper() != "UTC":
+        return _error(400, "invalid", f"unknown time zone {tz!r}")
+    async with current_app.db_sessionmaker() as db:
+        if await db.get(AnsibleGroup, form.get("group_id") or "") is None:
+            return _error(400, "invalid", "pick an inventory group")
+        plan = HostMaintenancePlan(name=name, group_id=form["group_id"], weekday=weekday,
+                                   at_time=f"{cadence.parse_at(at_time):%H:%M}", time_zone=tz,
+                                   window_minutes=window, reboot_policy=policy, enabled=True,
+                                   created_by=session.get("user_id"),
+                                   # the first slot is the next one, not one already past
+                                   last_slot_at=datetime.now(timezone.utc))
+        db.add(plan)
+        await db.commit()
+    await audit_request("host_agent.maintenance_plan_created", "maintenance_plan", plan.id,
+                        {"name": name})
+    return redirect(url_for("host_agent.maintenance_page"))
+
+
+async def _plan_action(plan_id: str, act):
+    from .models import HostMaintenancePlan
+    async with current_app.db_sessionmaker() as db:
+        plan = await db.get(HostMaintenancePlan, plan_id)
+        if plan is None:
+            return _error(404, "not_found")
+        out = await act(db, plan)
+        if out is not None:
+            return out
+        await db.commit()
+    return redirect(url_for("host_agent.maintenance_page"))
+
+
+@host_agent_bp.post("/maintenance/plans/<plan_id>/toggle")
+@require_role(UserRole.admin)
+async def maintenance_plan_toggle(plan_id: str):
+    async def act(db, plan):
+        plan.enabled = not plan.enabled
+        if plan.enabled:
+            plan.last_slot_at = datetime.now(timezone.utc)   # no catching up on a missed slot
+    resp = await _plan_action(plan_id, act)
+    await audit_request("host_agent.maintenance_plan_toggled", "maintenance_plan", plan_id)
+    return resp
+
+
+@host_agent_bp.post("/maintenance/plans/<plan_id>/delete")
+@require_role(UserRole.admin)
+async def maintenance_plan_delete(plan_id: str):
+    async def act(db, plan):
+        await db.delete(plan)
+    resp = await _plan_action(plan_id, act)
+    await audit_request("host_agent.maintenance_plan_deleted", "maintenance_plan", plan_id)
+    return resp
+
+
+@host_agent_bp.post("/maintenance/plans/<plan_id>/run")
+@require_role(UserRole.admin)
+async def maintenance_plan_run_now(plan_id: str):
+    """Run the plan now. Its schedule keeps its time (Ansible ruling: Run now
+    leaves the schedule as it was)."""
+    from . import cadence
+    from .models import HostMaintenanceRun
+
+    async def act(db, plan):
+        busy = (await db.execute(select(HostMaintenanceRun.id).where(
+            HostMaintenanceRun.plan_id == plan.id,
+            HostMaintenanceRun.state == "running"))).first()
+        if busy is not None:
+            return _error(409, "busy", "this plan already has a run going")
+        await cadence.start_run(db, plan, datetime.now(timezone.utc), manual=True)
+    resp = await _plan_action(plan_id, act)
+    await audit_request("host_agent.maintenance_plan_run", "maintenance_plan", plan_id)
+    return resp

plugins/host_agent/scheduler.py

@@ -122,16 +122,27 @@ def make_task(app) -> ScheduledTask:
 
 
 def make_maintenance_task(app) -> ScheduledTask:
-    """Gives up on maintenance jobs past their deadline (#5767). Every other
-    move a job makes comes from its agent's posts."""
+    """Gives up on maintenance jobs past their deadline (#5767), starts due
+    maintenance plans and moves their runs on one host at a time (#5768), and
+    records each host's maintenance_failed for an alert rule. Every move a job
+    itself makes comes from its agent's posts."""
     async def _tick():
-        from . import maintenance
+        from . import cadence, maintenance
+        now = datetime.now(timezone.utc)
         try:
             async with app.db_sessionmaker() as session:
                 async with session.begin():
-                    gave_up = await maintenance.tick(session, datetime.now(timezone.utc))
+                    gave_up = await maintenance.tick(session, now)
+                    said = await cadence.tick(session, now)
+                    batch = MetricBatch()
+                    for name, value in (await cadence.failed_hosts(session, now)).items():
+                        batch.add(source_module="host_agent", resource_name=name,
+                                  metric_name="maintenance_failed", value=value, recorded_at=now)
+                    await batch.record(session)
             if gave_up:
                 logger.warning("host_agent: %d maintenance job(s) timed out", gave_up)
+            for line in said:
+                logger.info("host_agent maintenance: %s", line)
         except Exception:
             logger.exception("host_agent: maintenance tick failed")
 

plugins/host_agent/templates/maintenance.html

@@ -0,0 +1,143 @@
+{% extends "base.html" %}
+{% from "_macros.html" import crumbs, empty_state, page_header %}
+{% block title %}Maintenance — Steward{% endblock %}
+{% block breadcrumb %}{{ crumbs([("Hosts", "/hosts/"), ("Maintenance", "")]) }}{% endblock %}
+{% block content %}
+{# A group's updates and reboots on a weekly cadence, one host at a time
+   (#5768). What needs attention first: a run going now, a run that failed;
+   then the plans with their recent runs. #}
+{% macro run_hosts(rows) -%}
+<div style="display:flex;flex-direction:column;gap:0.15rem;font-size:0.82rem;">
+  {% for r in rows %}
+  <div style="display:flex;gap:0.6rem;align-items:baseline;" {% if r.detail %}title="{{ r.detail }}"{% endif %}>
+    <a href="/hosts/{{ r.host_id }}" style="min-width:10rem;">{{ r.name }}</a>
+    <span class="{{ 'tone-warn' if r.bad else '' }}" style="{{ '' if r.bad or r.busy else 'color:var(--text-muted);' }}">{{ r.text }}</span>
+  </div>
+  {% endfor %}
+</div>
+{%- endmacro %}
+
+{{ page_header("Maintenance") }}
+<p style="color:var(--text-muted);font-size:0.84rem;margin:-0.4rem 0 1.25rem;">
+  Each plan updates an inventory group's hosts once a week, one host at a time. A host reboots
+  only when its update needs it, unless the plan says otherwise; a swarm node is drained first
+  and made active again after. The next host starts once the last is back in service. A failure
+  stops the run, and the host's <em>Maintenance failed</em> figure can drive an alert rule.
+</p>
+
+{% if live %}
+{% for l in live %}
+<div class="card">
+  <h3 style="margin-bottom:0.5rem;">Running: {{ l.plan }} <span style="font-size:0.78rem;color:var(--text-muted);font-weight:normal;">· started {{ l.run.started_at|ago }}{% if l.run.manual %} · run now{% endif %}</span></h3>
+  {{ run_hosts(l.rows) }}
+</div>
+{% endfor %}
+{% endif %}
+
+{% for f in failed %}
+<div class="card" style="border-color:color-mix(in srgb,var(--yellow) 40%,var(--border));">
+  <span class="tone-warn">Failed: {{ f.plan }}</span>
+  <span style="font-size:0.82rem;color:var(--text-muted);">· {{ f.run.finished_at|ago }} · {{ f.run.detail or "no detail" }}</span>
+</div>
+{% endfor %}
+
+{% if plans %}
+{% for p in plans %}
+<div class="card">
+  <div style="display:flex;align-items:baseline;justify-content:space-between;gap:1rem;flex-wrap:wrap;">
+    <h3 style="margin-bottom:0.2rem;">{{ p.plan.name }}{% if not p.plan.enabled %} <span style="font-size:0.78rem;color:var(--text-muted);font-weight:normal;">· paused</span>{% endif %}</h3>
+    {% if session.user_role == 'admin' %}
+    <div style="display:flex;gap:0.4rem;">
+      <form method="post" action="/plugins/host_agent/maintenance/plans/{{ p.plan.id }}/run" style="margin:0;"
+            data-confirm="Run {{ p.plan.name }} now? Its {{ p.hosts }} host{{ 's' if p.hosts != 1 }} are updated one at a time; the schedule keeps its time.">{{ csrf_field() }}
+        <button type="submit" class="btn btn-sm">Run now</button>
+      </form>
+      <form method="post" action="/plugins/host_agent/maintenance/plans/{{ p.plan.id }}/toggle" style="margin:0;">{{ csrf_field() }}
+        <button type="submit" class="btn btn-sm btn-ghost">{{ 'Pause' if p.plan.enabled else 'Resume' }}</button>
+      </form>
+      <form method="post" action="/plugins/host_agent/maintenance/plans/{{ p.plan.id }}/delete" style="margin:0;"
+            data-confirm="Delete {{ p.plan.name }} and its run history? Hosts' own job history stays.">{{ csrf_field() }}
+        <button type="submit" class="btn btn-sm btn-danger">Delete</button>
+      </form>
+    </div>
+    {% endif %}
+  </div>
+  <div style="font-size:0.82rem;color:var(--text-muted);margin-bottom:0.6rem;">
+    Group <a href="/hosts/?group={{ p.group|urlencode }}">{{ p.group }}</a> ({{ p.hosts }} host{{ 's' if p.hosts != 1 }})
+    · {{ p.when }}
+    · reboot {{ {'if_required': 'only when required', 'always': 'always', 'never': 'never'}[p.plan.reboot_policy] }}
+    {% if p.next %}· next {{ p.next|when }}{% endif %}
+  </div>
+  {% for r in p.runs %}
+  <details {% if loop.first %}open{% endif %} style="margin-top:0.3rem;">
+    <summary style="cursor:pointer;font-size:0.82rem;">
+      <span class="{{ 'tone-warn' if r.run.state == 'failed' else '' }}">{{ {'running': 'Running', 'done': 'Done', 'failed': 'Failed', 'stopped': 'Stopped'}[r.run.state] }}</span>
+      <span style="color:var(--text-muted);">· {{ r.run.started_at|when }}{% if r.run.manual %} · run now{% endif %}{% if r.run.detail %} · {{ r.run.detail|truncate(100) }}{% endif %}</span>
+    </summary>
+    <div style="margin:0.3rem 0 0.4rem 1rem;">{{ run_hosts(r.rows) }}</div>
+  </details>
+  {% else %}
+  <p style="font-size:0.82rem;color:var(--text-dim);margin:0;">No runs yet.</p>
+  {% endfor %}
+</div>
+{% endfor %}
+{% else %}
+<div class="card">{{ empty_state("No maintenance plans yet." ~ (" Add one below." if session.user_role == 'admin' else "")) }}</div>
+{% endif %}
+
+{% if session.user_role == 'admin' %}
+<div class="card">
+  <h3 style="margin-bottom:0.6rem;">Add a plan</h3>
+  {% if groups %}
+  <form method="post" action="/plugins/host_agent/maintenance/plans"
+        style="display:flex;gap:0.75rem;align-items:flex-end;flex-wrap:wrap;">{{ csrf_field() }}
+    <div class="form-group" style="margin-bottom:0;">
+      <label>Name</label>
+      <input type="text" name="name" required maxlength="128" placeholder="Docker hosts, weekly" style="width:13rem;">
+    </div>
+    <div class="form-group" style="margin-bottom:0;">
+      <label>Group</label>
+      <select name="group_id" required>
+        {% for g in groups %}<option value="{{ g.id }}">{{ g.name }}</option>{% endfor %}
+      </select>
+    </div>
+    <div class="form-group" style="margin-bottom:0;">
+      <label>Day</label>
+      <select name="weekday">
+        {% for d in weekdays %}<option value="{{ loop.index0 }}" {% if loop.index0 == 6 %}selected{% endif %}>{{ d }}</option>{% endfor %}
+      </select>
+    </div>
+    <div class="form-group" style="margin-bottom:0;">
+      <label>Time</label>
+      <input type="time" name="at_time" value="03:00" required style="width:7rem;">
+    </div>
+    <div class="form-group" style="margin-bottom:0;">
+      <label>Time zone</label>
+      <input type="text" name="time_zone" value="{{ default_zone }}" required style="width:11rem;">
+    </div>
+    <div class="form-group" style="margin-bottom:0;">
+      <label>Window (min)</label>
+      <input type="number" name="window_minutes" value="240" min="30" max="1440" style="width:6rem;">
+    </div>
+    <div class="form-group" style="margin-bottom:0;">
+      <label>Reboot</label>
+      <select name="reboot_policy">
+        <option value="if_required" selected>Only when required</option>
+        <option value="always">Always</option>
+        <option value="never">Never</option>
+      </select>
+    </div>
+    <button type="submit" class="btn">Add plan</button>
+  </form>
+  <p style="font-size:0.78rem;color:var(--text-dim);margin:0.6rem 0 0;">
+    Hosts whose agent has no maintenance helper are skipped and listed; run Update agent on them.
+    No host is started once the window has passed.
+  </p>
+  {% else %}
+  <p style="font-size:0.85rem;color:var(--text-dim);margin:0;">
+    Plans work on inventory groups. Add one under <a href="/ansible/inventory/">Ansible → Inventory</a>.
+  </p>
+  {% endif %}
+</div>
+{% endif %}
+{% endblock %}

plugins/host_agent/templates/panel.html

@@ -27,7 +27,7 @@
     {% if maint_job %}
     {% set running = maint_job.state not in ('done', 'failed', 'timed_out') %}
     <div class="{{ 'tone-warn' if running or maint_job.state != 'done' else '' }}" title="{{ maint_job.detail or '' }}">
-      Maintenance: {{ maint_summary }}
+      <a href="/plugins/host_agent/maintenance" style="color:inherit;">Maintenance</a>: {{ maint_summary }}
       <span style="color:var(--text-muted);">· {% if running %}since {{ maint_job.state_at|ago }}{% else %}{{ maint_job.finished_at|ago }}{% endif %}{% if maint_job.state in ('failed', 'timed_out') and maint_job.detail %} · {{ maint_job.detail|truncate(90) }}{% endif %}</span>
     </div>
     {% endif %}

steward/audit/service.py

@@ -138,6 +138,10 @@ ACTIONS = {
     "host.created": "added host",
     "host.renamed": "renamed host",
     "host_agent.maintenance": "started maintenance (update, reboot if needed) on host",
+    "host_agent.maintenance_plan_created": "added maintenance plan",
+    "host_agent.maintenance_plan_toggled": "paused or resumed maintenance plan",
+    "host_agent.maintenance_plan_deleted": "deleted maintenance plan",
+    "host_agent.maintenance_plan_run": "ran maintenance plan now",
     "host.deleted": "deleted host",
     "monitor.created": "added monitor",
     "monitor.updated": "changed monitor",

steward/backup/contents.py

@@ -16,6 +16,8 @@ from __future__ import annotations
 KEPT: frozenset[str] = frozenset({
     "app_settings", "users", "user_invites", "api_tokens",
     "hosts", "host_agent_registrations",
+    # A group's maintenance cadence (#5768): configuration.
+    "host_maintenance_plans",
     "monitors",
     "alert_rules", "alert_states", "maintenance_windows",
     "ansible_targets", "ansible_groups", "ansible_target_groups", "ansible_schedules",
@@ -39,7 +41,7 @@ EXCLUDED: frozenset[str] = frozenset({
     # A host's downtime as its agent reported it: history, like the metrics.
     "host_downtime",
     # Maintenance jobs (#5767): run history, like ansible_runs.
-    "host_maintenance_jobs",
+    "host_maintenance_jobs", "host_maintenance_runs",
     "email_outbox", "email_outbox_events", "password_resets",
     # Run history; its logs live on disk, not in the database.
     "ansible_runs",

steward/core/formatting.py

@@ -345,6 +345,7 @@ METRICS: dict[str, tuple[str, str]] = {
     "updates_pending": ("Updates waiting", "packages the host's package manager would upgrade, checked hourly"),
     "updates_security": ("Security updates waiting", "of the updates waiting, those from a security source"),
     "reboot_required": ("Reboot required", "1 when the host needs a reboot to finish an update, 0 when not"),
+    "maintenance_failed": ("Maintenance failed", "1 when the host's latest scheduled or manual update failed, timed out or didn't come back into service"),
 }
 
 # Where a rule's metric comes from, as the picker and the rules list say it.

steward/templates/hosts/list.html

@@ -17,6 +17,7 @@
         {% if session.user_role == 'admin' %}
         <a class="btn btn-ghost" href="/plugins/host_agent/fleet/">Agent fleet</a>
         {% endif %}
+        <a class="btn btn-ghost" href="/plugins/host_agent/maintenance">Maintenance</a>
         {% if can_add %}
         <button type="button" class="btn" onclick="document.getElementById('add-host').showModal()">Add host</button>
         {% endif %}

tests/core/test_host_page.py

@@ -304,7 +304,7 @@ async def test_the_agent_card_follows_a_maintenance_job_and_says_why_it_cant_sta
             maint_blocker="a maintenance job is already running on it", **kw)
         free = await render_plugin_template(app, "host_agent", "panel.html", maint_job=None,
                                             maint_blocker=None, **kw)
-    assert busy.index("Maintenance: Rebooting") < busy.index("4 updates waiting")
+    assert busy.index("Maintenance</a>: Rebooting") < busy.index("4 updates waiting")
     assert "Can't update from here: a maintenance job is already running on it." in busy
     assert ">Update now</button>" not in busy
     assert ">Update now</button>" in free and 'name="reboot_policy" value="never"' in free

tests/integration/test_host_maintenance_cadence.py

@@ -0,0 +1,134 @@
+"""Integration (#5768): a plan updates its group's hosts one at a time; the
+second starts only once the first is back, and a failure stops the run and
+flags the host for an alert rule.
+
+The plugin's cadence and models are reached through the module the app loaded
+(see test_docker_disk_page): importing the plugin's models again raises
+"Table already defined"."""
+from __future__ import annotations
+
+import asyncio
+import hashlib
+import importlib
+import os
+import uuid
+from datetime import datetime, timezone
+
+import pytest
+
+pytestmark = pytest.mark.integration
+
+
+def _app():
+    if not os.environ.get("STEWARD_DATABASE_URL"):
+        pytest.skip("needs Postgres")
+    from steward.app import create_app
+    return create_app(testing=False)
+
+
+def _ha(sub):
+    from steward.core.plugin_manager import get_loaded_plugin_modules
+    loaded = get_loaded_plugin_modules().get("host_agent")
+    base = loaded.__name__ if loaded is not None else "plugins.host_agent"
+    return importlib.import_module(f"{base}.{sub}")
+
+
+async def _seed(app):
+    from sqlalchemy import text
+    from steward.models.ansible_inventory import AnsibleGroup, AnsibleTarget
+    from steward.models.hosts import Host
+    from steward.models.users import User
+    tag = uuid.uuid4().hex[:8]
+    out = {"tokens": {}, "hosts": {}}
+    async with app.db_sessionmaker() as s:
+        async with s.begin():
+            user = User(username=f"cad-{tag}", email=f"cad-{tag}@test", password_hash="x")
+            group = AnsibleGroup(name=f"cad-{tag}")
+            s.add_all([user, group])
+            await s.flush()
+            for n in ("a", "b"):
+                host = Host(name=f"cad-{n}-{tag}", address="10.9.9.6")
+                s.add(host)
+                await s.flush()
+                target = AnsibleTarget(name=f"cad-{n}-{tag}", address="10.9.9.6", host_id=host.id)
+                target.groups = [group]
+                s.add(target)
+                token = f"tok-{n}-{tag}-{uuid.uuid4().hex}"
+                await s.execute(text(
+                    "INSERT INTO host_agent_registrations "
+                    "(id, host_id, token_hash, token_created_at, last_seen_at, maint_helper, "
+                    " created_at, updated_at) VALUES (:id, :h, :hash, now(), now(), true, now(), now())"),
+                    {"id": str(uuid.uuid4()), "h": host.id,
+                     "hash": hashlib.sha256(token.encode()).hexdigest()})
+                out["tokens"][n], out["hosts"][n] = token, host
+            plan_id = str(uuid.uuid4())
+            await s.execute(text(
+                "INSERT INTO host_maintenance_plans (id, name, group_id, weekday, at_time, time_zone, "
+                " window_minutes, reboot_policy, enabled, created_by, created_at) "
+                "VALUES (:id, :name, :g, 6, '03:00', 'UTC', 240, 'if_required', true, :u, now())"),
+                {"id": plan_id, "name": f"plan-{tag}", "g": group.id, "u": user.id})
+    out["plan_id"] = plan_id
+    return out
+
+
+async def _post(app, token, events=()):
+    resp = await app.test_client().post(
+        "/plugins/host_agent/ingest",
+        json={"samples": [{"ts": datetime.now(timezone.utc).isoformat(), "cpu_pct": 1.0}],
+              "events": list(events)},
+        headers={"Authorization": f"Bearer {token}"})
+    assert resp.status_code == 200, await resp.get_data(as_text=True)
+    return (await resp.get_json()).get("maintenance", [])
+
+
+async def _tick(app):
+    cadence = _ha("cadence")
+    async with app.db_sessionmaker() as s:
+        async with s.begin():
+            await cadence.tick(s, datetime.now(timezone.utc))
+            return await cadence.failed_hosts(s, datetime.now(timezone.utc))
+
+
+async def _run(app, plan_id):
+    from sqlalchemy import text
+    async with app.db_sessionmaker() as s:
+        return (await s.execute(text(
+            "SELECT state, position, detail FROM host_maintenance_runs WHERE plan_id = :p"),
+            {"p": plan_id})).one()
+
+
+def test_hosts_go_one_at_a_time_and_a_failure_stops_the_run():
+    app = _app()
+
+    async def _go():
+        seed = await _seed(app)
+        cadence, models = _ha("cadence"), _ha("models")
+        async with app.db_sessionmaker() as s:
+            async with s.begin():
+                plan = await s.get(models.HostMaintenancePlan, seed["plan_id"])
+                await cadence.start_run(s, plan, datetime.now(timezone.utc), manual=True)
+        await _tick(app)
+
+        # Only the first host (by name) has a job; the second waits.
+        ta, tb = seed["tokens"]["a"], seed["tokens"]["b"]
+        assert await _post(app, tb) == []
+        (step,) = await _post(app, ta)
+        assert step["action"] == "update"
+        done = {"kind": "maintenance", "id": step["id"], "action": "update", "state": "done",
+                "ok": True, "packages": 2, "reboot_required": False}
+        await _post(app, ta, [done])
+
+        # Back in service (posting, not a swarm node): the next host starts.
+        await _tick(app)
+        assert (await _run(app, seed["plan_id"]))[:2] == ("running", 1)
+        (step_b,) = await _post(app, tb)
+        failed = {"kind": "maintenance", "id": step_b["id"], "action": "update",
+                  "state": "failed", "ok": False, "detail": "dpkg was interrupted"}
+        await _post(app, tb, [failed])
+
+        flags = await _tick(app)
+        state, _, detail = await _run(app, seed["plan_id"])
+        assert state == "failed" and seed["hosts"]["b"].name in detail and "dpkg" in detail
+        assert flags[seed["hosts"]["b"].name] == 1.0 and flags[seed["hosts"]["a"].name] == 0.0
+
+    asyncio.run(_go())

tests/plugins/host_agent/test_cadence.py

@@ -0,0 +1,133 @@
+# tests/plugins/host_agent/test_cadence.py
+"""#5768: when a plan is due, when a host is back in service, and what a run
+does next. Model-free: plans, runs, jobs and swarm rows are SimpleNamespaces."""
+import json
+from datetime import datetime, timedelta, timezone
+from types import SimpleNamespace as NS
+
+import pytest
+
+from plugins.host_agent import cadence as c
+from plugins.host_agent import maintenance as m
+
+# Wednesday 2026-10-14 23:00 UTC
+NOW = datetime(2026, 10, 14, 23, 0, tzinfo=timezone.utc)
+
+
+def _plan(**kw):
+    base = dict(weekday=6, at_time="03:00", time_zone="UTC", window_minutes=240, enabled=True,
+                last_slot_at=None, created_at=NOW - timedelta(days=30), reboot_policy="if_required")
+    base.update(kw)
+    return NS(**base)
+
+
+def test_the_last_slot_is_the_latest_weekday_and_time_in_the_plans_zone():
+    assert c.last_slot(_plan(), NOW) == datetime(2026, 10, 11, 3, 0, tzinfo=timezone.utc)
+    # 03:00 in New York is 07:00 UTC in October (EDT)
+    ny = _plan(time_zone="America/New_York")
+    assert c.last_slot(ny, NOW) == datetime(2026, 10, 11, 7, 0, tzinfo=timezone.utc)
+    assert c.next_slot(_plan(), NOW) == datetime(2026, 10, 18, 3, 0, tzinfo=timezone.utc)
+    today = _plan(weekday=2, at_time="22:30")
+    assert c.last_slot(today, NOW) == datetime(2026, 10, 14, 22, 30, tzinfo=timezone.utc)
+
+
+def test_a_slot_fires_once_and_only_while_its_window_is_open():
+    plan = _plan(weekday=2, at_time="22:30")
+    assert c.is_due(plan, NOW) == datetime(2026, 10, 14, 22, 30, tzinfo=timezone.utc)
+    plan.last_slot_at = c.is_due(plan, NOW)
+    assert c.is_due(plan, NOW) is None                          # fired already
+    late = _plan(weekday=2, at_time="18:00")                    # 5 h ago, window 4 h
+    assert c.is_due(late, NOW) is None
+    assert c.is_due(_plan(weekday=2, at_time="22:30", enabled=False), NOW) is None
+    new = _plan(weekday=2, at_time="22:30", created_at=NOW - timedelta(minutes=5))
+    assert c.is_due(new, NOW) is None                           # made after the slot
+
+
+def _node(**kw):
+    base = dict(status="ready", availability="active", updated_at=NOW)
+    base.update(kw)
+    return NS(**base)
+
+
+def _svc(name, running, desired, mode="replicated"):
+    return NS(service_name=name, running=running, desired=desired, mode=mode, updated_at=NOW)
+
+
+def test_a_host_is_back_once_posting_its_node_active_and_its_services_at_replicas():
+    finished = NOW - timedelta(minutes=5)
+    kw = dict(job_finished_at=finished, now=NOW)
+    assert c.back_in_service(last_seen_at=None, node=None, services=[], **kw) == "its agent isn't posting yet"
+    assert c.back_in_service(last_seen_at=NOW, node=None, services=[], **kw) is None
+    assert "drain" in c.back_in_service(last_seen_at=NOW, node=_node(availability="drain"),
+                                        services=[_svc("web", 1, 1)], **kw)
+    assert c.back_in_service(last_seen_at=NOW, node=_node(updated_at=finished - timedelta(minutes=1)),
+                             services=[], **kw) == "no swarm report since it came back"
+    assert c.back_in_service(last_seen_at=NOW, node=_node(),
+                             services=[_svc("web", 1, 2), _svc("agent", 2, 3, mode="global")],
+                             **kw) == "services below their replicas: web"
+    assert c.back_in_service(last_seen_at=NOW, node=_node(), services=[_svc("web", 2, 2)], **kw) is None
+
+
+def _run(**kw):
+    base = dict(host_ids=json.dumps(["h1", "h2"]), position=0, started_at=NOW - timedelta(minutes=10),
+                settle_since=None, skipped="[]")
+    base.update(kw)
+    return NS(**base)
+
+
+def _job(state, detail=None):
+    return NS(state=state, detail=detail)
+
+
+@pytest.mark.parametrize("job,back,run_kw,expected", [
+    (None, None, {}, ("start", None)),
+    (_job("rebooting"), None, {}, ("wait", None)),
+    (_job("done"), None, {}, ("next", None)),
+    (_job("done"), "its agent isn't posting yet", {}, ("settling", "its agent isn't posting yet")),
+    (_job("failed", "update failed: dpkg"), None, {}, ("fail", "update failed: dpkg")),
+    (_job("timed_out"), None, {}, ("fail", "Timed out")),
+    (None, None, {"position": 2}, ("done", None)),
+])
+def test_what_a_run_does_next(job, back, run_kw, expected):
+    assert c.next_move(_run(**run_kw), _plan(), job, back, NOW) == expected
+
+
+def test_a_host_that_never_settles_fails_the_run_and_the_window_stops_new_hosts():
+    run = _run(settle_since=NOW - timedelta(minutes=31))
+    move, why = c.next_move(run, _plan(), _job("done"), "services below their replicas: web", NOW)
+    assert move == "fail" and why.startswith("not back in service within 30 min")
+    late = _run(started_at=NOW - timedelta(hours=5), position=1)
+    assert c.next_move(late, _plan(), None, None, NOW) == ("stop", "the window ended with 1 host not started")
+
+
+def test_a_plan_says_when_it_runs():
+    assert c.describe(_plan()) == "Sundays at 03:00 (UTC), up to 4 h"
+    assert c.describe(_plan(window_minutes=90)) == "Sundays at 03:00 (UTC), up to 90 min"
+
+
+def test_always_reboots_even_when_the_update_needs_none():
+    job = NS(id="j", state="updating", step_id="j-update", step_seen=False, reboot_policy="always",
+             reboot_needed=None, node_id=None, drain_host_id=None, packages=None, state_at=NOW,
+             finished_at=None, detail=None)
+    m.apply_result(job, {"id": "j-update", "state": "done", "ok": True, "reboot_required": False}, NOW)
+    assert job.state == "rebooting"
+
+
+async def test_the_page_leads_with_a_running_run_and_lists_the_plans(app):
+    from quart import session
+    from tests._templates import render_plugin_template
+    plan = NS(id="p1", name="Docker hosts", enabled=True, reboot_policy="if_required")
+    run = NS(id="r1", state="running", started_at=NOW, manual=True, detail=None, finished_at=None)
+    rows = [{"host_id": "h1", "name": "docker02", "text": "Rebooting", "bad": False, "busy": True, "detail": None},
+            {"host_id": "h2", "name": "docker03", "text": "Waiting", "bad": False, "busy": False, "detail": None}]
+    async with app.test_request_context("/plugins/host_agent/maintenance"):
+        session["user_role"] = "admin"
+        page = await render_plugin_template(
+            app, "host_agent", "maintenance.html",
+            plans=[{"plan": plan, "group": "docker", "hosts": 2, "when": "Sundays at 03:00 (UTC), up to 4 h",
+                    "next": NOW, "runs": [{"run": run, "rows": rows}]}],
+            groups=[NS(id="g1", name="docker")], live=[{"run": run, "plan": "Docker hosts", "rows": rows}],
+            failed=[], weekdays=c.WEEKDAYS, policies=m.REBOOT_POLICIES, default_zone="UTC")
+    assert page.index("Running: Docker hosts") < page.index("Add a plan")
+    assert page.index("docker02") < page.index("docker03")
+    assert "reboot only when required" in page and ">Run now</button>" in page