feat(host_agent): a group's updates and reboots on a weekly cadence, one host at a time (#5768)

#5768 "A maintenance cadence: a group's updates and reboots, one host at a
time, on a schedule". A plan names an inventory group, a weekday and time
in a time zone, a window and a reboot policy (only when required by
default). At its slot a run takes the group's hosts in order and gives each
a #5767 maintenance job. The next host starts only when the last is back in
service: its agent is posting and, for a swarm node, the node is active and
every replicated service is at its replicas. A failure stops the run, and
no host is started once the window has passed. A host's latest failure
shows as host_agent/<host>/maintenance_failed, for an ordinary alert rule.
The page is Hosts → Maintenance.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

plugins/host_agent/cadence.py

@@ -0,0 +1,302 @@
+# plugins/host_agent/cadence.py
+"""A group's maintenance cadence (#5768): every week, its hosts updated one at
+a time, each rebooting only when its update needs it (or as the plan says).
+
+A plan names an inventory group, a weekday and a time in its time zone, and a
+window. At that time a run starts with the group's hosts in a fixed order. Each
+host gets a maintenance job (maintenance.py); the next host starts only when
+the last one is back in service: its agent posting, its swarm node ready and
+active, every replicated service at its replicas. A job that fails, or a host
+that doesn't come back into service, stops the run, and the host's
+maintenance_failed figure goes to 1 for an alert rule to catch. No host is
+started after the window; the run then stops and says which hosts it didn't
+reach.
+
+The decisions are model-free here (plans and runs are anything with the
+attributes); the database side is below them, and the plugin's minute tick
+drives it. Everything is on the rows, so a restart of Steward, or the reboot of
+the host it runs on, picks up where it was.
+"""
+from __future__ import annotations
+
+import json
+from datetime import datetime, time, timedelta, timezone
+
+from . import maintenance
+
+WEEKDAYS = ("Monday", "Tuesday", "Wednesday", "Thursday", "Friday", "Saturday", "Sunday")
+# How long a host may take to be back in service after its job finished.
+SETTLE_LIMIT = timedelta(minutes=30)
+# An agent heard from this recently is posting.
+POSTING_WITHIN = timedelta(minutes=3)
+RUN_STATES = ("running", "done", "failed", "stopped")
+
+
+def _zone(name: str):
+    from steward.core.formatting import zone
+    return zone(name)
+
+
+def parse_at(at_time: str) -> time:
+    h, m = (int(x) for x in at_time.split(":", 1))
+    if not (0 <= h < 24 and 0 <= m < 60):
+        raise ValueError(at_time)
+    return time(h, m)
+
+
+def last_slot(plan, now: datetime) -> datetime:
+    """The most recent scheduled time at or before `now`, in UTC."""
+    tz = _zone(plan.time_zone)
+    local = now.astimezone(tz)
+    day = local.date() - timedelta(days=(local.weekday() - plan.weekday) % 7)
+    slot = datetime.combine(day, parse_at(plan.at_time), tzinfo=tz)
+    if slot > local:
+        slot -= timedelta(days=7)
+    return slot.astimezone(timezone.utc)
+
+
+def next_slot(plan, now: datetime) -> datetime:
+    return last_slot(plan, now) + timedelta(days=7)
+
+
+def is_due(plan, now: datetime) -> datetime | None:
+    """The slot to fire now, or None. A slot fires once (last_slot_at), and
+    only while its window is still open: a Steward that was down over the slot
+    starts late, not a day late."""
+    if not plan.enabled:
+        return None
+    slot = last_slot(plan, now)
+    since = plan.last_slot_at or plan.created_at
+    if since is not None and slot <= since:
+        return None
+    if now - slot >= timedelta(minutes=plan.window_minutes):
+        return None
+    return slot
+
+
+def back_in_service(*, last_seen_at, node, services, job_finished_at, now: datetime) -> str | None:
+    """None when the host is back in service, else why not yet. `node` is the
+    freshest swarm-node row for the job's node (None outside a swarm, or when
+    the job has no node); `services` the freshest row per swarm service. Only
+    rows reported after the job finished count, so a stale "all fine" from
+    before the reboot never passes."""
+    if last_seen_at is None or now - last_seen_at > POSTING_WITHIN:
+        return "its agent isn't posting yet"
+    if node is None:
+        return None
+    if node.updated_at < job_finished_at:
+        return "no swarm report since it came back"
+    if node.status != "ready" or node.availability != "active":
+        return f"its swarm node is {node.status}/{node.availability}"
+    fresh = [s for s in services if s.updated_at >= job_finished_at]
+    if not fresh:
+        return "no swarm report since it came back"
+    short = sorted(s.service_name for s in fresh
+                   if s.mode == "replicated" and s.running < s.desired)
+    if short:
+        return "services below their replicas: " + ", ".join(short[:5])
+    return None
+
+
+def next_move(run, plan, job, back: str | None, now: datetime) -> tuple[str, str | None]:
+    """What a running run does now: ("start", None) the current host's job;
+    ("wait", None); ("next", None) on to the next host; ("settling", why);
+    ("fail", why); ("stop", why) the window ended; ("done", None)."""
+    hosts = json.loads(run.host_ids)
+    if run.position >= len(hosts):
+        return "done", None
+    if job is None:
+        if now >= run.started_at + timedelta(minutes=plan.window_minutes):
+            left = len(hosts) - run.position
+            return "stop", f"the window ended with {left} host{'s' if left != 1 else ''} not started"
+        return "start", None
+    if job.state in maintenance.ACTIVE:
+        return "wait", None
+    if job.state != "done":
+        return "fail", job.detail or maintenance.LABELS[job.state]
+    if back is None:
+        return "next", None
+    if run.settle_since is not None and now - run.settle_since > SETTLE_LIMIT:
+        return "fail", f"not back in service within {int(SETTLE_LIMIT.total_seconds() // 60)} min: {back}"
+    return "settling", back
+
+
+def describe(plan) -> str:
+    """"Sundays at 03:00 (Europe/London), up to 4 h" for the page."""
+    hours = plan.window_minutes / 60
+    span = f"{hours:g} h" if plan.window_minutes % 60 == 0 else f"{plan.window_minutes} min"
+    return f"{WEEKDAYS[plan.weekday]}s at {plan.at_time} ({plan.time_zone}), up to {span}"
+
+
+# ── applied to the database: the minute tick, Run now, the page ──────────────
+
+async def group_hosts(session, group_id: str) -> list:
+    """The group's hosts (through their linked Ansible targets), by name."""
+    from sqlalchemy import select
+    from steward.models.ansible_inventory import AnsibleTarget, ansible_target_groups
+    from steward.models.hosts import Host
+    rows = (await session.execute(
+        select(Host).join(AnsibleTarget, AnsibleTarget.host_id == Host.id)
+        .join(ansible_target_groups, ansible_target_groups.c.target_id == AnsibleTarget.id)
+        .where(ansible_target_groups.c.group_id == group_id))).scalars().all()
+    return sorted({h.id: h for h in rows}.values(), key=lambda h: h.name.lower())
+
+
+async def start_run(session, plan, now: datetime, manual: bool = False):
+    """A run of the plan with the group's hosts in order. A host whose agent
+    has no maintenance helper is passed over, and the run says so."""
+    from sqlalchemy import select
+    from .models import HostAgentRegistration, HostMaintenanceRun
+    hosts = await group_hosts(session, plan.group_id)
+    regs = {r.host_id: r for r in (await session.execute(select(HostAgentRegistration).where(
+        HostAgentRegistration.host_id.in_([h.id for h in hosts])))).scalars()} if hosts else {}
+    order, skipped = [], []
+    for h in hosts:
+        reg = regs.get(h.id)
+        if reg is None:
+            skipped.append({"host_id": h.id, "reason": "no agent"})
+        elif not reg.maint_helper:
+            skipped.append({"host_id": h.id, "reason": "no maintenance helper: run Update agent"})
+        else:
+            order.append(h.id)
+    run = HostMaintenanceRun(plan_id=plan.id, state="running", host_ids=json.dumps(order),
+                             position=0, skipped=json.dumps(skipped), manual=manual,
+                             started_at=now)
+    session.add(run)
+    await session.flush()
+    return run
+
+
+async def _back(session, host, job, reg, now: datetime) -> str | None:
+    from sqlalchemy import select
+    node = None
+    services: list = []
+    if job.node_id:
+        try:
+            from plugins.docker.models import DockerSwarmNode, DockerSwarmService
+        except ImportError:
+            DockerSwarmNode = DockerSwarmService = None
+        if DockerSwarmNode is not None:
+            node = (await session.execute(select(DockerSwarmNode).where(
+                DockerSwarmNode.node_id == job.node_id)
+                .order_by(DockerSwarmNode.updated_at.desc()).limit(1))).scalar_one_or_none()
+            freshest: dict = {}
+            for s in (await session.execute(select(DockerSwarmService))).scalars():
+                if s.service_name not in freshest or s.updated_at > freshest[s.service_name].updated_at:
+                    freshest[s.service_name] = s
+            services = list(freshest.values())
+    return back_in_service(last_seen_at=reg.last_seen_at if reg else None, node=node,
+                           services=services, job_finished_at=job.finished_at or now, now=now)
+
+
+async def advance(session, run, plan, now: datetime) -> str:
+    """Move one running run on as far as it can go now; returns its state."""
+    from sqlalchemy import select
+    from steward.models.hosts import Host
+    from .models import HostAgentRegistration, HostMaintenanceJob
+    for _ in range(64):                             # a host per pass; bounded
+        hosts = json.loads(run.host_ids)
+        host_id = hosts[run.position] if run.position < len(hosts) else None
+        job = host = reg = None
+        if host_id is not None:
+            host = await session.get(Host, host_id)
+            reg = (await session.execute(select(HostAgentRegistration).where(
+                HostAgentRegistration.host_id == host_id))).scalar_one_or_none()
+            job = (await session.execute(select(HostMaintenanceJob).where(
+                HostMaintenanceJob.run_id == run.id, HostMaintenanceJob.host_id == host_id)
+                .order_by(HostMaintenanceJob.created_at.desc()).limit(1))).scalar_one_or_none()
+        back = None
+        if job is not None and job.state == "done":
+            back = await _back(session, host, job, reg, now)
+        move, why = next_move(run, plan, job, back, now)
+        if move == "start":
+            if host is None:
+                _skip(run, host_id, "the host is gone")
+                continue
+            try:
+                await maintenance.create_job(session, host, reg, plan.created_by,
+                                             plan.reboot_policy, now, run_id=run.id)
+            except maintenance.CannotStart as exc:
+                _skip(run, host_id, str(exc))
+                continue
+            return run.state
+        if move == "next":
+            run.position += 1
+            run.settle_since = None
+            continue
+        if move == "settling":
+            run.settle_since = run.settle_since or now
+            return run.state
+        if move == "fail":
+            if job is not None and job.state == "done":
+                maintenance._finish(job, "failed", now, why)     # the host never came back into service
+            name = host.name if host else host_id
+            _end(run, "failed", now, f"{name}: {why}")
+        elif move == "stop":
+            _end(run, "stopped", now, why)
+        elif move == "done":
+            _end(run, "done", now, None)
+        return run.state
+    return run.state
+
+
+def _skip(run, host_id: str, reason: str) -> None:
+    skipped = json.loads(run.skipped)
+    skipped.append({"host_id": host_id, "reason": reason})
+    run.skipped = json.dumps(skipped)
+    hosts = json.loads(run.host_ids)
+    hosts.pop(run.position)
+    run.host_ids = json.dumps(hosts)
+
+
+def _end(run, state: str, now: datetime, detail: str | None) -> None:
+    run.state, run.finished_at, run.settle_since = state, now, None
+    if detail:
+        run.detail = detail[:maintenance.DETAIL_MAX]
+
+
+async def tick(session, now: datetime) -> list[str]:
+    """Start due plans and move running runs on; returns lines worth logging."""
+    from sqlalchemy import select
+    from .models import HostMaintenancePlan, HostMaintenanceRun
+    said = []
+    running = {r.plan_id: r for r in (await session.execute(select(HostMaintenanceRun).where(
+        HostMaintenanceRun.state == "running"))).scalars()}
+    for plan in (await session.execute(select(HostMaintenancePlan))).scalars():
+        slot = is_due(plan, now)
+        if slot is None:
+            continue
+        plan.last_slot_at = slot
+        if plan.id in running:
+            said.append(f"plan {plan.name!r}: the {slot:%Y-%m-%d %H:%M}Z slot came with a run still going")
+            continue
+        running[plan.id] = await start_run(session, plan, now)
+        said.append(f"plan {plan.name!r}: run started")
+    for run in running.values():
+        plan = await session.get(HostMaintenancePlan, run.plan_id)
+        if plan is None:
+            continue
+        before = run.state
+        after = await advance(session, run, plan, now)
+        if after != before:
+            said.append(f"plan {plan.name!r}: run {after}" + (f" ({run.detail})" if run.detail else ""))
+    return said
+
+
+async def failed_hosts(session, now: datetime) -> dict[str, float]:
+    """{host name: 1 or 0}: whether each host's latest finished maintenance job
+    failed or timed out, for the maintenance_failed figure an alert rule
+    watches. A later job that succeeds puts it back to 0."""
+    from sqlalchemy import select
+    from steward.models.hosts import Host
+    from .models import HostMaintenanceJob
+    rows = (await session.execute(
+        select(Host.name, HostMaintenanceJob.state, HostMaintenanceJob.finished_at)
+        .join(Host, Host.id == HostMaintenanceJob.host_id)
+        .where(HostMaintenanceJob.state.in_(maintenance.FINISHED),
+               HostMaintenanceJob.finished_at >= now - timedelta(days=30)))).all()
+    latest: dict = {}
+    for name, state, finished in rows:
+        if name not in latest or finished > latest[name][1]:
+            latest[name] = (state, finished)
+    return {name: 0.0 if state == "done" else 1.0 for name, (state, _) in latest.items()}