feat(host_agent): a group's updates and reboots on a weekly cadence, one host at a time (#5768)

#5768 "A maintenance cadence: a group's updates and reboots, one host at a
time, on a schedule". A plan names an inventory group, a weekday and time
in a time zone, a window and a reboot policy (only when required by
default). At its slot a run takes the group's hosts in order and gives each
a #5767 maintenance job. The next host starts only when the last is back in
service: its agent is posting and, for a swarm node, the node is active and
every replicated service is at its replicas. A failure stops the run, and
no host is started once the window has passed. A host's latest failure
shows as host_agent/<host>/maintenance_failed, for an ordinary alert rule.
The page is Hosts → Maintenance.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

tests/core/test_host_page.py

@@ -304,7 +304,7 @@ async def test_the_agent_card_follows_a_maintenance_job_and_says_why_it_cant_sta
             maint_blocker="a maintenance job is already running on it", **kw)
         free = await render_plugin_template(app, "host_agent", "panel.html", maint_job=None,
                                             maint_blocker=None, **kw)
-    assert busy.index("Maintenance: Rebooting") < busy.index("4 updates waiting")
+    assert busy.index("Maintenance</a>: Rebooting") < busy.index("4 updates waiting")
     assert "Can't update from here: a maintenance job is already running on it." in busy
     assert ">Update now</button>" not in busy
     assert ">Update now</button>" in free and 'name="reboot_policy" value="never"' in free

tests/integration/test_host_maintenance_cadence.py

@@ -0,0 +1,134 @@
+"""Integration (#5768): a plan updates its group's hosts one at a time; the
+second starts only once the first is back, and a failure stops the run and
+flags the host for an alert rule.
+
+The plugin's cadence and models are reached through the module the app loaded
+(see test_docker_disk_page): importing the plugin's models again raises
+"Table already defined"."""
+from __future__ import annotations
+
+import asyncio
+import hashlib
+import importlib
+import os
+import uuid
+from datetime import datetime, timezone
+
+import pytest
+
+pytestmark = pytest.mark.integration
+
+
+def _app():
+    if not os.environ.get("STEWARD_DATABASE_URL"):
+        pytest.skip("needs Postgres")
+    from steward.app import create_app
+    return create_app(testing=False)
+
+
+def _ha(sub):
+    from steward.core.plugin_manager import get_loaded_plugin_modules
+    loaded = get_loaded_plugin_modules().get("host_agent")
+    base = loaded.__name__ if loaded is not None else "plugins.host_agent"
+    return importlib.import_module(f"{base}.{sub}")
+
+
+async def _seed(app):
+    from sqlalchemy import text
+    from steward.models.ansible_inventory import AnsibleGroup, AnsibleTarget
+    from steward.models.hosts import Host
+    from steward.models.users import User
+    tag = uuid.uuid4().hex[:8]
+    out = {"tokens": {}, "hosts": {}}
+    async with app.db_sessionmaker() as s:
+        async with s.begin():
+            user = User(username=f"cad-{tag}", email=f"cad-{tag}@test", password_hash="x")
+            group = AnsibleGroup(name=f"cad-{tag}")
+            s.add_all([user, group])
+            await s.flush()
+            for n in ("a", "b"):
+                host = Host(name=f"cad-{n}-{tag}", address="10.9.9.6")
+                s.add(host)
+                await s.flush()
+                target = AnsibleTarget(name=f"cad-{n}-{tag}", address="10.9.9.6", host_id=host.id)
+                target.groups = [group]
+                s.add(target)
+                token = f"tok-{n}-{tag}-{uuid.uuid4().hex}"
+                await s.execute(text(
+                    "INSERT INTO host_agent_registrations "
+                    "(id, host_id, token_hash, token_created_at, last_seen_at, maint_helper, "
+                    " created_at, updated_at) VALUES (:id, :h, :hash, now(), now(), true, now(), now())"),
+                    {"id": str(uuid.uuid4()), "h": host.id,
+                     "hash": hashlib.sha256(token.encode()).hexdigest()})
+                out["tokens"][n], out["hosts"][n] = token, host
+            plan_id = str(uuid.uuid4())
+            await s.execute(text(
+                "INSERT INTO host_maintenance_plans (id, name, group_id, weekday, at_time, time_zone, "
+                " window_minutes, reboot_policy, enabled, created_by, created_at) "
+                "VALUES (:id, :name, :g, 6, '03:00', 'UTC', 240, 'if_required', true, :u, now())"),
+                {"id": plan_id, "name": f"plan-{tag}", "g": group.id, "u": user.id})
+    out["plan_id"] = plan_id
+    return out
+
+
+async def _post(app, token, events=()):
+    resp = await app.test_client().post(
+        "/plugins/host_agent/ingest",
+        json={"samples": [{"ts": datetime.now(timezone.utc).isoformat(), "cpu_pct": 1.0}],
+              "events": list(events)},
+        headers={"Authorization": f"Bearer {token}"})
+    assert resp.status_code == 200, await resp.get_data(as_text=True)
+    return (await resp.get_json()).get("maintenance", [])
+
+
+async def _tick(app):
+    cadence = _ha("cadence")
+    async with app.db_sessionmaker() as s:
+        async with s.begin():
+            await cadence.tick(s, datetime.now(timezone.utc))
+            return await cadence.failed_hosts(s, datetime.now(timezone.utc))
+
+
+async def _run(app, plan_id):
+    from sqlalchemy import text
+    async with app.db_sessionmaker() as s:
+        return (await s.execute(text(
+            "SELECT state, position, detail FROM host_maintenance_runs WHERE plan_id = :p"),
+            {"p": plan_id})).one()
+
+
+def test_hosts_go_one_at_a_time_and_a_failure_stops_the_run():
+    app = _app()
+
+    async def _go():
+        seed = await _seed(app)
+        cadence, models = _ha("cadence"), _ha("models")
+        async with app.db_sessionmaker() as s:
+            async with s.begin():
+                plan = await s.get(models.HostMaintenancePlan, seed["plan_id"])
+                await cadence.start_run(s, plan, datetime.now(timezone.utc), manual=True)
+        await _tick(app)
+
+        # Only the first host (by name) has a job; the second waits.
+        ta, tb = seed["tokens"]["a"], seed["tokens"]["b"]
+        assert await _post(app, tb) == []
+        (step,) = await _post(app, ta)
+        assert step["action"] == "update"
+        done = {"kind": "maintenance", "id": step["id"], "action": "update", "state": "done",
+                "ok": True, "packages": 2, "reboot_required": False}
+        await _post(app, ta, [done])
+
+        # Back in service (posting, not a swarm node): the next host starts.
+        await _tick(app)
+        assert (await _run(app, seed["plan_id"]))[:2] == ("running", 1)
+        (step_b,) = await _post(app, tb)
+        failed = {"kind": "maintenance", "id": step_b["id"], "action": "update",
+                  "state": "failed", "ok": False, "detail": "dpkg was interrupted"}
+        await _post(app, tb, [failed])
+
+        flags = await _tick(app)
+        state, _, detail = await _run(app, seed["plan_id"])
+        assert state == "failed" and seed["hosts"]["b"].name in detail and "dpkg" in detail
+        assert flags[seed["hosts"]["b"].name] == 1.0 and flags[seed["hosts"]["a"].name] == 0.0
+
+    asyncio.run(_go())

tests/plugins/host_agent/test_cadence.py

@@ -0,0 +1,133 @@
+# tests/plugins/host_agent/test_cadence.py
+"""#5768: when a plan is due, when a host is back in service, and what a run
+does next. Model-free: plans, runs, jobs and swarm rows are SimpleNamespaces."""
+import json
+from datetime import datetime, timedelta, timezone
+from types import SimpleNamespace as NS
+
+import pytest
+
+from plugins.host_agent import cadence as c
+from plugins.host_agent import maintenance as m
+
+# Wednesday 2026-10-14 23:00 UTC
+NOW = datetime(2026, 10, 14, 23, 0, tzinfo=timezone.utc)
+
+
+def _plan(**kw):
+    base = dict(weekday=6, at_time="03:00", time_zone="UTC", window_minutes=240, enabled=True,
+                last_slot_at=None, created_at=NOW - timedelta(days=30), reboot_policy="if_required")
+    base.update(kw)
+    return NS(**base)
+
+
+def test_the_last_slot_is_the_latest_weekday_and_time_in_the_plans_zone():
+    assert c.last_slot(_plan(), NOW) == datetime(2026, 10, 11, 3, 0, tzinfo=timezone.utc)
+    # 03:00 in New York is 07:00 UTC in October (EDT)
+    ny = _plan(time_zone="America/New_York")
+    assert c.last_slot(ny, NOW) == datetime(2026, 10, 11, 7, 0, tzinfo=timezone.utc)
+    assert c.next_slot(_plan(), NOW) == datetime(2026, 10, 18, 3, 0, tzinfo=timezone.utc)
+    today = _plan(weekday=2, at_time="22:30")
+    assert c.last_slot(today, NOW) == datetime(2026, 10, 14, 22, 30, tzinfo=timezone.utc)
+
+
+def test_a_slot_fires_once_and_only_while_its_window_is_open():
+    plan = _plan(weekday=2, at_time="22:30")
+    assert c.is_due(plan, NOW) == datetime(2026, 10, 14, 22, 30, tzinfo=timezone.utc)
+    plan.last_slot_at = c.is_due(plan, NOW)
+    assert c.is_due(plan, NOW) is None                          # fired already
+    late = _plan(weekday=2, at_time="18:00")                    # 5 h ago, window 4 h
+    assert c.is_due(late, NOW) is None
+    assert c.is_due(_plan(weekday=2, at_time="22:30", enabled=False), NOW) is None
+    new = _plan(weekday=2, at_time="22:30", created_at=NOW - timedelta(minutes=5))
+    assert c.is_due(new, NOW) is None                           # made after the slot
+
+
+def _node(**kw):
+    base = dict(status="ready", availability="active", updated_at=NOW)
+    base.update(kw)
+    return NS(**base)
+
+
+def _svc(name, running, desired, mode="replicated"):
+    return NS(service_name=name, running=running, desired=desired, mode=mode, updated_at=NOW)
+
+
+def test_a_host_is_back_once_posting_its_node_active_and_its_services_at_replicas():
+    finished = NOW - timedelta(minutes=5)
+    kw = dict(job_finished_at=finished, now=NOW)
+    assert c.back_in_service(last_seen_at=None, node=None, services=[], **kw) == "its agent isn't posting yet"
+    assert c.back_in_service(last_seen_at=NOW, node=None, services=[], **kw) is None
+    assert "drain" in c.back_in_service(last_seen_at=NOW, node=_node(availability="drain"),
+                                        services=[_svc("web", 1, 1)], **kw)
+    assert c.back_in_service(last_seen_at=NOW, node=_node(updated_at=finished - timedelta(minutes=1)),
+                             services=[], **kw) == "no swarm report since it came back"
+    assert c.back_in_service(last_seen_at=NOW, node=_node(),
+                             services=[_svc("web", 1, 2), _svc("agent", 2, 3, mode="global")],
+                             **kw) == "services below their replicas: web"
+    assert c.back_in_service(last_seen_at=NOW, node=_node(), services=[_svc("web", 2, 2)], **kw) is None
+
+
+def _run(**kw):
+    base = dict(host_ids=json.dumps(["h1", "h2"]), position=0, started_at=NOW - timedelta(minutes=10),
+                settle_since=None, skipped="[]")
+    base.update(kw)
+    return NS(**base)
+
+
+def _job(state, detail=None):
+    return NS(state=state, detail=detail)
+
+
+@pytest.mark.parametrize("job,back,run_kw,expected", [
+    (None, None, {}, ("start", None)),
+    (_job("rebooting"), None, {}, ("wait", None)),
+    (_job("done"), None, {}, ("next", None)),
+    (_job("done"), "its agent isn't posting yet", {}, ("settling", "its agent isn't posting yet")),
+    (_job("failed", "update failed: dpkg"), None, {}, ("fail", "update failed: dpkg")),
+    (_job("timed_out"), None, {}, ("fail", "Timed out")),
+    (None, None, {"position": 2}, ("done", None)),
+])
+def test_what_a_run_does_next(job, back, run_kw, expected):
+    assert c.next_move(_run(**run_kw), _plan(), job, back, NOW) == expected
+
+
+def test_a_host_that_never_settles_fails_the_run_and_the_window_stops_new_hosts():
+    run = _run(settle_since=NOW - timedelta(minutes=31))
+    move, why = c.next_move(run, _plan(), _job("done"), "services below their replicas: web", NOW)
+    assert move == "fail" and why.startswith("not back in service within 30 min")
+    late = _run(started_at=NOW - timedelta(hours=5), position=1)
+    assert c.next_move(late, _plan(), None, None, NOW) == ("stop", "the window ended with 1 host not started")
+
+
+def test_a_plan_says_when_it_runs():
+    assert c.describe(_plan()) == "Sundays at 03:00 (UTC), up to 4 h"
+    assert c.describe(_plan(window_minutes=90)) == "Sundays at 03:00 (UTC), up to 90 min"
+
+
+def test_always_reboots_even_when_the_update_needs_none():
+    job = NS(id="j", state="updating", step_id="j-update", step_seen=False, reboot_policy="always",
+             reboot_needed=None, node_id=None, drain_host_id=None, packages=None, state_at=NOW,
+             finished_at=None, detail=None)
+    m.apply_result(job, {"id": "j-update", "state": "done", "ok": True, "reboot_required": False}, NOW)
+    assert job.state == "rebooting"
+
+
+async def test_the_page_leads_with_a_running_run_and_lists_the_plans(app):
+    from quart import session
+    from tests._templates import render_plugin_template
+    plan = NS(id="p1", name="Docker hosts", enabled=True, reboot_policy="if_required")
+    run = NS(id="r1", state="running", started_at=NOW, manual=True, detail=None, finished_at=None)
+    rows = [{"host_id": "h1", "name": "docker02", "text": "Rebooting", "bad": False, "busy": True, "detail": None},
+            {"host_id": "h2", "name": "docker03", "text": "Waiting", "bad": False, "busy": False, "detail": None}]
+    async with app.test_request_context("/plugins/host_agent/maintenance"):
+        session["user_role"] = "admin"
+        page = await render_plugin_template(
+            app, "host_agent", "maintenance.html",
+            plans=[{"plan": plan, "group": "docker", "hosts": 2, "when": "Sundays at 03:00 (UTC), up to 4 h",
+                    "next": NOW, "runs": [{"run": run, "rows": rows}]}],
+            groups=[NS(id="g1", name="docker")], live=[{"run": run, "plan": "Docker hosts", "rows": rows}],
+            failed=[], weekdays=c.WEEKDAYS, policies=m.REBOOT_POLICIES, default_zone="UTC")
+    assert page.index("Running: Docker hosts") < page.index("Add a plan")
+    assert page.index("docker02") < page.index("docker03")
+    assert "reboot only when required" in page and ">Run now</button>" in page