From 200384e52dc7ace6c1614f40665cd8bfdac7d90c Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Wed, 14 Oct 2026 18:59:07 -0400 Subject: [PATCH] feat(host_agent): a group's updates and reboots on a weekly cadence, one host at a time (#5768) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit #5768 "A maintenance cadence: a group's updates and reboots, one host at a time, on a schedule". A plan names an inventory group, a weekday and time in a time zone, a window and a reboot policy (only when required by default). At its slot a run takes the group's hosts in order and gives each a #5767 maintenance job. The next host starts only when the last is back in service: its agent is posting and, for a swarm node, the node is active and every replicated service is at its replicas. A failure stops the run, and no host is started once the window has passed. A host's latest failure shows as host_agent//maintenance_failed, for an ordinary alert rule. The page is Hosts → Maintenance. Co-Authored-By: Claude Opus 5.5 --- docs/plugins/host-agent-design.md | 31 ++ plugins/host_agent/__init__.py | 12 +- plugins/host_agent/cadence.py | 302 ++++++++++++++++++ plugins/host_agent/maintenance.py | 25 +- .../versions/host_agent_008_maint_plans.py | 61 ++++ plugins/host_agent/models.py | 49 +++ plugins/host_agent/routes.py | 169 ++++++++++ plugins/host_agent/scheduler.py | 19 +- plugins/host_agent/templates/maintenance.html | 143 +++++++++ plugins/host_agent/templates/panel.html | 2 +- steward/audit/service.py | 4 + steward/backup/contents.py | 4 +- steward/core/formatting.py | 1 + steward/templates/hosts/list.html | 1 + tests/core/test_host_page.py | 2 +- .../test_host_maintenance_cadence.py | 134 ++++++++ tests/plugins/host_agent/test_cadence.py | 133 ++++++++ 17 files changed, 1077 insertions(+), 15 deletions(-) create mode 100644 plugins/host_agent/cadence.py create mode 100644 plugins/host_agent/migrations/versions/host_agent_008_maint_plans.py create mode 100644 plugins/host_agent/templates/maintenance.html create mode 100644 tests/integration/test_host_maintenance_cadence.py create mode 100644 tests/plugins/host_agent/test_cadence.py diff --git a/docs/plugins/host-agent-design.md b/docs/plugins/host-agent-design.md index d23e50f..d6f9688 100644 --- a/docs/plugins/host-agent-design.md +++ b/docs/plugins/host-agent-design.md @@ -221,6 +221,37 @@ since-when; a finished one says what it did ("Done: 12 packages updated, rebooted"). Under Manage agent, **Update now** (with "never reboot") starts a job, or the panel says why one can't start. +### A maintenance cadence: a group, one host at a time (#5768) + +`plugins/host_agent/cadence.py`, driven by the same minute tick as the jobs. +A **plan** (`host_maintenance_plans`) names an inventory group, a weekday, a +time in a time zone (the creator's, by default), a window and a reboot policy: +`if_required` (the default), `always` or `never`. The page is **Hosts → +Maintenance** (`/plugins/host_agent/maintenance`). + +- **When.** A plan's slot is the latest weekday-and-time at or before now, in + its zone. A slot fires once (`last_slot_at`) and only while its window is + open, so a Steward that was down over the slot starts late, not a day late. + A new or resumed plan starts from the next slot. **Run now** starts a run + without touching the schedule (the Ansible ruling on Run now). +- **A run** (`host_maintenance_runs`) fixes its hosts at the start, by name: + the group's hosts through their linked Ansible targets. A host with no agent, + or no maintenance helper, is skipped and listed. +- **One at a time.** Each host gets a #5767 job (`run_id` on the job). The next + host starts only when the last is **back in service**: its agent posting in + the last 3 min and, for a swarm node, the node `ready`/`active` and every + replicated service at its replicas, judged only from swarm reports made + after the job finished. A host not back within 30 min fails the run, and + its job is marked failed with the reason. +- **Stops.** A failed or timed-out job fails the run. No host is started after + the window; the run then stops and says how many weren't reached. +- **Alerting** is an ordinary alert rule on `host_agent//maintenance_failed`: + 1 when the host's latest job (in 30 days) failed, timed out or didn't come + back into service, 0 once a later one succeeds. +- The page leads with a running run and its hosts, then failed runs from the + last week, then each plan with its schedule, next slot and last five runs. + Each host's Agent panel shows its last maintenance job. + ### What happened to the host (1.14.0, #5756) A post may carry `events`, said once each, with the next post that gets through: diff --git a/plugins/host_agent/__init__.py b/plugins/host_agent/__init__.py index 0b7ca2e..5d83b0b 100644 --- a/plugins/host_agent/__init__.py +++ b/plugins/host_agent/__init__.py @@ -36,6 +36,12 @@ def setup(app: "Quart") -> None: prune_by_age(HostDowntime.started_at) +def get_audit_entities() -> dict: + """Audit rows about a maintenance plan name it and link to the plans page.""" + from .models import HostMaintenancePlan + return {"maintenance_plan": (HostMaintenancePlan, "/plugins/host_agent/maintenance")} + + def get_scheduled_tasks() -> list: from .scheduler import make_maintenance_task, make_task return [make_task(_app), make_maintenance_task(_app)] @@ -70,12 +76,14 @@ def get_alert_metrics() -> dict[str, list[str]]: healthy agent sits in tens of MB and a few %. updates_pending and updates_security are what the package manager has waiting, and reboot_required is 1 when the host needs a reboot (agent ≥ 1.16.0): - "security updates waiting" is updates_security > 0.""" + "security updates waiting" is updates_security > 0. maintenance_failed + comes from the scheduler (#5768): 1 when the host's latest maintenance job + failed or timed out, 0 once one succeeds: "maintenance failed" is > 0.""" return {"host_agent": [ "cpu_pct", "mem_used_pct", "mem_available_bytes", "swap_used_bytes", "disk_used_pct_worst", "disk_used_pct", "inodes_used_pct", "load_1m", "load_5m", "load_15m", "load_pct", "uptime_secs", "seconds_since_seen", "clock_off_secs", "mounts_stalled", "agent_rss_bytes", "agent_cpu_pct", - "updates_pending", "updates_security", "reboot_required", + "updates_pending", "updates_security", "reboot_required", "maintenance_failed", ]} diff --git a/plugins/host_agent/cadence.py b/plugins/host_agent/cadence.py new file mode 100644 index 0000000..113b3e7 --- /dev/null +++ b/plugins/host_agent/cadence.py @@ -0,0 +1,302 @@ +# plugins/host_agent/cadence.py +"""A group's maintenance cadence (#5768): every week, its hosts updated one at +a time, each rebooting only when its update needs it (or as the plan says). + +A plan names an inventory group, a weekday and a time in its time zone, and a +window. At that time a run starts with the group's hosts in a fixed order. Each +host gets a maintenance job (maintenance.py); the next host starts only when +the last one is back in service: its agent posting, its swarm node ready and +active, every replicated service at its replicas. A job that fails, or a host +that doesn't come back into service, stops the run, and the host's +maintenance_failed figure goes to 1 for an alert rule to catch. No host is +started after the window; the run then stops and says which hosts it didn't +reach. + +The decisions are model-free here (plans and runs are anything with the +attributes); the database side is below them, and the plugin's minute tick +drives it. Everything is on the rows, so a restart of Steward, or the reboot of +the host it runs on, picks up where it was. +""" +from __future__ import annotations + +import json +from datetime import datetime, time, timedelta, timezone + +from . import maintenance + +WEEKDAYS = ("Monday", "Tuesday", "Wednesday", "Thursday", "Friday", "Saturday", "Sunday") +# How long a host may take to be back in service after its job finished. +SETTLE_LIMIT = timedelta(minutes=30) +# An agent heard from this recently is posting. +POSTING_WITHIN = timedelta(minutes=3) +RUN_STATES = ("running", "done", "failed", "stopped") + + +def _zone(name: str): + from steward.core.formatting import zone + return zone(name) + + +def parse_at(at_time: str) -> time: + h, m = (int(x) for x in at_time.split(":", 1)) + if not (0 <= h < 24 and 0 <= m < 60): + raise ValueError(at_time) + return time(h, m) + + +def last_slot(plan, now: datetime) -> datetime: + """The most recent scheduled time at or before `now`, in UTC.""" + tz = _zone(plan.time_zone) + local = now.astimezone(tz) + day = local.date() - timedelta(days=(local.weekday() - plan.weekday) % 7) + slot = datetime.combine(day, parse_at(plan.at_time), tzinfo=tz) + if slot > local: + slot -= timedelta(days=7) + return slot.astimezone(timezone.utc) + + +def next_slot(plan, now: datetime) -> datetime: + return last_slot(plan, now) + timedelta(days=7) + + +def is_due(plan, now: datetime) -> datetime | None: + """The slot to fire now, or None. A slot fires once (last_slot_at), and + only while its window is still open: a Steward that was down over the slot + starts late, not a day late.""" + if not plan.enabled: + return None + slot = last_slot(plan, now) + since = plan.last_slot_at or plan.created_at + if since is not None and slot <= since: + return None + if now - slot >= timedelta(minutes=plan.window_minutes): + return None + return slot + + +def back_in_service(*, last_seen_at, node, services, job_finished_at, now: datetime) -> str | None: + """None when the host is back in service, else why not yet. `node` is the + freshest swarm-node row for the job's node (None outside a swarm, or when + the job has no node); `services` the freshest row per swarm service. Only + rows reported after the job finished count, so a stale "all fine" from + before the reboot never passes.""" + if last_seen_at is None or now - last_seen_at > POSTING_WITHIN: + return "its agent isn't posting yet" + if node is None: + return None + if node.updated_at < job_finished_at: + return "no swarm report since it came back" + if node.status != "ready" or node.availability != "active": + return f"its swarm node is {node.status}/{node.availability}" + fresh = [s for s in services if s.updated_at >= job_finished_at] + if not fresh: + return "no swarm report since it came back" + short = sorted(s.service_name for s in fresh + if s.mode == "replicated" and s.running < s.desired) + if short: + return "services below their replicas: " + ", ".join(short[:5]) + return None + + +def next_move(run, plan, job, back: str | None, now: datetime) -> tuple[str, str | None]: + """What a running run does now: ("start", None) the current host's job; + ("wait", None); ("next", None) on to the next host; ("settling", why); + ("fail", why); ("stop", why) the window ended; ("done", None).""" + hosts = json.loads(run.host_ids) + if run.position >= len(hosts): + return "done", None + if job is None: + if now >= run.started_at + timedelta(minutes=plan.window_minutes): + left = len(hosts) - run.position + return "stop", f"the window ended with {left} host{'s' if left != 1 else ''} not started" + return "start", None + if job.state in maintenance.ACTIVE: + return "wait", None + if job.state != "done": + return "fail", job.detail or maintenance.LABELS[job.state] + if back is None: + return "next", None + if run.settle_since is not None and now - run.settle_since > SETTLE_LIMIT: + return "fail", f"not back in service within {int(SETTLE_LIMIT.total_seconds() // 60)} min: {back}" + return "settling", back + + +def describe(plan) -> str: + """"Sundays at 03:00 (Europe/London), up to 4 h" for the page.""" + hours = plan.window_minutes / 60 + span = f"{hours:g} h" if plan.window_minutes % 60 == 0 else f"{plan.window_minutes} min" + return f"{WEEKDAYS[plan.weekday]}s at {plan.at_time} ({plan.time_zone}), up to {span}" + + +# ── applied to the database: the minute tick, Run now, the page ────────────── + +async def group_hosts(session, group_id: str) -> list: + """The group's hosts (through their linked Ansible targets), by name.""" + from sqlalchemy import select + from steward.models.ansible_inventory import AnsibleTarget, ansible_target_groups + from steward.models.hosts import Host + rows = (await session.execute( + select(Host).join(AnsibleTarget, AnsibleTarget.host_id == Host.id) + .join(ansible_target_groups, ansible_target_groups.c.target_id == AnsibleTarget.id) + .where(ansible_target_groups.c.group_id == group_id))).scalars().all() + return sorted({h.id: h for h in rows}.values(), key=lambda h: h.name.lower()) + + +async def start_run(session, plan, now: datetime, manual: bool = False): + """A run of the plan with the group's hosts in order. A host whose agent + has no maintenance helper is passed over, and the run says so.""" + from sqlalchemy import select + from .models import HostAgentRegistration, HostMaintenanceRun + hosts = await group_hosts(session, plan.group_id) + regs = {r.host_id: r for r in (await session.execute(select(HostAgentRegistration).where( + HostAgentRegistration.host_id.in_([h.id for h in hosts])))).scalars()} if hosts else {} + order, skipped = [], [] + for h in hosts: + reg = regs.get(h.id) + if reg is None: + skipped.append({"host_id": h.id, "reason": "no agent"}) + elif not reg.maint_helper: + skipped.append({"host_id": h.id, "reason": "no maintenance helper: run Update agent"}) + else: + order.append(h.id) + run = HostMaintenanceRun(plan_id=plan.id, state="running", host_ids=json.dumps(order), + position=0, skipped=json.dumps(skipped), manual=manual, + started_at=now) + session.add(run) + await session.flush() + return run + + +async def _back(session, host, job, reg, now: datetime) -> str | None: + from sqlalchemy import select + node = None + services: list = [] + if job.node_id: + try: + from plugins.docker.models import DockerSwarmNode, DockerSwarmService + except ImportError: + DockerSwarmNode = DockerSwarmService = None + if DockerSwarmNode is not None: + node = (await session.execute(select(DockerSwarmNode).where( + DockerSwarmNode.node_id == job.node_id) + .order_by(DockerSwarmNode.updated_at.desc()).limit(1))).scalar_one_or_none() + freshest: dict = {} + for s in (await session.execute(select(DockerSwarmService))).scalars(): + if s.service_name not in freshest or s.updated_at > freshest[s.service_name].updated_at: + freshest[s.service_name] = s + services = list(freshest.values()) + return back_in_service(last_seen_at=reg.last_seen_at if reg else None, node=node, + services=services, job_finished_at=job.finished_at or now, now=now) + + +async def advance(session, run, plan, now: datetime) -> str: + """Move one running run on as far as it can go now; returns its state.""" + from sqlalchemy import select + from steward.models.hosts import Host + from .models import HostAgentRegistration, HostMaintenanceJob + for _ in range(64): # a host per pass; bounded + hosts = json.loads(run.host_ids) + host_id = hosts[run.position] if run.position < len(hosts) else None + job = host = reg = None + if host_id is not None: + host = await session.get(Host, host_id) + reg = (await session.execute(select(HostAgentRegistration).where( + HostAgentRegistration.host_id == host_id))).scalar_one_or_none() + job = (await session.execute(select(HostMaintenanceJob).where( + HostMaintenanceJob.run_id == run.id, HostMaintenanceJob.host_id == host_id) + .order_by(HostMaintenanceJob.created_at.desc()).limit(1))).scalar_one_or_none() + back = None + if job is not None and job.state == "done": + back = await _back(session, host, job, reg, now) + move, why = next_move(run, plan, job, back, now) + if move == "start": + if host is None: + _skip(run, host_id, "the host is gone") + continue + try: + await maintenance.create_job(session, host, reg, plan.created_by, + plan.reboot_policy, now, run_id=run.id) + except maintenance.CannotStart as exc: + _skip(run, host_id, str(exc)) + continue + return run.state + if move == "next": + run.position += 1 + run.settle_since = None + continue + if move == "settling": + run.settle_since = run.settle_since or now + return run.state + if move == "fail": + if job is not None and job.state == "done": + maintenance._finish(job, "failed", now, why) # the host never came back into service + name = host.name if host else host_id + _end(run, "failed", now, f"{name}: {why}") + elif move == "stop": + _end(run, "stopped", now, why) + elif move == "done": + _end(run, "done", now, None) + return run.state + return run.state + + +def _skip(run, host_id: str, reason: str) -> None: + skipped = json.loads(run.skipped) + skipped.append({"host_id": host_id, "reason": reason}) + run.skipped = json.dumps(skipped) + hosts = json.loads(run.host_ids) + hosts.pop(run.position) + run.host_ids = json.dumps(hosts) + + +def _end(run, state: str, now: datetime, detail: str | None) -> None: + run.state, run.finished_at, run.settle_since = state, now, None + if detail: + run.detail = detail[:maintenance.DETAIL_MAX] + + +async def tick(session, now: datetime) -> list[str]: + """Start due plans and move running runs on; returns lines worth logging.""" + from sqlalchemy import select + from .models import HostMaintenancePlan, HostMaintenanceRun + said = [] + running = {r.plan_id: r for r in (await session.execute(select(HostMaintenanceRun).where( + HostMaintenanceRun.state == "running"))).scalars()} + for plan in (await session.execute(select(HostMaintenancePlan))).scalars(): + slot = is_due(plan, now) + if slot is None: + continue + plan.last_slot_at = slot + if plan.id in running: + said.append(f"plan {plan.name!r}: the {slot:%Y-%m-%d %H:%M}Z slot came with a run still going") + continue + running[plan.id] = await start_run(session, plan, now) + said.append(f"plan {plan.name!r}: run started") + for run in running.values(): + plan = await session.get(HostMaintenancePlan, run.plan_id) + if plan is None: + continue + before = run.state + after = await advance(session, run, plan, now) + if after != before: + said.append(f"plan {plan.name!r}: run {after}" + (f" ({run.detail})" if run.detail else "")) + return said + + +async def failed_hosts(session, now: datetime) -> dict[str, float]: + """{host name: 1 or 0}: whether each host's latest finished maintenance job + failed or timed out, for the maintenance_failed figure an alert rule + watches. A later job that succeeds puts it back to 0.""" + from sqlalchemy import select + from steward.models.hosts import Host + from .models import HostMaintenanceJob + rows = (await session.execute( + select(Host.name, HostMaintenanceJob.state, HostMaintenanceJob.finished_at) + .join(Host, Host.id == HostMaintenanceJob.host_id) + .where(HostMaintenanceJob.state.in_(maintenance.FINISHED), + HostMaintenanceJob.finished_at >= now - timedelta(days=30)))).all() + latest: dict = {} + for name, state, finished in rows: + if name not in latest or finished > latest[name][1]: + latest[name] = (state, finished) + return {name: 0.0 if state == "done" else 1.0 for name, (state, _) in latest.items()} diff --git a/plugins/host_agent/maintenance.py b/plugins/host_agent/maintenance.py index 0cd612d..2d8efff 100644 --- a/plugins/host_agent/maintenance.py +++ b/plugins/host_agent/maintenance.py @@ -86,8 +86,19 @@ def handoff(job) -> dict | None: return out +REBOOT_POLICIES = ("if_required", "always", "never") + + +def wants_reboot(job) -> bool: + """Reboot only when the update says one is needed (the operator's + default), always (a plan may ask), or never.""" + if job.reboot_policy == "always": + return True + return bool(job.reboot_needed) and job.reboot_policy == "if_required" + + def _after_update(job, now: datetime) -> None: - if not (job.reboot_needed and job.reboot_policy != "never"): + if not wants_reboot(job): _enter(job, "done", now) elif job.node_id and job.drain_host_id: _enter(job, "draining", now) @@ -296,7 +307,7 @@ async def plan_job(session, host, reg, reboot_policy: str = "if_required") -> tu from .models import HostAgentRegistration, HostMaintenanceJob if reg is None or not reg.maint_helper: raise CannotStart("its agent has no maintenance helper: run Update agent first") - if reboot_policy not in ("if_required", "never"): + if reboot_policy not in REBOOT_POLICIES: raise CannotStart(f"unknown reboot policy {reboot_policy!r}") busy = (await session.execute(select(HostMaintenanceJob.id).where( HostMaintenanceJob.host_id == host.id, @@ -312,21 +323,23 @@ async def plan_job(session, host, reg, reboot_policy: str = "if_required") -> tu async def create_job(session, host, reg, user_id: str | None, - reboot_policy: str = "if_required", now: datetime | None = None): + reboot_policy: str = "if_required", now: datetime | None = None, + run_id: str | None = None): """Queue a maintenance job for `host`; it starts with the agent's next post.""" from .models import HostMaintenanceJob now = now or utcnow() node_id, drain_host_id = await plan_job(session, host, reg, reboot_policy) job = HostMaintenanceJob(host_id=host.id, state="queued", state_at=now, reboot_policy=reboot_policy, node_id=node_id, - drain_host_id=drain_host_id, created_by=user_id, created_at=now) + drain_host_id=drain_host_id, created_by=user_id, created_at=now, + run_id=run_id) session.add(job) await session.flush() return job -async def latest_job(session, host_id: str, now: datetime, days: int = 7): - """The host's running job, else its last one in `days`, for the panel.""" +async def latest_job(session, host_id: str, now: datetime, days: int = 366): + """The host's running job, else its last one (within a year), for the panel.""" from sqlalchemy import select from .models import HostMaintenanceJob return (await session.execute(select(HostMaintenanceJob).where( diff --git a/plugins/host_agent/migrations/versions/host_agent_008_maint_plans.py b/plugins/host_agent/migrations/versions/host_agent_008_maint_plans.py new file mode 100644 index 0000000..ab81e46 --- /dev/null +++ b/plugins/host_agent/migrations/versions/host_agent_008_maint_plans.py @@ -0,0 +1,61 @@ +"""host_agent: a group's maintenance cadence and its runs (#5768) + +Revision ID: host_agent_008_maint_plans +""" +from typing import Sequence, Union +from alembic import op +import sqlalchemy as sa + +revision: str = "host_agent_008_maint_plans" +down_revision: Union[str, None] = "host_agent_007_maint_jobs" +branch_labels: Union[str, Sequence[str], None] = None +# ansible_groups (core 0016) must exist before the group_id key. +depends_on: Union[str, Sequence[str], None] = ("0016_ansible_inventory_groups",) + + +def upgrade() -> None: + op.create_table( + "host_maintenance_plans", + sa.Column("id", sa.String(36), primary_key=True), + sa.Column("name", sa.String(128), nullable=False), + sa.Column("group_id", sa.String(36), sa.ForeignKey("ansible_groups.id", ondelete="CASCADE"), + nullable=False), + sa.Column("weekday", sa.Integer(), nullable=False), + sa.Column("at_time", sa.String(5), nullable=False), + sa.Column("time_zone", sa.String(64), nullable=False, server_default="UTC"), + sa.Column("window_minutes", sa.Integer(), nullable=False, server_default="240"), + sa.Column("reboot_policy", sa.String(16), nullable=False, server_default="if_required"), + sa.Column("enabled", sa.Boolean(), nullable=False, server_default=sa.true()), + sa.Column("last_slot_at", sa.DateTime(timezone=True), nullable=True), + sa.Column("created_by", sa.String(36), sa.ForeignKey("users.id", ondelete="SET NULL"), + nullable=True), + sa.Column("created_at", sa.DateTime(timezone=True), nullable=False), + ) + op.create_table( + "host_maintenance_runs", + sa.Column("id", sa.String(36), primary_key=True), + sa.Column("plan_id", sa.String(36), + sa.ForeignKey("host_maintenance_plans.id", ondelete="CASCADE"), nullable=False), + sa.Column("state", sa.String(16), nullable=False), + sa.Column("host_ids", sa.Text(), nullable=False, server_default="[]"), + sa.Column("position", sa.Integer(), nullable=False, server_default="0"), + sa.Column("skipped", sa.Text(), nullable=False, server_default="[]"), + sa.Column("settle_since", sa.DateTime(timezone=True), nullable=True), + sa.Column("detail", sa.Text(), nullable=True), + sa.Column("manual", sa.Boolean(), nullable=False, server_default=sa.false()), + sa.Column("started_at", sa.DateTime(timezone=True), nullable=False), + sa.Column("finished_at", sa.DateTime(timezone=True), nullable=True), + ) + op.create_index("ix_host_maintenance_runs_plan_id", "host_maintenance_runs", ["plan_id"]) + op.create_index("ix_host_maintenance_runs_state", "host_maintenance_runs", ["state"]) + op.add_column("host_maintenance_jobs", sa.Column( + "run_id", sa.String(36), sa.ForeignKey("host_maintenance_runs.id", ondelete="SET NULL"), + nullable=True)) + op.create_index("ix_host_maintenance_jobs_run_id", "host_maintenance_jobs", ["run_id"]) + + +def downgrade() -> None: + op.drop_index("ix_host_maintenance_jobs_run_id", "host_maintenance_jobs") + op.drop_column("host_maintenance_jobs", "run_id") + op.drop_table("host_maintenance_runs") + op.drop_table("host_maintenance_plans") diff --git a/plugins/host_agent/models.py b/plugins/host_agent/models.py index ef7e13b..f7cd6b8 100644 --- a/plugins/host_agent/models.py +++ b/plugins/host_agent/models.py @@ -106,3 +106,52 @@ class HostMaintenanceJob(Base): created_by = Column(String(36), ForeignKey("users.id", ondelete="SET NULL"), nullable=True) created_at = Column(DateTime(timezone=True), nullable=False, default=_utcnow) finished_at = Column(DateTime(timezone=True), nullable=True) + # The cadence run this job belongs to (#5768); None for Update now. + run_id = Column(String(36), ForeignKey("host_maintenance_runs.id", ondelete="SET NULL"), + nullable=True, index=True) + + +class HostMaintenancePlan(Base): + """A group's maintenance cadence (#5768): every week on `weekday` at + `at_time` in `time_zone`, its hosts are updated one at a time, each + rebooting per `reboot_policy`. No host is started after `window_minutes` + from the run's start. last_slot_at is the last scheduled slot that fired; + Run now leaves it alone, so the schedule keeps its time.""" + __tablename__ = "host_maintenance_plans" + + id = Column(String(36), primary_key=True, default=_uuid) + name = Column(String(128), nullable=False) + group_id = Column(String(36), ForeignKey("ansible_groups.id", ondelete="CASCADE"), + nullable=False) + weekday = Column(Integer, nullable=False) # 0 = Monday + at_time = Column(String(5), nullable=False) # "HH:MM" + time_zone = Column(String(64), nullable=False, default="UTC") + window_minutes = Column(Integer, nullable=False, default=240) + reboot_policy = Column(String(16), nullable=False, default="if_required") + enabled = Column(Boolean, nullable=False, default=True) + last_slot_at = Column(DateTime(timezone=True), nullable=True) + created_by = Column(String(36), ForeignKey("users.id", ondelete="SET NULL"), nullable=True) + created_at = Column(DateTime(timezone=True), nullable=False, default=_utcnow) + + +class HostMaintenanceRun(Base): + """One run of a plan: its hosts in order, one at a time (#5768). + + host_ids is the order fixed at the start (JSON list); position is the host + being worked on; skipped is [{"host_id", "reason"}] for hosts the run + passed over. settle_since is when the current host's job finished and the + run began waiting for the host to be back in service.""" + __tablename__ = "host_maintenance_runs" + + id = Column(String(36), primary_key=True, default=_uuid) + plan_id = Column(String(36), ForeignKey("host_maintenance_plans.id", ondelete="CASCADE"), + nullable=False, index=True) + state = Column(String(16), nullable=False, default="running", index=True) + host_ids = Column(Text, nullable=False, default="[]") + position = Column(Integer, nullable=False, default=0) + skipped = Column(Text, nullable=False, default="[]") + settle_since = Column(DateTime(timezone=True), nullable=True) + detail = Column(Text, nullable=True) + manual = Column(Boolean, nullable=False, default=False) + started_at = Column(DateTime(timezone=True), nullable=False, default=_utcnow) + finished_at = Column(DateTime(timezone=True), nullable=True) diff --git a/plugins/host_agent/routes.py b/plugins/host_agent/routes.py index c9eae93..9e15da3 100644 --- a/plugins/host_agent/routes.py +++ b/plugins/host_agent/routes.py @@ -1343,3 +1343,172 @@ async def update_via_ansible(): if err: return _error(400, "update_failed", err) return redirect(url_for("ansible.run_detail", run_id=run.id)) + + +# ── Maintenance cadence (#5768) ─────────────────────────────────────────────── + +def _run_rows(run, jobs: dict, names: dict) -> list[dict]: + """A run's hosts in order, each with where it stands, then those passed over.""" + import json as _json + rows = [] + for i, host_id in enumerate(_json.loads(run.host_ids)): + job = jobs.get((run.id, host_id)) + if job is not None: + text = maintenance.summary(job) + bad = job.state in ("failed", "timed_out") + busy = job.state in maintenance.ACTIVE + elif run.state == "running" and i == run.position: + text, bad, busy = "Starting", False, True + else: + text, bad, busy = ("Not reached" if run.state != "running" else "Waiting"), False, False + if run.state == "running" and i == run.position and job is not None and job.state == "done": + text, busy = "Coming back into service", True + rows.append({"host_id": host_id, "name": names.get(host_id, host_id), "text": text, + "bad": bad, "busy": busy, "detail": job.detail if job is not None else None}) + for s in _json.loads(run.skipped): + rows.append({"host_id": s["host_id"], "name": names.get(s["host_id"], s["host_id"]), + "text": f"Skipped: {s['reason']}", "bad": False, "busy": False, "detail": None}) + return rows + + +@host_agent_bp.get("/maintenance") +@require_role(UserRole.viewer) +async def maintenance_page(): + """Maintenance plans: a group's updates and reboots on a weekly cadence, + one host at a time. A running run first, then a failed one, then the plans + and their recent runs.""" + from steward.models.ansible_inventory import AnsibleGroup + from steward.models.users import User + from . import cadence + from .models import HostMaintenanceJob, HostMaintenancePlan, HostMaintenanceRun + now = datetime.now(timezone.utc) + async with current_app.db_sessionmaker() as db: + plans = (await db.execute(select(HostMaintenancePlan) + .order_by(HostMaintenancePlan.name))).scalars().all() + groups = (await db.execute(select(AnsibleGroup).order_by(AnsibleGroup.name))).scalars().all() + runs = (await db.execute(select(HostMaintenanceRun) + .order_by(HostMaintenanceRun.started_at.desc()).limit(60))).scalars().all() + jobs = {} + if runs: + for j in (await db.execute(select(HostMaintenanceJob).where( + HostMaintenanceJob.run_id.in_([r.id for r in runs])) + .order_by(HostMaintenanceJob.created_at))).scalars(): + jobs[(j.run_id, j.host_id)] = j + names = dict((await db.execute(select(Host.id, Host.name))).all()) + me = await db.get(User, session.get("user_id")) if session.get("user_id") else None + sizes = {g.id: len(await cadence.group_hosts(db, g.id)) for g in groups} + group_name = {g.id: g.name for g in groups} + by_plan: dict = {} + for r in runs: + by_plan.setdefault(r.plan_id, []).append(r) + view = [] + for p in plans: + prs = by_plan.get(p.id, [])[:5] + view.append({"plan": p, "group": group_name.get(p.group_id, "—"), + "hosts": sizes.get(p.group_id, 0), "when": cadence.describe(p), + "next": cadence.next_slot(p, now) if p.enabled else None, + "runs": [{"run": r, "rows": _run_rows(r, jobs, names)} for r in prs]}) + # What needs attention first: a run going now, then a run that failed lately. + plan_name = {p.id: p.name for p in plans} + live = [{"run": r, "plan": plan_name.get(r.plan_id, "—"), "rows": _run_rows(r, jobs, names)} + for r in runs if r.state == "running"] + failed = [{"run": r, "plan": plan_name.get(r.plan_id, "—")} for r in runs + if r.state == "failed" and r.finished_at and now - r.finished_at < timedelta(days=7)] + return await render_template( + "maintenance.html", plans=view, groups=groups, live=live, failed=failed, + weekdays=cadence.WEEKDAYS, policies=maintenance.REBOOT_POLICIES, + default_zone=(me.time_zone if me and me.time_zone else "UTC")) + + +@host_agent_bp.post("/maintenance/plans") +@require_role(UserRole.admin) +async def maintenance_plan_create(): + from steward.core.formatting import zone + from steward.models.ansible_inventory import AnsibleGroup + from . import cadence + from .models import HostMaintenancePlan + form = await request.form + name = (form.get("name") or "").strip()[:128] + try: + weekday = int(form.get("weekday", "")) + at_time = (form.get("at_time") or "").strip() + cadence.parse_at(at_time) + window = int(form.get("window_minutes") or 240) + except ValueError: + return _error(400, "invalid", "weekday, time (HH:MM) and window must be numbers") + policy = form.get("reboot_policy") or "if_required" + tz = (form.get("time_zone") or "UTC").strip() + if not name or not 0 <= weekday <= 6 or not 30 <= window <= 24 * 60 \ + or policy not in maintenance.REBOOT_POLICIES: + return _error(400, "invalid", "name, a weekday, a window of 30 min to 24 h and a reboot policy are needed") + if zone(tz) is timezone.utc and tz.upper() != "UTC": + return _error(400, "invalid", f"unknown time zone {tz!r}") + async with current_app.db_sessionmaker() as db: + if await db.get(AnsibleGroup, form.get("group_id") or "") is None: + return _error(400, "invalid", "pick an inventory group") + plan = HostMaintenancePlan(name=name, group_id=form["group_id"], weekday=weekday, + at_time=f"{cadence.parse_at(at_time):%H:%M}", time_zone=tz, + window_minutes=window, reboot_policy=policy, enabled=True, + created_by=session.get("user_id"), + # the first slot is the next one, not one already past + last_slot_at=datetime.now(timezone.utc)) + db.add(plan) + await db.commit() + await audit_request("host_agent.maintenance_plan_created", "maintenance_plan", plan.id, + {"name": name}) + return redirect(url_for("host_agent.maintenance_page")) + + +async def _plan_action(plan_id: str, act): + from .models import HostMaintenancePlan + async with current_app.db_sessionmaker() as db: + plan = await db.get(HostMaintenancePlan, plan_id) + if plan is None: + return _error(404, "not_found") + out = await act(db, plan) + if out is not None: + return out + await db.commit() + return redirect(url_for("host_agent.maintenance_page")) + + +@host_agent_bp.post("/maintenance/plans//toggle") +@require_role(UserRole.admin) +async def maintenance_plan_toggle(plan_id: str): + async def act(db, plan): + plan.enabled = not plan.enabled + if plan.enabled: + plan.last_slot_at = datetime.now(timezone.utc) # no catching up on a missed slot + resp = await _plan_action(plan_id, act) + await audit_request("host_agent.maintenance_plan_toggled", "maintenance_plan", plan_id) + return resp + + +@host_agent_bp.post("/maintenance/plans//delete") +@require_role(UserRole.admin) +async def maintenance_plan_delete(plan_id: str): + async def act(db, plan): + await db.delete(plan) + resp = await _plan_action(plan_id, act) + await audit_request("host_agent.maintenance_plan_deleted", "maintenance_plan", plan_id) + return resp + + +@host_agent_bp.post("/maintenance/plans//run") +@require_role(UserRole.admin) +async def maintenance_plan_run_now(plan_id: str): + """Run the plan now. Its schedule keeps its time (Ansible ruling: Run now + leaves the schedule as it was).""" + from . import cadence + from .models import HostMaintenanceRun + + async def act(db, plan): + busy = (await db.execute(select(HostMaintenanceRun.id).where( + HostMaintenanceRun.plan_id == plan.id, + HostMaintenanceRun.state == "running"))).first() + if busy is not None: + return _error(409, "busy", "this plan already has a run going") + await cadence.start_run(db, plan, datetime.now(timezone.utc), manual=True) + resp = await _plan_action(plan_id, act) + await audit_request("host_agent.maintenance_plan_run", "maintenance_plan", plan_id) + return resp diff --git a/plugins/host_agent/scheduler.py b/plugins/host_agent/scheduler.py index 40775d6..1f9b8e7 100644 --- a/plugins/host_agent/scheduler.py +++ b/plugins/host_agent/scheduler.py @@ -122,16 +122,27 @@ def make_task(app) -> ScheduledTask: def make_maintenance_task(app) -> ScheduledTask: - """Gives up on maintenance jobs past their deadline (#5767). Every other - move a job makes comes from its agent's posts.""" + """Gives up on maintenance jobs past their deadline (#5767), starts due + maintenance plans and moves their runs on one host at a time (#5768), and + records each host's maintenance_failed for an alert rule. Every move a job + itself makes comes from its agent's posts.""" async def _tick(): - from . import maintenance + from . import cadence, maintenance + now = datetime.now(timezone.utc) try: async with app.db_sessionmaker() as session: async with session.begin(): - gave_up = await maintenance.tick(session, datetime.now(timezone.utc)) + gave_up = await maintenance.tick(session, now) + said = await cadence.tick(session, now) + batch = MetricBatch() + for name, value in (await cadence.failed_hosts(session, now)).items(): + batch.add(source_module="host_agent", resource_name=name, + metric_name="maintenance_failed", value=value, recorded_at=now) + await batch.record(session) if gave_up: logger.warning("host_agent: %d maintenance job(s) timed out", gave_up) + for line in said: + logger.info("host_agent maintenance: %s", line) except Exception: logger.exception("host_agent: maintenance tick failed") diff --git a/plugins/host_agent/templates/maintenance.html b/plugins/host_agent/templates/maintenance.html new file mode 100644 index 0000000..aa9e4c4 --- /dev/null +++ b/plugins/host_agent/templates/maintenance.html @@ -0,0 +1,143 @@ +{% extends "base.html" %} +{% from "_macros.html" import crumbs, empty_state, page_header %} +{% block title %}Maintenance — Steward{% endblock %} +{% block breadcrumb %}{{ crumbs([("Hosts", "/hosts/"), ("Maintenance", "")]) }}{% endblock %} +{% block content %} +{# A group's updates and reboots on a weekly cadence, one host at a time + (#5768). What needs attention first: a run going now, a run that failed; + then the plans with their recent runs. #} +{% macro run_hosts(rows) -%} +
+ {% for r in rows %} +
+ {{ r.name }} + {{ r.text }} +
+ {% endfor %} +
+{%- endmacro %} + +{{ page_header("Maintenance") }} +

+ Each plan updates an inventory group's hosts once a week, one host at a time. A host reboots + only when its update needs it, unless the plan says otherwise; a swarm node is drained first + and made active again after. The next host starts once the last is back in service. A failure + stops the run, and the host's Maintenance failed figure can drive an alert rule. +

+ +{% if live %} +{% for l in live %} +
+

Running: {{ l.plan }} · started {{ l.run.started_at|ago }}{% if l.run.manual %} · run now{% endif %}

+ {{ run_hosts(l.rows) }} +
+{% endfor %} +{% endif %} + +{% for f in failed %} +
+ Failed: {{ f.plan }} + · {{ f.run.finished_at|ago }} · {{ f.run.detail or "no detail" }} +
+{% endfor %} + +{% if plans %} +{% for p in plans %} +
+
+

{{ p.plan.name }}{% if not p.plan.enabled %} · paused{% endif %}

+ {% if session.user_role == 'admin' %} +
+
{{ csrf_field() }} + +
+
{{ csrf_field() }} + +
+
{{ csrf_field() }} + +
+
+ {% endif %} +
+
+ Group {{ p.group }} ({{ p.hosts }} host{{ 's' if p.hosts != 1 }}) + · {{ p.when }} + · reboot {{ {'if_required': 'only when required', 'always': 'always', 'never': 'never'}[p.plan.reboot_policy] }} + {% if p.next %}· next {{ p.next|when }}{% endif %} +
+ {% for r in p.runs %} +
+ + {{ {'running': 'Running', 'done': 'Done', 'failed': 'Failed', 'stopped': 'Stopped'}[r.run.state] }} + · {{ r.run.started_at|when }}{% if r.run.manual %} · run now{% endif %}{% if r.run.detail %} · {{ r.run.detail|truncate(100) }}{% endif %} + +
{{ run_hosts(r.rows) }}
+
+ {% else %} +

No runs yet.

+ {% endfor %} +
+{% endfor %} +{% else %} +
{{ empty_state("No maintenance plans yet." ~ (" Add one below." if session.user_role == 'admin' else "")) }}
+{% endif %} + +{% if session.user_role == 'admin' %} +
+

Add a plan

+ {% if groups %} +
{{ csrf_field() }} +
+ + +
+
+ + +
+
+ + +
+
+ + +
+
+ + +
+
+ + +
+
+ + +
+ +
+

+ Hosts whose agent has no maintenance helper are skipped and listed; run Update agent on them. + No host is started once the window has passed. +

+ {% else %} +

+ Plans work on inventory groups. Add one under Ansible → Inventory. +

+ {% endif %} +
+{% endif %} +{% endblock %} diff --git a/plugins/host_agent/templates/panel.html b/plugins/host_agent/templates/panel.html index b6108c4..f595aeb 100644 --- a/plugins/host_agent/templates/panel.html +++ b/plugins/host_agent/templates/panel.html @@ -27,7 +27,7 @@ {% if maint_job %} {% set running = maint_job.state not in ('done', 'failed', 'timed_out') %}
- Maintenance: {{ maint_summary }} + Maintenance: {{ maint_summary }} · {% if running %}since {{ maint_job.state_at|ago }}{% else %}{{ maint_job.finished_at|ago }}{% endif %}{% if maint_job.state in ('failed', 'timed_out') and maint_job.detail %} · {{ maint_job.detail|truncate(90) }}{% endif %}
{% endif %} diff --git a/steward/audit/service.py b/steward/audit/service.py index 3aac462..e7683b8 100644 --- a/steward/audit/service.py +++ b/steward/audit/service.py @@ -138,6 +138,10 @@ ACTIONS = { "host.created": "added host", "host.renamed": "renamed host", "host_agent.maintenance": "started maintenance (update, reboot if needed) on host", + "host_agent.maintenance_plan_created": "added maintenance plan", + "host_agent.maintenance_plan_toggled": "paused or resumed maintenance plan", + "host_agent.maintenance_plan_deleted": "deleted maintenance plan", + "host_agent.maintenance_plan_run": "ran maintenance plan now", "host.deleted": "deleted host", "monitor.created": "added monitor", "monitor.updated": "changed monitor", diff --git a/steward/backup/contents.py b/steward/backup/contents.py index 074fe39..870ef2c 100644 --- a/steward/backup/contents.py +++ b/steward/backup/contents.py @@ -16,6 +16,8 @@ from __future__ import annotations KEPT: frozenset[str] = frozenset({ "app_settings", "users", "user_invites", "api_tokens", "hosts", "host_agent_registrations", + # A group's maintenance cadence (#5768): configuration. + "host_maintenance_plans", "monitors", "alert_rules", "alert_states", "maintenance_windows", "ansible_targets", "ansible_groups", "ansible_target_groups", "ansible_schedules", @@ -39,7 +41,7 @@ EXCLUDED: frozenset[str] = frozenset({ # A host's downtime as its agent reported it: history, like the metrics. "host_downtime", # Maintenance jobs (#5767): run history, like ansible_runs. - "host_maintenance_jobs", + "host_maintenance_jobs", "host_maintenance_runs", "email_outbox", "email_outbox_events", "password_resets", # Run history; its logs live on disk, not in the database. "ansible_runs", diff --git a/steward/core/formatting.py b/steward/core/formatting.py index 9c1b4a9..c9e0597 100644 --- a/steward/core/formatting.py +++ b/steward/core/formatting.py @@ -345,6 +345,7 @@ METRICS: dict[str, tuple[str, str]] = { "updates_pending": ("Updates waiting", "packages the host's package manager would upgrade, checked hourly"), "updates_security": ("Security updates waiting", "of the updates waiting, those from a security source"), "reboot_required": ("Reboot required", "1 when the host needs a reboot to finish an update, 0 when not"), + "maintenance_failed": ("Maintenance failed", "1 when the host's latest scheduled or manual update failed, timed out or didn't come back into service"), } # Where a rule's metric comes from, as the picker and the rules list say it. diff --git a/steward/templates/hosts/list.html b/steward/templates/hosts/list.html index c8211e1..bb5fc5b 100644 --- a/steward/templates/hosts/list.html +++ b/steward/templates/hosts/list.html @@ -17,6 +17,7 @@ {% if session.user_role == 'admin' %} Agent fleet {% endif %} + Maintenance {% if can_add %} {% endif %} diff --git a/tests/core/test_host_page.py b/tests/core/test_host_page.py index 893b82d..91326b9 100644 --- a/tests/core/test_host_page.py +++ b/tests/core/test_host_page.py @@ -304,7 +304,7 @@ async def test_the_agent_card_follows_a_maintenance_job_and_says_why_it_cant_sta maint_blocker="a maintenance job is already running on it", **kw) free = await render_plugin_template(app, "host_agent", "panel.html", maint_job=None, maint_blocker=None, **kw) - assert busy.index("Maintenance: Rebooting") < busy.index("4 updates waiting") + assert busy.index("Maintenance: Rebooting") < busy.index("4 updates waiting") assert "Can't update from here: a maintenance job is already running on it." in busy assert ">Update now" not in busy assert ">Update now" in free and 'name="reboot_policy" value="never"' in free diff --git a/tests/integration/test_host_maintenance_cadence.py b/tests/integration/test_host_maintenance_cadence.py new file mode 100644 index 0000000..141d654 --- /dev/null +++ b/tests/integration/test_host_maintenance_cadence.py @@ -0,0 +1,134 @@ +"""Integration (#5768): a plan updates its group's hosts one at a time; the +second starts only once the first is back, and a failure stops the run and +flags the host for an alert rule. + +The plugin's cadence and models are reached through the module the app loaded +(see test_docker_disk_page): importing the plugin's models again raises +"Table already defined".""" +from __future__ import annotations + +import asyncio +import hashlib +import importlib +import os +import uuid +from datetime import datetime, timezone + +import pytest + +pytestmark = pytest.mark.integration + + +def _app(): + if not os.environ.get("STEWARD_DATABASE_URL"): + pytest.skip("needs Postgres") + from steward.app import create_app + return create_app(testing=False) + + +def _ha(sub): + from steward.core.plugin_manager import get_loaded_plugin_modules + loaded = get_loaded_plugin_modules().get("host_agent") + base = loaded.__name__ if loaded is not None else "plugins.host_agent" + return importlib.import_module(f"{base}.{sub}") + + +async def _seed(app): + from sqlalchemy import text + from steward.models.ansible_inventory import AnsibleGroup, AnsibleTarget + from steward.models.hosts import Host + from steward.models.users import User + tag = uuid.uuid4().hex[:8] + out = {"tokens": {}, "hosts": {}} + async with app.db_sessionmaker() as s: + async with s.begin(): + user = User(username=f"cad-{tag}", email=f"cad-{tag}@test", password_hash="x") + group = AnsibleGroup(name=f"cad-{tag}") + s.add_all([user, group]) + await s.flush() + for n in ("a", "b"): + host = Host(name=f"cad-{n}-{tag}", address="10.9.9.6") + s.add(host) + await s.flush() + target = AnsibleTarget(name=f"cad-{n}-{tag}", address="10.9.9.6", host_id=host.id) + target.groups = [group] + s.add(target) + token = f"tok-{n}-{tag}-{uuid.uuid4().hex}" + await s.execute(text( + "INSERT INTO host_agent_registrations " + "(id, host_id, token_hash, token_created_at, last_seen_at, maint_helper, " + " created_at, updated_at) VALUES (:id, :h, :hash, now(), now(), true, now(), now())"), + {"id": str(uuid.uuid4()), "h": host.id, + "hash": hashlib.sha256(token.encode()).hexdigest()}) + out["tokens"][n], out["hosts"][n] = token, host + plan_id = str(uuid.uuid4()) + await s.execute(text( + "INSERT INTO host_maintenance_plans (id, name, group_id, weekday, at_time, time_zone, " + " window_minutes, reboot_policy, enabled, created_by, created_at) " + "VALUES (:id, :name, :g, 6, '03:00', 'UTC', 240, 'if_required', true, :u, now())"), + {"id": plan_id, "name": f"plan-{tag}", "g": group.id, "u": user.id}) + out["plan_id"] = plan_id + return out + + +async def _post(app, token, events=()): + resp = await app.test_client().post( + "/plugins/host_agent/ingest", + json={"samples": [{"ts": datetime.now(timezone.utc).isoformat(), "cpu_pct": 1.0}], + "events": list(events)}, + headers={"Authorization": f"Bearer {token}"}) + assert resp.status_code == 200, await resp.get_data(as_text=True) + return (await resp.get_json()).get("maintenance", []) + + +async def _tick(app): + cadence = _ha("cadence") + async with app.db_sessionmaker() as s: + async with s.begin(): + await cadence.tick(s, datetime.now(timezone.utc)) + return await cadence.failed_hosts(s, datetime.now(timezone.utc)) + + +async def _run(app, plan_id): + from sqlalchemy import text + async with app.db_sessionmaker() as s: + return (await s.execute(text( + "SELECT state, position, detail FROM host_maintenance_runs WHERE plan_id = :p"), + {"p": plan_id})).one() + + +def test_hosts_go_one_at_a_time_and_a_failure_stops_the_run(): + app = _app() + + async def _go(): + seed = await _seed(app) + cadence, models = _ha("cadence"), _ha("models") + async with app.db_sessionmaker() as s: + async with s.begin(): + plan = await s.get(models.HostMaintenancePlan, seed["plan_id"]) + await cadence.start_run(s, plan, datetime.now(timezone.utc), manual=True) + await _tick(app) + + # Only the first host (by name) has a job; the second waits. + ta, tb = seed["tokens"]["a"], seed["tokens"]["b"] + assert await _post(app, tb) == [] + (step,) = await _post(app, ta) + assert step["action"] == "update" + done = {"kind": "maintenance", "id": step["id"], "action": "update", "state": "done", + "ok": True, "packages": 2, "reboot_required": False} + await _post(app, ta, [done]) + + # Back in service (posting, not a swarm node): the next host starts. + await _tick(app) + assert (await _run(app, seed["plan_id"]))[:2] == ("running", 1) + (step_b,) = await _post(app, tb) + failed = {"kind": "maintenance", "id": step_b["id"], "action": "update", + "state": "failed", "ok": False, "detail": "dpkg was interrupted"} + await _post(app, tb, [failed]) + + flags = await _tick(app) + state, _, detail = await _run(app, seed["plan_id"]) + assert state == "failed" and seed["hosts"]["b"].name in detail and "dpkg" in detail + assert flags[seed["hosts"]["b"].name] == 1.0 and flags[seed["hosts"]["a"].name] == 0.0 + + asyncio.run(_go()) diff --git a/tests/plugins/host_agent/test_cadence.py b/tests/plugins/host_agent/test_cadence.py new file mode 100644 index 0000000..693c6c7 --- /dev/null +++ b/tests/plugins/host_agent/test_cadence.py @@ -0,0 +1,133 @@ +# tests/plugins/host_agent/test_cadence.py +"""#5768: when a plan is due, when a host is back in service, and what a run +does next. Model-free: plans, runs, jobs and swarm rows are SimpleNamespaces.""" +import json +from datetime import datetime, timedelta, timezone +from types import SimpleNamespace as NS + +import pytest + +from plugins.host_agent import cadence as c +from plugins.host_agent import maintenance as m + +# Wednesday 2026-10-14 23:00 UTC +NOW = datetime(2026, 10, 14, 23, 0, tzinfo=timezone.utc) + + +def _plan(**kw): + base = dict(weekday=6, at_time="03:00", time_zone="UTC", window_minutes=240, enabled=True, + last_slot_at=None, created_at=NOW - timedelta(days=30), reboot_policy="if_required") + base.update(kw) + return NS(**base) + + +def test_the_last_slot_is_the_latest_weekday_and_time_in_the_plans_zone(): + assert c.last_slot(_plan(), NOW) == datetime(2026, 10, 11, 3, 0, tzinfo=timezone.utc) + # 03:00 in New York is 07:00 UTC in October (EDT) + ny = _plan(time_zone="America/New_York") + assert c.last_slot(ny, NOW) == datetime(2026, 10, 11, 7, 0, tzinfo=timezone.utc) + assert c.next_slot(_plan(), NOW) == datetime(2026, 10, 18, 3, 0, tzinfo=timezone.utc) + today = _plan(weekday=2, at_time="22:30") + assert c.last_slot(today, NOW) == datetime(2026, 10, 14, 22, 30, tzinfo=timezone.utc) + + +def test_a_slot_fires_once_and_only_while_its_window_is_open(): + plan = _plan(weekday=2, at_time="22:30") + assert c.is_due(plan, NOW) == datetime(2026, 10, 14, 22, 30, tzinfo=timezone.utc) + plan.last_slot_at = c.is_due(plan, NOW) + assert c.is_due(plan, NOW) is None # fired already + late = _plan(weekday=2, at_time="18:00") # 5 h ago, window 4 h + assert c.is_due(late, NOW) is None + assert c.is_due(_plan(weekday=2, at_time="22:30", enabled=False), NOW) is None + new = _plan(weekday=2, at_time="22:30", created_at=NOW - timedelta(minutes=5)) + assert c.is_due(new, NOW) is None # made after the slot + + +def _node(**kw): + base = dict(status="ready", availability="active", updated_at=NOW) + base.update(kw) + return NS(**base) + + +def _svc(name, running, desired, mode="replicated"): + return NS(service_name=name, running=running, desired=desired, mode=mode, updated_at=NOW) + + +def test_a_host_is_back_once_posting_its_node_active_and_its_services_at_replicas(): + finished = NOW - timedelta(minutes=5) + kw = dict(job_finished_at=finished, now=NOW) + assert c.back_in_service(last_seen_at=None, node=None, services=[], **kw) == "its agent isn't posting yet" + assert c.back_in_service(last_seen_at=NOW, node=None, services=[], **kw) is None + assert "drain" in c.back_in_service(last_seen_at=NOW, node=_node(availability="drain"), + services=[_svc("web", 1, 1)], **kw) + assert c.back_in_service(last_seen_at=NOW, node=_node(updated_at=finished - timedelta(minutes=1)), + services=[], **kw) == "no swarm report since it came back" + assert c.back_in_service(last_seen_at=NOW, node=_node(), + services=[_svc("web", 1, 2), _svc("agent", 2, 3, mode="global")], + **kw) == "services below their replicas: web" + assert c.back_in_service(last_seen_at=NOW, node=_node(), services=[_svc("web", 2, 2)], **kw) is None + + +def _run(**kw): + base = dict(host_ids=json.dumps(["h1", "h2"]), position=0, started_at=NOW - timedelta(minutes=10), + settle_since=None, skipped="[]") + base.update(kw) + return NS(**base) + + +def _job(state, detail=None): + return NS(state=state, detail=detail) + + +@pytest.mark.parametrize("job,back,run_kw,expected", [ + (None, None, {}, ("start", None)), + (_job("rebooting"), None, {}, ("wait", None)), + (_job("done"), None, {}, ("next", None)), + (_job("done"), "its agent isn't posting yet", {}, ("settling", "its agent isn't posting yet")), + (_job("failed", "update failed: dpkg"), None, {}, ("fail", "update failed: dpkg")), + (_job("timed_out"), None, {}, ("fail", "Timed out")), + (None, None, {"position": 2}, ("done", None)), +]) +def test_what_a_run_does_next(job, back, run_kw, expected): + assert c.next_move(_run(**run_kw), _plan(), job, back, NOW) == expected + + +def test_a_host_that_never_settles_fails_the_run_and_the_window_stops_new_hosts(): + run = _run(settle_since=NOW - timedelta(minutes=31)) + move, why = c.next_move(run, _plan(), _job("done"), "services below their replicas: web", NOW) + assert move == "fail" and why.startswith("not back in service within 30 min") + late = _run(started_at=NOW - timedelta(hours=5), position=1) + assert c.next_move(late, _plan(), None, None, NOW) == ("stop", "the window ended with 1 host not started") + + +def test_a_plan_says_when_it_runs(): + assert c.describe(_plan()) == "Sundays at 03:00 (UTC), up to 4 h" + assert c.describe(_plan(window_minutes=90)) == "Sundays at 03:00 (UTC), up to 90 min" + + +def test_always_reboots_even_when_the_update_needs_none(): + job = NS(id="j", state="updating", step_id="j-update", step_seen=False, reboot_policy="always", + reboot_needed=None, node_id=None, drain_host_id=None, packages=None, state_at=NOW, + finished_at=None, detail=None) + m.apply_result(job, {"id": "j-update", "state": "done", "ok": True, "reboot_required": False}, NOW) + assert job.state == "rebooting" + + +async def test_the_page_leads_with_a_running_run_and_lists_the_plans(app): + from quart import session + from tests._templates import render_plugin_template + plan = NS(id="p1", name="Docker hosts", enabled=True, reboot_policy="if_required") + run = NS(id="r1", state="running", started_at=NOW, manual=True, detail=None, finished_at=None) + rows = [{"host_id": "h1", "name": "docker02", "text": "Rebooting", "bad": False, "busy": True, "detail": None}, + {"host_id": "h2", "name": "docker03", "text": "Waiting", "bad": False, "busy": False, "detail": None}] + async with app.test_request_context("/plugins/host_agent/maintenance"): + session["user_role"] = "admin" + page = await render_plugin_template( + app, "host_agent", "maintenance.html", + plans=[{"plan": plan, "group": "docker", "hosts": 2, "when": "Sundays at 03:00 (UTC), up to 4 h", + "next": NOW, "runs": [{"run": run, "rows": rows}]}], + groups=[NS(id="g1", name="docker")], live=[{"run": run, "plan": "Docker hosts", "rows": rows}], + failed=[], weekdays=c.WEEKDAYS, policies=m.REBOOT_POLICIES, default_zone="UTC") + assert page.index("Running: Docker hosts") < page.index("Add a plan") + assert page.index("docker02") < page.index("docker03") + assert "reboot only when required" in page and ">Run now" in page