FabledSteward/plugins/host_agent/cadence.py

# plugins/host_agent/cadence.py
"""A group's maintenance cadence (#5768): every week, its hosts updated one at
a time, each rebooting only when its update needs it (or as the plan says).

A plan names an inventory group, a weekday and a time in its time zone, and a
window. At that time a run starts with the group's hosts in a fixed order. Each
host gets a maintenance job (maintenance.py); the next host starts only when
the last one is back in service: its agent posting, its swarm node ready and
active, every replicated service at its replicas. A job that fails, or a host
that doesn't come back into service, stops the run, and the host's
maintenance_failed figure goes to 1 for an alert rule to catch. No host is
started after the window; the run then stops and says which hosts it didn't
reach.

The decisions are model-free here (plans and runs are anything with the
attributes); the database side is below them, and the plugin's minute tick
drives it. Everything is on the rows, so a restart of Steward, or the reboot of
the host it runs on, picks up where it was.
"""
from __future__ import annotations

import json
from datetime import datetime, time, timedelta, timezone

from . import maintenance

WEEKDAYS = ("Monday", "Tuesday", "Wednesday", "Thursday", "Friday", "Saturday", "Sunday")
# How long a host may take to be back in service after its job finished.
SETTLE_LIMIT = timedelta(minutes=30)
# An agent heard from this recently is posting.
POSTING_WITHIN = timedelta(minutes=3)
RUN_STATES = ("running", "done", "failed", "stopped")


def _zone(name: str):
    from steward.core.formatting import zone
    return zone(name)


def parse_at(at_time: str) -> time:
    h, m = (int(x) for x in at_time.split(":", 1))
    if not (0 <= h < 24 and 0 <= m < 60):
        raise ValueError(at_time)
    return time(h, m)


def last_slot(plan, now: datetime) -> datetime:
    """The most recent scheduled time at or before `now`, in UTC."""
    tz = _zone(plan.time_zone)
    local = now.astimezone(tz)
    day = local.date() - timedelta(days=(local.weekday() - plan.weekday) % 7)
    slot = datetime.combine(day, parse_at(plan.at_time), tzinfo=tz)
    if slot > local:
        slot -= timedelta(days=7)
    return slot.astimezone(timezone.utc)


def next_slot(plan, now: datetime) -> datetime:
    return last_slot(plan, now) + timedelta(days=7)


def is_due(plan, now: datetime) -> datetime | None:
    """The slot to fire now, or None. A slot fires once (last_slot_at), and
    only while its window is still open: a Steward that was down over the slot
    starts late, not a day late."""
    if not plan.enabled:
        return None
    slot = last_slot(plan, now)
    since = plan.last_slot_at or plan.created_at
    if since is not None and slot <= since:
        return None
    if now - slot >= timedelta(minutes=plan.window_minutes):
        return None
    return slot


def back_in_service(*, last_seen_at, node, services, job_finished_at, now: datetime) -> str | None:
    """None when the host is back in service, else why not yet. `node` is the
    freshest swarm-node row for the job's node (None outside a swarm, or when
    the job has no node); `services` the freshest row per swarm service. Only
    rows reported after the job finished count, so a stale "all fine" from
    before the reboot never passes."""
    if last_seen_at is None or now - last_seen_at > POSTING_WITHIN:
        return "its agent isn't posting yet"
    if node is None:
        return None
    if node.updated_at < job_finished_at:
        return "no swarm report since it came back"
    if node.status != "ready" or node.availability != "active":
        return f"its swarm node is {node.status}/{node.availability}"
    fresh = [s for s in services if s.updated_at >= job_finished_at]
    if not fresh:
        return "no swarm report since it came back"
    short = sorted(s.service_name for s in fresh
                   if s.mode == "replicated" and s.running < s.desired)
    if short:
        return "services below their replicas: " + ", ".join(short[:5])
    return None


def next_move(run, plan, job, back: str | None, now: datetime) -> tuple[str, str | None]:
    """What a running run does now: ("start", None) the current host's job;
    ("wait", None); ("next", None) on to the next host; ("settling", why);
    ("fail", why); ("stop", why) the window ended; ("done", None)."""
    hosts = json.loads(run.host_ids)
    if run.position >= len(hosts):
        return "done", None
    if job is None:
        if now >= run.started_at + timedelta(minutes=plan.window_minutes):
            left = len(hosts) - run.position
            return "stop", f"the window ended with {left} host{'s' if left != 1 else ''} not started"
        return "start", None
    if job.state in maintenance.ACTIVE:
        return "wait", None
    if job.state != "done":
        return "fail", job.detail or maintenance.LABELS[job.state]
    if back is None:
        return "next", None
    if run.settle_since is not None and now - run.settle_since > SETTLE_LIMIT:
        return "fail", f"not back in service within {int(SETTLE_LIMIT.total_seconds() // 60)} min: {back}"
    return "settling", back


def describe(plan) -> str:
    """"Sundays at 03:00 (Europe/London), up to 4 h" for the page."""
    hours = plan.window_minutes / 60
    span = f"{hours:g} h" if plan.window_minutes % 60 == 0 else f"{plan.window_minutes} min"
    return f"{WEEKDAYS[plan.weekday]}s at {plan.at_time} ({plan.time_zone}), up to {span}"


# ── applied to the database: the minute tick, Run now, the page ──────────────

async def group_hosts(session, group_id: str) -> list:
    """The group's hosts (through their linked Ansible targets), by name."""
    from sqlalchemy import select
    from steward.models.ansible_inventory import AnsibleTarget, ansible_target_groups
    from steward.models.hosts import Host
    rows = (await session.execute(
        select(Host).join(AnsibleTarget, AnsibleTarget.host_id == Host.id)
        .join(ansible_target_groups, ansible_target_groups.c.target_id == AnsibleTarget.id)
        .where(ansible_target_groups.c.group_id == group_id))).scalars().all()
    return sorted({h.id: h for h in rows}.values(), key=lambda h: h.name.lower())


async def start_run(session, plan, now: datetime, manual: bool = False):
    """A run of the plan with the group's hosts in order. A host whose agent
    has no maintenance helper is passed over, and the run says so."""
    from sqlalchemy import select
    from .models import HostAgentRegistration, HostMaintenanceRun
    hosts = await group_hosts(session, plan.group_id)
    regs = {r.host_id: r for r in (await session.execute(select(HostAgentRegistration).where(
        HostAgentRegistration.host_id.in_([h.id for h in hosts])))).scalars()} if hosts else {}
    order, skipped = [], []
    for h in hosts:
        reg = regs.get(h.id)
        if reg is None:
            skipped.append({"host_id": h.id, "reason": "no agent"})
        elif not reg.maint_helper:
            skipped.append({"host_id": h.id, "reason": "no maintenance helper: run Update agent"})
        else:
            order.append(h.id)
    run = HostMaintenanceRun(plan_id=plan.id, state="running", host_ids=json.dumps(order),
                             position=0, skipped=json.dumps(skipped), manual=manual,
                             started_at=now)
    session.add(run)
    await session.flush()
    return run


async def _back(session, host, job, reg, now: datetime) -> str | None:
    from sqlalchemy import select
    node = None
    services: list = []
    if job.node_id:
        try:
            from plugins.docker.models import DockerSwarmNode, DockerSwarmService
        except ImportError:
            DockerSwarmNode = DockerSwarmService = None
        if DockerSwarmNode is not None:
            node = (await session.execute(select(DockerSwarmNode).where(
                DockerSwarmNode.node_id == job.node_id)
                .order_by(DockerSwarmNode.updated_at.desc()).limit(1))).scalar_one_or_none()
            freshest: dict = {}
            for s in (await session.execute(select(DockerSwarmService))).scalars():
                if s.service_name not in freshest or s.updated_at > freshest[s.service_name].updated_at:
                    freshest[s.service_name] = s
            services = list(freshest.values())
    return back_in_service(last_seen_at=reg.last_seen_at if reg else None, node=node,
                           services=services, job_finished_at=job.finished_at or now, now=now)


async def advance(session, run, plan, now: datetime) -> str:
    """Move one running run on as far as it can go now; returns its state."""
    from sqlalchemy import select
    from steward.models.hosts import Host
    from .models import HostAgentRegistration, HostMaintenanceJob
    for _ in range(64):                             # a host per pass; bounded
        hosts = json.loads(run.host_ids)
        host_id = hosts[run.position] if run.position < len(hosts) else None
        job = host = reg = None
        if host_id is not None:
            host = await session.get(Host, host_id)
            reg = (await session.execute(select(HostAgentRegistration).where(
                HostAgentRegistration.host_id == host_id))).scalar_one_or_none()
            job = (await session.execute(select(HostMaintenanceJob).where(
                HostMaintenanceJob.run_id == run.id, HostMaintenanceJob.host_id == host_id)
                .order_by(HostMaintenanceJob.created_at.desc()).limit(1))).scalar_one_or_none()
        back = None
        if job is not None and job.state == "done":
            back = await _back(session, host, job, reg, now)
        move, why = next_move(run, plan, job, back, now)
        if move == "start":
            if host is None:
                _skip(run, host_id, "the host is gone")
                continue
            try:
                await maintenance.create_job(session, host, reg, plan.created_by,
                                             plan.reboot_policy, now, run_id=run.id)
            except maintenance.CannotStart as exc:
                _skip(run, host_id, str(exc))
                continue
            return run.state
        if move == "next":
            run.position += 1
            run.settle_since = None
            continue
        if move == "settling":
            run.settle_since = run.settle_since or now
            return run.state
        if move == "fail":
            if job is not None and job.state == "done":
                maintenance._finish(job, "failed", now, why)     # the host never came back into service
            name = host.name if host else host_id
            _end(run, "failed", now, f"{name}: {why}")
        elif move == "stop":
            _end(run, "stopped", now, why)
        elif move == "done":
            _end(run, "done", now, None)
        return run.state
    return run.state


def _skip(run, host_id: str, reason: str) -> None:
    skipped = json.loads(run.skipped)
    skipped.append({"host_id": host_id, "reason": reason})
    run.skipped = json.dumps(skipped)
    hosts = json.loads(run.host_ids)
    hosts.pop(run.position)
    run.host_ids = json.dumps(hosts)


def _end(run, state: str, now: datetime, detail: str | None) -> None:
    run.state, run.finished_at, run.settle_since = state, now, None
    if detail:
        run.detail = detail[:maintenance.DETAIL_MAX]


async def tick(session, now: datetime) -> list[str]:
    """Start due plans and move running runs on; returns lines worth logging."""
    from sqlalchemy import select
    from .models import HostMaintenancePlan, HostMaintenanceRun
    said = []
    running = {r.plan_id: r for r in (await session.execute(select(HostMaintenanceRun).where(
        HostMaintenanceRun.state == "running"))).scalars()}
    for plan in (await session.execute(select(HostMaintenancePlan))).scalars():
        slot = is_due(plan, now)
        if slot is None:
            continue
        plan.last_slot_at = slot
        if plan.id in running:
            said.append(f"plan {plan.name!r}: the {slot:%Y-%m-%d %H:%M}Z slot came with a run still going")
            continue
        running[plan.id] = await start_run(session, plan, now)
        said.append(f"plan {plan.name!r}: run started")
    for run in running.values():
        plan = await session.get(HostMaintenancePlan, run.plan_id)
        if plan is None:
            continue
        before = run.state
        after = await advance(session, run, plan, now)
        if after != before:
            said.append(f"plan {plan.name!r}: run {after}" + (f" ({run.detail})" if run.detail else ""))
    return said


async def failed_hosts(session, now: datetime) -> dict[str, float]:
    """{host name: 1 or 0}: whether each host's latest finished maintenance job
    failed or timed out, for the maintenance_failed figure an alert rule
    watches. A later job that succeeds puts it back to 0."""
    from sqlalchemy import select
    from steward.models.hosts import Host
    from .models import HostMaintenanceJob
    rows = (await session.execute(
        select(Host.name, HostMaintenanceJob.state, HostMaintenanceJob.finished_at)
        .join(Host, Host.id == HostMaintenanceJob.host_id)
        .where(HostMaintenanceJob.state.in_(maintenance.FINISHED),
               HostMaintenanceJob.finished_at >= now - timedelta(days=30)))).all()
    latest: dict = {}
    for name, state, finished in rows:
        if name not in latest or finished > latest[name][1]:
            latest[name] = (state, finished)
    return {name: 0.0 if state == "done" else 1.0 for name, (state, _) in latest.items()}