FabledSteward/plugins/host_agent/downtime.py

# plugins/host_agent/downtime.py
"""What a host's agent says happened to it while it was away (#5756).

An agent ≥ 1.14.0 tells Steward when it stops (and whether its host is going
down with it), when it starts (and when its host booted), and how long it
couldn't reach Steward. Each becomes a HostDowntime row, so a reboot reads as
a reboot with its length, and "agent offline" is left for silence nobody
explained. A post can be retried, so every event is applied idempotently.
"""
from __future__ import annotations

from datetime import datetime, timedelta, timezone

from sqlalchemy import select

from .models import HostDowntime

# A stop the agent announced: the silence after it is explained, for a while.
ANNOUNCED = ("reboot", "agent_stop")

LABELS = {
    "reboot": "Rebooted",
    "agent_stop": "Agent stopped",
    "unclean": "Restarted without shutting down",
    "lost_contact": "Lost contact",
}


def restart_grace_seconds(config) -> int:
    """How long an announced stop explains the agent's silence (Settings →
    Plugins → host_agent, default 10 minutes): past it, agent offline fires."""
    return int(((config.get("PLUGINS") or {}).get("host_agent") or {})
               .get("reboot_grace_seconds", 600))


def _at(value) -> datetime | None:
    if not isinstance(value, str):
        return None
    try:
        got = datetime.fromisoformat(value.replace("Z", "+00:00"))
    except ValueError:
        return None
    return got if got.tzinfo else got.replace(tzinfo=timezone.utc)


def _when(event: dict) -> datetime | None:
    return _at(event.get("at") or event.get("from"))


async def record_events(session, host_id: str, events, last_seen_at: datetime | None) -> int:
    """Apply one post's events to the host's downtime, oldest first; returns
    how many rows were added or changed. `last_seen_at` is when the agent was
    last heard before this post: a boot after it that nobody announced is a
    restart without a shutdown. Each row added or changed goes on the
    timeline, kept up to date as it ends."""
    if not isinstance(events, list):
        return 0
    timed = sorted(((t, e) for e in events if isinstance(e, dict)
                    for t in [_when(e)] if t is not None), key=lambda te: te[0])
    changed = []
    for at, event in timed:
        kind = event.get("kind")
        row = None
        if kind == "stopping":
            row = await _stopping(session, host_id, at, event.get("host_shutdown"))
        elif kind == "started":
            row = await _started(session, host_id, at, _at(event.get("boot_at")), last_seen_at)
        elif kind == "lost_contact":
            row = await _lost_contact(session, host_id, at, _at(event.get("to")))
        if row is not None and row not in changed:
            changed.append(row)
    if changed:
        from . import timeline
        await session.flush()
        for row in changed:
            await timeline.write(session, f"host downtime {row.id}", timeline.downtime_entry(session, row))
    return len(changed)


async def _stopping(session, host_id: str, at: datetime, host_shutdown) -> HostDowntime | None:
    seen = (await session.execute(select(HostDowntime.id).where(
        HostDowntime.host_id == host_id, HostDowntime.kind.in_(ANNOUNCED),
        HostDowntime.started_at == at))).first()
    if seen:
        return None
    row = HostDowntime(host_id=host_id, kind="reboot" if host_shutdown else "agent_stop", started_at=at)
    session.add(row)
    return row


async def _started(session, host_id: str, at: datetime, boot_at: datetime | None,
                   last_seen_at: datetime | None) -> HostDowntime | None:
    stop = (await session.execute(select(HostDowntime).where(
        HostDowntime.host_id == host_id, HostDowntime.kind.in_(ANNOUNCED),
        HostDowntime.ended_at.is_(None), HostDowntime.started_at <= at)
        .order_by(HostDowntime.started_at.desc()).limit(1))).scalar_one_or_none()
    if stop is not None:
        stop.ended_at = at
        # The boot time settles what the stop was: the host came back from a
        # boot after it, or it stayed up while only the agent restarted.
        if boot_at is not None:
            stop.kind = "reboot" if boot_at > stop.started_at else "agent_stop"
        return stop
    if boot_at is None or last_seen_at is None or boot_at <= last_seen_at:
        return None
    seen = (await session.execute(select(HostDowntime.id).where(
        HostDowntime.host_id == host_id, HostDowntime.kind == "unclean",
        HostDowntime.ended_at == at))).first()
    if seen:
        return None
    row = HostDowntime(host_id=host_id, kind="unclean", started_at=last_seen_at, ended_at=at)
    session.add(row)
    return row


async def _lost_contact(session, host_id: str, since: datetime,
                        until: datetime | None) -> HostDowntime | None:
    if until is None or until <= since:
        return None
    row = (await session.execute(select(HostDowntime).where(
        HostDowntime.host_id == host_id, HostDowntime.kind == "lost_contact",
        HostDowntime.started_at == since))).scalar_one_or_none()
    if row is None:
        row = HostDowntime(host_id=host_id, kind="lost_contact", started_at=since, ended_at=until)
        session.add(row)
        return row
    if row.ended_at is None or until > row.ended_at:
        row.ended_at = until
        return row
    return None


async def announced_stops(session, now: datetime, grace_seconds: int) -> dict[str, datetime]:
    """{host id: when the silence its agent announced stops being explained}
    for every announced stop still going and inside its grace."""
    rows = (await session.execute(select(HostDowntime.host_id, HostDowntime.started_at).where(
        HostDowntime.kind.in_(ANNOUNCED), HostDowntime.ended_at.is_(None),
        HostDowntime.started_at > now - timedelta(seconds=grace_seconds)))).all()
    out: dict[str, datetime] = {}
    for host_id, started in rows:
        until = started + timedelta(seconds=grace_seconds)
        out[host_id] = max(out.get(host_id, until), until)
    return out


async def hosts_away(session, now: datetime, grace_seconds: int) -> set[str]:
    """Names of the hosts in announced_stops: their silence has a reason."""
    from steward.models.hosts import Host
    ids = list(await announced_stops(session, now, grace_seconds))
    if not ids:
        return set()
    return set((await session.execute(select(Host.name).where(Host.id.in_(ids)))).scalars())


def describe(row: HostDowntime, now: datetime) -> dict:
    """One downtime row as the host page shows it."""
    label = LABELS.get(row.kind, row.kind)
    if row.kind == "agent_stop" and row.ended_at is not None:
        label = "Agent restarted"
    end = row.ended_at or now
    return {"kind": row.kind, "label": label, "started_at": row.started_at,
            "ended_at": row.ended_at, "ongoing": row.ended_at is None,
            "seconds": max(0.0, (end - row.started_at).total_seconds())}


async def recent(session, host_id: str, now: datetime, days: int = 7, limit: int = 6) -> list[dict]:
    """The host's downtime in the last `days`, newest first."""
    rows = (await session.execute(select(HostDowntime).where(
        HostDowntime.host_id == host_id,
        HostDowntime.started_at > now - timedelta(days=days))
        .order_by(HostDowntime.started_at.desc()).limit(limit))).scalars().all()
    return [describe(r, now) for r in rows]