# plugins/host_agent/downtime.py """What a host's agent says happened to it while it was away (#5756). An agent ≥ 1.14.0 tells Steward when it stops (and whether its host is going down with it), when it starts (and when its host booted), and how long it couldn't reach Steward. Each becomes a HostDowntime row, so a reboot reads as a reboot with its length, and "agent offline" is left for silence nobody explained. A post can be retried, so every event is applied idempotently. """ from __future__ import annotations from datetime import datetime, timedelta, timezone from sqlalchemy import select from .models import HostDowntime # A stop the agent announced: the silence after it is explained, for a while. ANNOUNCED = ("reboot", "agent_stop") LABELS = { "reboot": "Rebooted", "agent_stop": "Agent stopped", "unclean": "Restarted without shutting down", "lost_contact": "Lost contact", } def restart_grace_seconds(config) -> int: """How long an announced stop explains the agent's silence (Settings → Plugins → host_agent, default 10 minutes): past it, agent offline fires.""" return int(((config.get("PLUGINS") or {}).get("host_agent") or {}) .get("reboot_grace_seconds", 600)) def _at(value) -> datetime | None: if not isinstance(value, str): return None try: got = datetime.fromisoformat(value.replace("Z", "+00:00")) except ValueError: return None return got if got.tzinfo else got.replace(tzinfo=timezone.utc) def _when(event: dict) -> datetime | None: return _at(event.get("at") or event.get("from")) async def record_events(session, host_id: str, events, last_seen_at: datetime | None) -> int: """Apply one post's events to the host's downtime, oldest first; returns how many rows were added or changed. `last_seen_at` is when the agent was last heard before this post: a boot after it that nobody announced is a restart without a shutdown. Each row added or changed goes on the timeline, kept up to date as it ends.""" if not isinstance(events, list): return 0 timed = sorted(((t, e) for e in events if isinstance(e, dict) for t in [_when(e)] if t is not None), key=lambda te: te[0]) changed = [] for at, event in timed: kind = event.get("kind") row = None if kind == "stopping": row = await _stopping(session, host_id, at, event.get("host_shutdown")) elif kind == "started": row = await _started(session, host_id, at, _at(event.get("boot_at")), last_seen_at) elif kind == "lost_contact": row = await _lost_contact(session, host_id, at, _at(event.get("to"))) if row is not None and row not in changed: changed.append(row) if changed: from . import timeline await session.flush() for row in changed: await timeline.write(session, f"host downtime {row.id}", timeline.downtime_entry(session, row)) return len(changed) async def _stopping(session, host_id: str, at: datetime, host_shutdown) -> HostDowntime | None: seen = (await session.execute(select(HostDowntime.id).where( HostDowntime.host_id == host_id, HostDowntime.kind.in_(ANNOUNCED), HostDowntime.started_at == at))).first() if seen: return None row = HostDowntime(host_id=host_id, kind="reboot" if host_shutdown else "agent_stop", started_at=at) session.add(row) return row async def _started(session, host_id: str, at: datetime, boot_at: datetime | None, last_seen_at: datetime | None) -> HostDowntime | None: stop = (await session.execute(select(HostDowntime).where( HostDowntime.host_id == host_id, HostDowntime.kind.in_(ANNOUNCED), HostDowntime.ended_at.is_(None), HostDowntime.started_at <= at) .order_by(HostDowntime.started_at.desc()).limit(1))).scalar_one_or_none() if stop is not None: stop.ended_at = at # The boot time settles what the stop was: the host came back from a # boot after it, or it stayed up while only the agent restarted. if boot_at is not None: stop.kind = "reboot" if boot_at > stop.started_at else "agent_stop" return stop if boot_at is None or last_seen_at is None or boot_at <= last_seen_at: return None seen = (await session.execute(select(HostDowntime.id).where( HostDowntime.host_id == host_id, HostDowntime.kind == "unclean", HostDowntime.ended_at == at))).first() if seen: return None row = HostDowntime(host_id=host_id, kind="unclean", started_at=last_seen_at, ended_at=at) session.add(row) return row async def _lost_contact(session, host_id: str, since: datetime, until: datetime | None) -> HostDowntime | None: if until is None or until <= since: return None row = (await session.execute(select(HostDowntime).where( HostDowntime.host_id == host_id, HostDowntime.kind == "lost_contact", HostDowntime.started_at == since))).scalar_one_or_none() if row is None: row = HostDowntime(host_id=host_id, kind="lost_contact", started_at=since, ended_at=until) session.add(row) return row if row.ended_at is None or until > row.ended_at: row.ended_at = until return row return None async def announced_stops(session, now: datetime, grace_seconds: int) -> dict[str, datetime]: """{host id: when the silence its agent announced stops being explained} for every announced stop still going and inside its grace.""" rows = (await session.execute(select(HostDowntime.host_id, HostDowntime.started_at).where( HostDowntime.kind.in_(ANNOUNCED), HostDowntime.ended_at.is_(None), HostDowntime.started_at > now - timedelta(seconds=grace_seconds)))).all() out: dict[str, datetime] = {} for host_id, started in rows: until = started + timedelta(seconds=grace_seconds) out[host_id] = max(out.get(host_id, until), until) return out async def hosts_away(session, now: datetime, grace_seconds: int) -> set[str]: """Names of the hosts in announced_stops: their silence has a reason.""" from steward.models.hosts import Host ids = list(await announced_stops(session, now, grace_seconds)) if not ids: return set() return set((await session.execute(select(Host.name).where(Host.id.in_(ids)))).scalars()) def describe(row: HostDowntime, now: datetime) -> dict: """One downtime row as the host page shows it.""" label = LABELS.get(row.kind, row.kind) if row.kind == "agent_stop" and row.ended_at is not None: label = "Agent restarted" end = row.ended_at or now return {"kind": row.kind, "label": label, "started_at": row.started_at, "ended_at": row.ended_at, "ongoing": row.ended_at is None, "seconds": max(0.0, (end - row.started_at).total_seconds())} async def recent(session, host_id: str, now: datetime, days: int = 7, limit: int = 6) -> list[dict]: """The host's downtime in the last `days`, newest first.""" rows = (await session.execute(select(HostDowntime).where( HostDowntime.host_id == host_id, HostDowntime.started_at > now - timedelta(days=days)) .order_by(HostDowntime.started_at.desc()).limit(limit))).scalars().all() return [describe(r, now) for r in rows]