FabledSteward/plugins/host_agent/timeline.py

# plugins/host_agent/timeline.py
"""What the host agent puts on the timeline: each maintenance job once it
ends, the swarm services a maintenance run spread back out after its last
host, and each stretch a host was away (a reboot, a crash, lost contact, the
agent restarting). One entry per job and per downtime row, kept up to date
as the row changes: a downtime that ends gets its length, a failed job
whose host came back late reads as back.
"""
from __future__ import annotations

import logging
from datetime import datetime

from sqlalchemy import select

from steward.timeline import catalog
from steward.timeline.record import record_or_update, username

from . import downtime, maintenance

logger = logging.getLogger(__name__)

TYPES = (
    ("maintenance.done", "Maintenance run", "automation", "host_os", True,
     "A host's agent updated it, and rebooted it when the update needed one."),
    ("maintenance.failed", "Maintenance failed", "automation", "host_os", True,
     "A maintenance job failed or timed out."),
    ("maintenance.rebalanced", "Swarm services spread back out", "automation", "service", True,
     "After a maintenance run, its swarm services were moved back onto the nodes it had drained."),
    ("host.rebooted", "Host rebooted", "observed", "host_os", True,
     "A host went down and came back, announced or not (a crash, a power cut)."),
    ("host.lost_contact", "Agent lost contact", "observed", "network", True,
     "A host's agent ran but couldn't reach Steward."),
    ("host.agent_restarted", "Agent stopped or restarted", "observed", "host_os", False,
     "The agent stopped or restarted while its host stayed up."),
)


def register() -> None:
    for t in TYPES:
        catalog.register_type(*t)
    from .models import HostDowntime, HostMaintenanceJob, HostMaintenanceRun
    catalog.register_ref("maintenance_job", HostMaintenanceJob)
    catalog.register_ref("maintenance_run", HostMaintenanceRun)
    catalog.register_ref("host_downtime", HostDowntime)


async def _host_name(session, host_id: str) -> str | None:
    from steward.models.hosts import Host
    host = await session.get(Host, host_id)
    return host.name if host is not None else None


async def job_entry(session, job, host_name: str | None = None):
    """The entry for a finished maintenance job, written or brought up to date."""
    if job.state not in maintenance.FINISHED:
        return None
    host_name = host_name or await _host_name(session, job.host_id) or job.host_id
    return await record_or_update(
        session, ("maintenance_job", job.id),
        type="maintenance.failed" if maintenance.still_failed(job) else "maintenance.done",
        at=job.created_at, until=job.finished_at,
        summary=f"Maintenance on {host_name}: {maintenance.summary(job)}",
        hosts=[host_name], subject=host_name,
        detail={"state": job.state, "packages": job.packages, "rebooted": job.back_at is not None,
                "note": job.detail, "url": f"/hosts/{job.host_id}"},
        actor=await username(session, job.created_by) or "Steward")


async def run_entry(session, run):
    """The entry for a run's spreading of swarm services back out, once it ended."""
    from . import rebalance
    from .models import HostMaintenancePlan
    rb = run.rebalance or {}
    if rb.get("state") not in ("done", "failed", "timed_out"):
        return None
    plan = await session.get(HostMaintenancePlan, run.plan_id)
    summary = rebalance.outcome(run)[1] or "swarm services spread back out"
    return await record_or_update(
        session, ("maintenance_run", run.id), type="maintenance.rebalanced",
        at=datetime.fromisoformat(rb["at"]),
        until=datetime.fromisoformat(rb["finished_at"]) if rb.get("finished_at") else None,
        summary=f"{plan.name if plan else 'Maintenance'}: {summary}",
        hosts=[], subject=None,
        detail={"state": rb["state"], "moved": rb.get("moved") or [], "left": rb.get("left") or [],
                "note": rb.get("detail"), "url": "/plugins/host_agent/maintenance"},
        actor=await username(session, plan.created_by if plan else None) or "Steward")


_DOWNTIME_TYPES = {"reboot": "host.rebooted", "unclean": "host.rebooted",
                   "lost_contact": "host.lost_contact", "agent_stop": "host.agent_restarted"}


async def downtime_entry(session, row, host_name: str | None = None):
    """The entry for one stretch a host was away, written or brought up to date."""
    from steward.core.formatting import human_duration
    host_name = host_name or await _host_name(session, row.host_id) or row.host_id
    seen = downtime.describe(row, row.ended_at or row.started_at)
    said = seen["label"][:1].lower() + seen["label"][1:]
    said += f" for {human_duration(seen['seconds'])}" if row.ended_at is not None else ", not back yet"
    return await record_or_update(
        session, ("host_downtime", row.id),
        type=_DOWNTIME_TYPES.get(row.kind, "host.rebooted"),
        at=row.started_at, until=row.ended_at, summary=f"{host_name}: {said}",
        hosts=[host_name], subject=host_name,
        detail={"kind": row.kind, "url": f"/hosts/{row.host_id}"}, actor="Steward")


async def write(session, what: str, coro) -> None:
    """Run one entry writer in a savepoint: the source's own row matters more
    than its entry, so a failure is logged and nothing else is undone."""
    try:
        async with session.begin_nested():
            await coro
    except Exception:
        logger.exception("Timeline: the entry for %s could not be written", what)


async def backfill(session, since) -> list:
    """Entries for the jobs and downtime that predate the timeline."""
    from .models import HostDowntime, HostMaintenanceJob
    out = []
    for job in (await session.execute(select(HostMaintenanceJob).where(
            HostMaintenanceJob.created_at >= since,
            HostMaintenanceJob.state.in_(maintenance.FINISHED)))).scalars().all():
        out.append(await job_entry(session, job))
    for row in (await session.execute(select(HostDowntime).where(
            HostDowntime.started_at >= since))).scalars().all():
        out.append(await downtime_entry(session, row))
    return out