# plugins/host_agent/timeline.py """What the host agent puts on the timeline: each maintenance job once it ends, the swarm services a maintenance run spread back out after its last host, and each stretch a host was away (a reboot, a crash, lost contact, the agent restarting). One entry per job and per downtime row, kept up to date as the row changes: a downtime that ends gets its length, a failed job whose host came back late reads as back. """ from __future__ import annotations import logging from datetime import datetime from sqlalchemy import select from steward.timeline import catalog from steward.timeline.record import record_or_update, username from . import downtime, maintenance logger = logging.getLogger(__name__) TYPES = ( ("maintenance.done", "Maintenance run", "automation", "host_os", True, "A host's agent updated it, and rebooted it when the update needed one."), ("maintenance.failed", "Maintenance failed", "automation", "host_os", True, "A maintenance job failed or timed out."), ("maintenance.rebalanced", "Swarm services spread back out", "automation", "service", True, "After a maintenance run, its swarm services were moved back onto the nodes it had drained."), ("host.rebooted", "Host rebooted", "observed", "host_os", True, "A host went down and came back, announced or not (a crash, a power cut)."), ("host.lost_contact", "Agent lost contact", "observed", "network", True, "A host's agent ran but couldn't reach Steward."), ("host.agent_restarted", "Agent stopped or restarted", "observed", "host_os", False, "The agent stopped or restarted while its host stayed up."), ) def register() -> None: for t in TYPES: catalog.register_type(*t) from .models import HostDowntime, HostMaintenanceJob, HostMaintenanceRun catalog.register_ref("maintenance_job", HostMaintenanceJob) catalog.register_ref("maintenance_run", HostMaintenanceRun) catalog.register_ref("host_downtime", HostDowntime) async def _host_name(session, host_id: str) -> str | None: from steward.models.hosts import Host host = await session.get(Host, host_id) return host.name if host is not None else None async def job_entry(session, job, host_name: str | None = None): """The entry for a finished maintenance job, written or brought up to date.""" if job.state not in maintenance.FINISHED: return None host_name = host_name or await _host_name(session, job.host_id) or job.host_id return await record_or_update( session, ("maintenance_job", job.id), type="maintenance.failed" if maintenance.still_failed(job) else "maintenance.done", at=job.created_at, until=job.finished_at, summary=f"Maintenance on {host_name}: {maintenance.summary(job)}", hosts=[host_name], subject=host_name, detail={"state": job.state, "packages": job.packages, "rebooted": job.back_at is not None, "note": job.detail, "url": f"/hosts/{job.host_id}"}, actor=await username(session, job.created_by) or "Steward") async def run_entry(session, run): """The entry for a run's spreading of swarm services back out, once it ended.""" from . import rebalance from .models import HostMaintenancePlan rb = run.rebalance or {} if rb.get("state") not in ("done", "failed", "timed_out"): return None plan = await session.get(HostMaintenancePlan, run.plan_id) summary = rebalance.outcome(run)[1] or "swarm services spread back out" return await record_or_update( session, ("maintenance_run", run.id), type="maintenance.rebalanced", at=datetime.fromisoformat(rb["at"]), until=datetime.fromisoformat(rb["finished_at"]) if rb.get("finished_at") else None, summary=f"{plan.name if plan else 'Maintenance'}: {summary}", hosts=[], subject=None, detail={"state": rb["state"], "moved": rb.get("moved") or [], "left": rb.get("left") or [], "note": rb.get("detail"), "url": "/plugins/host_agent/maintenance"}, actor=await username(session, plan.created_by if plan else None) or "Steward") _DOWNTIME_TYPES = {"reboot": "host.rebooted", "unclean": "host.rebooted", "lost_contact": "host.lost_contact", "agent_stop": "host.agent_restarted"} async def downtime_entry(session, row, host_name: str | None = None): """The entry for one stretch a host was away, written or brought up to date.""" from steward.core.formatting import human_duration host_name = host_name or await _host_name(session, row.host_id) or row.host_id seen = downtime.describe(row, row.ended_at or row.started_at) said = seen["label"][:1].lower() + seen["label"][1:] said += f" for {human_duration(seen['seconds'])}" if row.ended_at is not None else ", not back yet" return await record_or_update( session, ("host_downtime", row.id), type=_DOWNTIME_TYPES.get(row.kind, "host.rebooted"), at=row.started_at, until=row.ended_at, summary=f"{host_name}: {said}", hosts=[host_name], subject=host_name, detail={"kind": row.kind, "url": f"/hosts/{row.host_id}"}, actor="Steward") async def write(session, what: str, coro) -> None: """Run one entry writer in a savepoint: the source's own row matters more than its entry, so a failure is logged and nothing else is undone.""" try: async with session.begin_nested(): await coro except Exception: logger.exception("Timeline: the entry for %s could not be written", what) async def backfill(session, since) -> list: """Entries for the jobs and downtime that predate the timeline.""" from .models import HostDowntime, HostMaintenanceJob out = [] for job in (await session.execute(select(HostMaintenanceJob).where( HostMaintenanceJob.created_at >= since, HostMaintenanceJob.state.in_(maintenance.FINISHED)))).scalars().all(): out.append(await job_entry(session, job)) for row in (await session.execute(select(HostDowntime).where( HostDowntime.started_at >= since))).scalars().all(): out.append(await downtime_entry(session, row)) return out