# plugins/host_agent/timeline.py """What the host agent puts on the timeline: each maintenance job once it ends, and each stretch a host was away (a reboot, a crash, lost contact, the agent restarting). One entry per job and per downtime row, kept up to date as the row changes: a downtime that ends gets its length, a failed job whose host came back late reads as back. """ from __future__ import annotations import logging from sqlalchemy import select from steward.timeline import catalog from steward.timeline.record import record_or_update, username from . import downtime, maintenance logger = logging.getLogger(__name__) TYPES = ( ("maintenance.done", "Maintenance run", "automation", "host_os", True, "A host's agent updated it, and rebooted it when the update needed one."), ("maintenance.failed", "Maintenance failed", "automation", "host_os", True, "A maintenance job failed or timed out."), ("host.rebooted", "Host rebooted", "observed", "host_os", True, "A host went down and came back, announced or not (a crash, a power cut)."), ("host.lost_contact", "Agent lost contact", "observed", "network", True, "A host's agent ran but couldn't reach Steward."), ("host.agent_restarted", "Agent stopped or restarted", "observed", "host_os", False, "The agent stopped or restarted while its host stayed up."), ) def register() -> None: for t in TYPES: catalog.register_type(*t) from .models import HostDowntime, HostMaintenanceJob catalog.register_ref("maintenance_job", HostMaintenanceJob) catalog.register_ref("host_downtime", HostDowntime) async def _host_name(session, host_id: str) -> str | None: from steward.models.hosts import Host host = await session.get(Host, host_id) return host.name if host is not None else None async def job_entry(session, job, host_name: str | None = None): """The entry for a finished maintenance job, written or brought up to date.""" if job.state not in maintenance.FINISHED: return None host_name = host_name or await _host_name(session, job.host_id) or job.host_id return await record_or_update( session, ("maintenance_job", job.id), type="maintenance.failed" if maintenance.still_failed(job) else "maintenance.done", at=job.created_at, until=job.finished_at, summary=f"Maintenance on {host_name}: {maintenance.summary(job)}", hosts=[host_name], subject=host_name, detail={"state": job.state, "packages": job.packages, "rebooted": job.back_at is not None, "note": job.detail, "url": f"/hosts/{job.host_id}"}, actor=await username(session, job.created_by) or "Steward") _DOWNTIME_TYPES = {"reboot": "host.rebooted", "unclean": "host.rebooted", "lost_contact": "host.lost_contact", "agent_stop": "host.agent_restarted"} async def downtime_entry(session, row, host_name: str | None = None): """The entry for one stretch a host was away, written or brought up to date.""" from steward.core.formatting import human_duration host_name = host_name or await _host_name(session, row.host_id) or row.host_id seen = downtime.describe(row, row.ended_at or row.started_at) said = seen["label"][:1].lower() + seen["label"][1:] said += f" for {human_duration(seen['seconds'])}" if row.ended_at is not None else ", not back yet" return await record_or_update( session, ("host_downtime", row.id), type=_DOWNTIME_TYPES.get(row.kind, "host.rebooted"), at=row.started_at, until=row.ended_at, summary=f"{host_name}: {said}", hosts=[host_name], subject=host_name, detail={"kind": row.kind, "url": f"/hosts/{row.host_id}"}, actor="Steward") async def write(session, what: str, coro) -> None: """Run one entry writer in a savepoint: the source's own row matters more than its entry, so a failure is logged and nothing else is undone.""" try: async with session.begin_nested(): await coro except Exception: logger.exception("Timeline: the entry for %s could not be written", what) async def backfill(session, since) -> list: """Entries for the jobs and downtime that predate the timeline.""" from .models import HostDowntime, HostMaintenanceJob out = [] for job in (await session.execute(select(HostMaintenanceJob).where( HostMaintenanceJob.created_at >= since, HostMaintenanceJob.state.in_(maintenance.FINISHED)))).scalars().all(): out.append(await job_entry(session, job)) for row in (await session.execute(select(HostDowntime).where( HostDowntime.started_at >= since))).scalars().all(): out.append(await downtime_entry(session, row)) return out