bvandeusenandClaude Opus 5.5
2b97c36fd4
The host agent writes each maintenance job once it ends, and each stretch a host was away (a reboot, a crash, lost contact, the agent restarting), kept up to date until it is back. Docker judges each snapshot: a unit on a new image, or a container recreated, is a deploy; deaths past the Thresholds crash-loop count are one crash-loop entry that later deaths extend; single restarts are recorded but hidden. An anomaly has one entry whose type follows its review: only those judged real are shown by default. A source whose record changes keeps one entry for it (record_or_update). Plugins register the records their entries point to, so an aged-out job or downtime reads "Details no longer kept". The first fill runs once per source (timeline.filled), the host agent's jobs and downtime included. get_anomaly's "what else was going on" reads the timeline for the host or resource instead of querying alerts, runs, maintenance, downtime and container deaths itself; Docker's own source keeps only starts, stops and health changes. A new kind, Observed, is for what Steward saw happen with no rule or person behind it. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
106 lines
4.7 KiB
Python
106 lines
4.7 KiB
Python
# plugins/host_agent/timeline.py
|
|
"""What the host agent puts on the timeline: each maintenance job once it
|
|
ends, and each stretch a host was away (a reboot, a crash, lost contact, the
|
|
agent restarting). One entry per job and per downtime row, kept up to date
|
|
as the row changes: a downtime that ends gets its length, a failed job
|
|
whose host came back late reads as back.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import logging
|
|
|
|
from sqlalchemy import select
|
|
|
|
from steward.timeline import catalog
|
|
from steward.timeline.record import record_or_update, username
|
|
|
|
from . import downtime, maintenance
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
TYPES = (
|
|
("maintenance.done", "Maintenance run", "automation", "host_os", True,
|
|
"A host's agent updated it, and rebooted it when the update needed one."),
|
|
("maintenance.failed", "Maintenance failed", "automation", "host_os", True,
|
|
"A maintenance job failed or timed out."),
|
|
("host.rebooted", "Host rebooted", "observed", "host_os", True,
|
|
"A host went down and came back, announced or not (a crash, a power cut)."),
|
|
("host.lost_contact", "Agent lost contact", "observed", "network", True,
|
|
"A host's agent ran but couldn't reach Steward."),
|
|
("host.agent_restarted", "Agent stopped or restarted", "observed", "host_os", False,
|
|
"The agent stopped or restarted while its host stayed up."),
|
|
)
|
|
|
|
|
|
def register() -> None:
|
|
for t in TYPES:
|
|
catalog.register_type(*t)
|
|
from .models import HostDowntime, HostMaintenanceJob
|
|
catalog.register_ref("maintenance_job", HostMaintenanceJob)
|
|
catalog.register_ref("host_downtime", HostDowntime)
|
|
|
|
|
|
async def _host_name(session, host_id: str) -> str | None:
|
|
from steward.models.hosts import Host
|
|
host = await session.get(Host, host_id)
|
|
return host.name if host is not None else None
|
|
|
|
|
|
async def job_entry(session, job, host_name: str | None = None):
|
|
"""The entry for a finished maintenance job, written or brought up to date."""
|
|
if job.state not in maintenance.FINISHED:
|
|
return None
|
|
host_name = host_name or await _host_name(session, job.host_id) or job.host_id
|
|
return await record_or_update(
|
|
session, ("maintenance_job", job.id),
|
|
type="maintenance.failed" if maintenance.still_failed(job) else "maintenance.done",
|
|
at=job.created_at, until=job.finished_at,
|
|
summary=f"Maintenance on {host_name}: {maintenance.summary(job)}",
|
|
hosts=[host_name], subject=host_name,
|
|
detail={"state": job.state, "packages": job.packages, "rebooted": job.back_at is not None,
|
|
"note": job.detail, "url": f"/hosts/{job.host_id}"},
|
|
actor=await username(session, job.created_by) or "Steward")
|
|
|
|
|
|
_DOWNTIME_TYPES = {"reboot": "host.rebooted", "unclean": "host.rebooted",
|
|
"lost_contact": "host.lost_contact", "agent_stop": "host.agent_restarted"}
|
|
|
|
|
|
async def downtime_entry(session, row, host_name: str | None = None):
|
|
"""The entry for one stretch a host was away, written or brought up to date."""
|
|
from steward.core.formatting import human_duration
|
|
host_name = host_name or await _host_name(session, row.host_id) or row.host_id
|
|
seen = downtime.describe(row, row.ended_at or row.started_at)
|
|
said = seen["label"][:1].lower() + seen["label"][1:]
|
|
said += f" for {human_duration(seen['seconds'])}" if row.ended_at is not None else ", not back yet"
|
|
return await record_or_update(
|
|
session, ("host_downtime", row.id),
|
|
type=_DOWNTIME_TYPES.get(row.kind, "host.rebooted"),
|
|
at=row.started_at, until=row.ended_at, summary=f"{host_name}: {said}",
|
|
hosts=[host_name], subject=host_name,
|
|
detail={"kind": row.kind, "url": f"/hosts/{row.host_id}"}, actor="Steward")
|
|
|
|
|
|
async def write(session, what: str, coro) -> None:
|
|
"""Run one entry writer in a savepoint: the source's own row matters more
|
|
than its entry, so a failure is logged and nothing else is undone."""
|
|
try:
|
|
async with session.begin_nested():
|
|
await coro
|
|
except Exception:
|
|
logger.exception("Timeline: the entry for %s could not be written", what)
|
|
|
|
|
|
async def backfill(session, since) -> list:
|
|
"""Entries for the jobs and downtime that predate the timeline."""
|
|
from .models import HostDowntime, HostMaintenanceJob
|
|
out = []
|
|
for job in (await session.execute(select(HostMaintenanceJob).where(
|
|
HostMaintenanceJob.created_at >= since,
|
|
HostMaintenanceJob.state.in_(maintenance.FINISHED)))).scalars().all():
|
|
out.append(await job_entry(session, job))
|
|
for row in (await session.execute(select(HostDowntime).where(
|
|
HostDowntime.started_at >= since))).scalars().all():
|
|
out.append(await downtime_entry(session, row))
|
|
return out
|