feat(host_agent): a reboot reads as a reboot, and lost contact as its span (#5756)
#5756 "The agent says when its host rebooted and when it lost contact, so downtime is recorded as what it was". Agent 1.14.0 posts events with the next post that gets through: started (with the host's boot time), stopping (on SIGTERM, with whether systemd says the host is going down) and lost_contact (from its first unanswered post to the one carrying it). Steward keeps them as the host's downtime (host_downtime): reboot, agent_stop, unclean (a boot after the agent was last heard that nobody announced) or lost_contact. Retried posts apply once. An announced stop explains the silence for reboot_grace_seconds (Settings -> Plugins -> host_agent, 10 minutes): seconds_since_seen reads 0 until then, and the host's other rules go quiet with it (agents_away beside #5721's registry). Past the grace, agent offline fires. The host's Agent card lists the last 7 days of downtime. Rows are pruned at data.retention_days. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -12,6 +12,7 @@ from steward.core.watch import watch
|
||||
from steward.hosts.service import agent_stale_after_seconds
|
||||
from steward.models.hosts import Host
|
||||
from steward.models.metrics import PluginMetricLatest
|
||||
from .downtime import announced_stops, restart_grace_seconds
|
||||
from .models import HostAgentRegistration
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -36,6 +37,7 @@ def seconds_since_seen(regs: Iterable, host_names: dict[str, str],
|
||||
now: datetime, watching_since: datetime | None = None,
|
||||
arrived: dict[str, datetime] | None = None,
|
||||
already_silent: set[str] | frozenset = frozenset(),
|
||||
explained_until: dict[str, datetime] | None = None,
|
||||
) -> list[tuple[str, float]]:
|
||||
"""(host name, seconds since its agent last reported) per registration.
|
||||
|
||||
@@ -52,6 +54,11 @@ def seconds_since_seen(regs: Iterable, host_names: dict[str, str],
|
||||
whole silence counts. Counting them from `watching_since` read them as
|
||||
just seen, which resolved their alert on every restart and fired it again
|
||||
three minutes later (#5719).
|
||||
|
||||
`explained_until` ({host id: time}) is a stop the agent announced, a reboot
|
||||
or its own restart, plus its grace (#5756): until then the agent reads as
|
||||
just seen. A host still silent past it reads its whole silence, so agent
|
||||
offline fires as the grace runs out, for silence nobody explained.
|
||||
"""
|
||||
out = []
|
||||
for r in regs:
|
||||
@@ -62,6 +69,8 @@ def seconds_since_seen(regs: Iterable, host_names: dict[str, str],
|
||||
continue
|
||||
if watching_since is not None and name not in already_silent:
|
||||
seen = max(seen, watching_since)
|
||||
if r.host_id in (explained_until or {}):
|
||||
seen = max(seen, explained_until[r.host_id])
|
||||
out.append((name, max(0.0, (now - seen).total_seconds())))
|
||||
return out
|
||||
|
||||
@@ -88,9 +97,10 @@ async def record_seconds_since_seen(app) -> None:
|
||||
PluginMetricLatest.metric_name == "seconds_since_seen",
|
||||
PluginMetricLatest.resource_name.in_(list(host_names.values())),
|
||||
PluginMetricLatest.value > stale_after))).scalars())
|
||||
explained = await announced_stops(session, now, restart_grace_seconds(app.config))
|
||||
batch = MetricBatch()
|
||||
for name, seconds in seconds_since_seen(regs, host_names, now, watching_since,
|
||||
arrivals(app), already_silent):
|
||||
arrivals(app), already_silent, explained):
|
||||
batch.add(source_module="host_agent", resource_name=name,
|
||||
metric_name="seconds_since_seen", value=seconds, recorded_at=now)
|
||||
await batch.record(session)
|
||||
|
||||
Reference in New Issue
Block a user