feat(host_agent): a reboot reads as a reboot, and lost contact as its span (#5756)
#5756 "The agent says when its host rebooted and when it lost contact, so downtime is recorded as what it was". Agent 1.14.0 posts events with the next post that gets through: started (with the host's boot time), stopping (on SIGTERM, with whether systemd says the host is going down) and lost_contact (from its first unanswered post to the one carrying it). Steward keeps them as the host's downtime (host_downtime): reboot, agent_stop, unclean (a boot after the agent was last heard that nobody announced) or lost_contact. Retried posts apply once. An announced stop explains the silence for reboot_grace_seconds (Settings -> Plugins -> host_agent, 10 minutes): seconds_since_seen reads 0 until then, and the host's other rules go quiet with it (agents_away beside #5721's registry). Past the grace, agent offline fires. The host's Agent card lists the last 7 days of downtime. Rows are pruned at data.retention_days. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
+14
-3
@@ -4,7 +4,7 @@ import logging
|
||||
import uuid
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from typing import TYPE_CHECKING, Callable
|
||||
from typing import TYPE_CHECKING, Awaitable, Callable
|
||||
|
||||
from sqlalchemy import event as sa_event, select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
@@ -53,6 +53,7 @@ class Heartbeat:
|
||||
|
||||
_HEARTBEAT: Heartbeat | None = None
|
||||
_AGENT_FED: dict[str, Callable[[str], str]] = {}
|
||||
_AWAY: Callable[[AsyncSession], Awaitable[set[str]]] | None = None
|
||||
|
||||
|
||||
def agent_heartbeat(source_module: str, metric_name: str,
|
||||
@@ -69,6 +70,14 @@ def agent_fed(source_module: str, host_of: Callable[[str], str]) -> None:
|
||||
_AGENT_FED[source_module] = host_of
|
||||
|
||||
|
||||
def agents_away(hosts_away: Callable[[AsyncSession], Awaitable[set[str]]]) -> None:
|
||||
"""`await hosts_away(session)` names the hosts whose agent said it was
|
||||
stopping and is still inside its grace (#5756): their silence has a
|
||||
reason, so their other rules go quiet with it as when it is offline."""
|
||||
global _AWAY
|
||||
_AWAY = hosts_away
|
||||
|
||||
|
||||
def is_heartbeat(rule) -> bool:
|
||||
"""Whether `rule` watches the agent heartbeat ("agent offline")."""
|
||||
return _HEARTBEAT is not None and (rule.source_module, rule.metric_name) == (
|
||||
@@ -85,14 +94,16 @@ def host_behind(rule) -> str | None:
|
||||
|
||||
|
||||
async def offline_agents(session) -> set[str]:
|
||||
"""Hosts whose agent's last heartbeat was past its offline window."""
|
||||
"""Hosts whose agent's last heartbeat was past its offline window, and
|
||||
those whose agent said it was stopping (agents_away)."""
|
||||
if _HEARTBEAT is None:
|
||||
return set()
|
||||
return set((await session.execute(
|
||||
offline = set((await session.execute(
|
||||
select(PluginMetricLatest.resource_name).where(
|
||||
PluginMetricLatest.source_module == _HEARTBEAT.source_module,
|
||||
PluginMetricLatest.metric_name == _HEARTBEAT.metric_name,
|
||||
PluginMetricLatest.value > _HEARTBEAT.offline_after()))).scalars())
|
||||
return offline | (await _AWAY(session) if _AWAY is not None else set())
|
||||
|
||||
|
||||
async def _went_quiet_with_its_agent(session, rule_id: str) -> bool:
|
||||
|
||||
Reference in New Issue
Block a user