From 264d44ac81a3362c7604c684712732069f981d36 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Wed, 14 Oct 2026 16:46:11 -0400 Subject: [PATCH] fix(host_agent): an agent already offline stays offline through a Steward restart (#5719) Counting every agent's silence from when Steward started watching again (#5441) read an agent that was already gone as just seen: its agent-offline alert resolved at 0 s and fired again three minutes later, two false emails per restart (VDNT-Docker03, four times in a day). An agent whose last recorded seconds_since_seen was already past the offline window now keeps counting its whole silence; one that was fine when Steward stopped still counts from the restart, so Steward's own downtime still fires nothing. Co-Authored-By: Claude Opus 5.5 --- plugins/host_agent/scheduler.py | 24 +++++++++++++++++++--- tests/plugins/host_agent/test_scheduler.py | 20 ++++++++++++++++++ 2 files changed, 41 insertions(+), 3 deletions(-) diff --git a/plugins/host_agent/scheduler.py b/plugins/host_agent/scheduler.py index 2f246a8..c9beb1a 100644 --- a/plugins/host_agent/scheduler.py +++ b/plugins/host_agent/scheduler.py @@ -9,7 +9,9 @@ from sqlalchemy import select from steward.core.alerts import MetricBatch from steward.core.scheduler import ScheduledTask from steward.core.watch import watch +from steward.hosts.service import agent_stale_after_seconds from steward.models.hosts import Host +from steward.models.metrics import PluginMetricLatest from .models import HostAgentRegistration logger = logging.getLogger(__name__) @@ -32,7 +34,8 @@ def arrivals(app) -> dict[str, datetime]: def seconds_since_seen(regs: Iterable, host_names: dict[str, str], now: datetime, watching_since: datetime | None = None, - arrived: dict[str, datetime] | None = None + arrived: dict[str, datetime] | None = None, + already_silent: set[str] | frozenset = frozenset(), ) -> list[tuple[str, float]]: """(host name, seconds since its agent last reported) per registration. @@ -43,6 +46,12 @@ def seconds_since_seen(regs: Iterable, host_names: dict[str, str], (#5441). An agent that is really gone reads as offline once it has been quiet that long while Steward watched. `arrived` (arrivals(app)) counts a post Steward received but couldn't store as the agent seen (#5514). + + `already_silent` names hosts whose last recorded figure was already past + the offline window: their agent was gone before Steward stopped, so their + whole silence counts. Counting them from `watching_since` read them as + just seen, which resolved their alert on every restart and fired it again + three minutes later (#5719). """ out = [] for r in regs: @@ -51,7 +60,7 @@ def seconds_since_seen(regs: Iterable, host_names: dict[str, str], if t is not None), default=None) if name is None or seen is None: continue - if watching_since is not None: + if watching_since is not None and name not in already_silent: seen = max(seen, watching_since) out.append((name, max(0.0, (now - seen).total_seconds()))) return out @@ -70,9 +79,18 @@ async def record_seconds_since_seen(app) -> None: host_names = dict((await session.execute( select(Host.id, Host.name).where(Host.id.in_([r.host_id for r in regs])) )).all()) + # Agents whose last figure was already past the offline window keep + # counting through a restart of Steward (#5719). + stale_after = agent_stale_after_seconds(app.config) + already_silent = set((await session.execute( + select(PluginMetricLatest.resource_name).where( + PluginMetricLatest.source_module == "host_agent", + PluginMetricLatest.metric_name == "seconds_since_seen", + PluginMetricLatest.resource_name.in_(list(host_names.values())), + PluginMetricLatest.value > stale_after))).scalars()) batch = MetricBatch() for name, seconds in seconds_since_seen(regs, host_names, now, watching_since, - arrivals(app)): + arrivals(app), already_silent): batch.add(source_module="host_agent", resource_name=name, metric_name="seconds_since_seen", value=seconds, recorded_at=now) await batch.record(session) diff --git a/tests/plugins/host_agent/test_scheduler.py b/tests/plugins/host_agent/test_scheduler.py index 00e566f..2cf81ca 100644 --- a/tests/plugins/host_agent/test_scheduler.py +++ b/tests/plugins/host_agent/test_scheduler.py @@ -71,3 +71,23 @@ def test_arrivals_are_kept_per_app(): app, other = SimpleNamespace(), SimpleNamespace() arrivals(app)["h1"] = NOW assert arrivals(app) == {"h1": NOW} and arrivals(other) == {} + + +def test_an_agent_already_offline_stays_offline_through_a_restart(): + """#5719: VDNT-Docker03's agent was silent for hours when Steward + restarted, and counting it from the restart read it as just seen: its + alert resolved, then fired again three minutes later, two emails a + restart. An agent already past the offline window keeps its whole + silence; one that was fine when Steward stopped still counts from the + restart (#5441).""" + back = NOW - timedelta(minutes=1) + regs = [FakeReg("h1", NOW - timedelta(hours=5), NOW - timedelta(days=9)), + FakeReg("h2", NOW - timedelta(minutes=10), NOW - timedelta(days=9))] + assert seconds_since_seen(regs, NAMES, NOW, back, already_silent={"nas"}) == [ + ("nas", 5 * 3600.0), ("web", 60.0)] + + +def test_an_offline_agent_that_reports_after_the_restart_is_back(): + back = NOW - timedelta(minutes=1) + regs = [FakeReg("h1", NOW - timedelta(seconds=15), NOW - timedelta(days=9))] + assert seconds_since_seen(regs, NAMES, NOW, back, already_silent={"nas"}) == [("nas", 15.0)]