fix(host_agent): an agent already offline stays offline through a Steward restart (#5719)

Counting every agent's silence from when Steward started watching again
(#5441) read an agent that was already gone as just seen: its agent-offline
alert resolved at 0 s and fired again three minutes later, two false emails
per restart (VDNT-Docker03, four times in a day). An agent whose last
recorded seconds_since_seen was already past the offline window now keeps
counting its whole silence; one that was fine when Steward stopped still
counts from the restart, so Steward's own downtime still fires nothing.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

plugins/host_agent/scheduler.py

@@ -9,7 +9,9 @@ from sqlalchemy import select
 from steward.core.alerts import MetricBatch
 from steward.core.scheduler import ScheduledTask
 from steward.core.watch import watch
+from steward.hosts.service import agent_stale_after_seconds
 from steward.models.hosts import Host
+from steward.models.metrics import PluginMetricLatest
 from .models import HostAgentRegistration
 
 logger = logging.getLogger(__name__)
@@ -32,7 +34,8 @@ def arrivals(app) -> dict[str, datetime]:
 
 def seconds_since_seen(regs: Iterable, host_names: dict[str, str],
                        now: datetime, watching_since: datetime | None = None,
-                       arrived: dict[str, datetime] | None = None
+                       arrived: dict[str, datetime] | None = None,
+                       already_silent: set[str] | frozenset = frozenset(),
                        ) -> list[tuple[str, float]]:
     """(host name, seconds since its agent last reported) per registration.
 
@@ -43,6 +46,12 @@ def seconds_since_seen(regs: Iterable, host_names: dict[str, str],
     (#5441). An agent that is really gone reads as offline once it has been
     quiet that long while Steward watched. `arrived` (arrivals(app)) counts a
     post Steward received but couldn't store as the agent seen (#5514).
+
+    `already_silent` names hosts whose last recorded figure was already past
+    the offline window: their agent was gone before Steward stopped, so their
+    whole silence counts. Counting them from `watching_since` read them as
+    just seen, which resolved their alert on every restart and fired it again
+    three minutes later (#5719).
     """
     out = []
     for r in regs:
@@ -51,7 +60,7 @@ def seconds_since_seen(regs: Iterable, host_names: dict[str, str],
                     if t is not None), default=None)
         if name is None or seen is None:
             continue
-        if watching_since is not None:
+        if watching_since is not None and name not in already_silent:
             seen = max(seen, watching_since)
         out.append((name, max(0.0, (now - seen).total_seconds())))
     return out
@@ -70,9 +79,18 @@ async def record_seconds_since_seen(app) -> None:
             host_names = dict((await session.execute(
                 select(Host.id, Host.name).where(Host.id.in_([r.host_id for r in regs]))
             )).all())
+            # Agents whose last figure was already past the offline window keep
+            # counting through a restart of Steward (#5719).
+            stale_after = agent_stale_after_seconds(app.config)
+            already_silent = set((await session.execute(
+                select(PluginMetricLatest.resource_name).where(
+                    PluginMetricLatest.source_module == "host_agent",
+                    PluginMetricLatest.metric_name == "seconds_since_seen",
+                    PluginMetricLatest.resource_name.in_(list(host_names.values())),
+                    PluginMetricLatest.value > stale_after))).scalars())
             batch = MetricBatch()
             for name, seconds in seconds_since_seen(regs, host_names, now, watching_since,
-                                                    arrivals(app)):
+                                                    arrivals(app), already_silent):
                 batch.add(source_module="host_agent", resource_name=name,
                           metric_name="seconds_since_seen", value=seconds, recorded_at=now)
             await batch.record(session)

tests/plugins/host_agent/test_scheduler.py

@@ -71,3 +71,23 @@ def test_arrivals_are_kept_per_app():
     app, other = SimpleNamespace(), SimpleNamespace()
     arrivals(app)["h1"] = NOW
     assert arrivals(app) == {"h1": NOW} and arrivals(other) == {}
+
+
+def test_an_agent_already_offline_stays_offline_through_a_restart():
+    """#5719: VDNT-Docker03's agent was silent for hours when Steward
+    restarted, and counting it from the restart read it as just seen: its
+    alert resolved, then fired again three minutes later, two emails a
+    restart. An agent already past the offline window keeps its whole
+    silence; one that was fine when Steward stopped still counts from the
+    restart (#5441)."""
+    back = NOW - timedelta(minutes=1)
+    regs = [FakeReg("h1", NOW - timedelta(hours=5), NOW - timedelta(days=9)),
+            FakeReg("h2", NOW - timedelta(minutes=10), NOW - timedelta(days=9))]
+    assert seconds_since_seen(regs, NAMES, NOW, back, already_silent={"nas"}) == [
+        ("nas", 5 * 3600.0), ("web", 60.0)]
+
+
+def test_an_offline_agent_that_reports_after_the_restart_is_back():
+    back = NOW - timedelta(minutes=1)
+    regs = [FakeReg("h1", NOW - timedelta(seconds=15), NOW - timedelta(days=9))]
+    assert seconds_since_seen(regs, NAMES, NOW, back, already_silent={"nas"}) == [("nas", 15.0)]