fix(host_agent): a post that reached Steward counts as the agent seen, even if storing it failed (#5514)

At 20:25-20:39Z the NFS storage under steward_db slowed badly (checkpoint
sync 4.6 s, then 70.7 s with one file at 21.9 s, against the usual 0.002 s).
Ingest commits stalled past the agent's 10 s timeout and were dropped, and
last_seen_at, written in that same transaction, went with them. Agents
posting every few seconds read as silent for 3 minutes, and "Agent offline"
fired and emailed for all three hosts.

Ingest now notes each agent's arrival in memory right after the (read-only)
registration lookup, before any write; seconds_since_seen counts from the
later of last_seen_at and that arrival.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

plugins/host_agent/routes.py

@@ -24,6 +24,7 @@ from steward.core.time_range import parse_range, RANGE_OPTIONS
 from steward.models.hosts import Host
 from steward.models.metrics import PluginMetric
 from .models import HostAgentRegistration
+from .scheduler import arrivals
 # Query helpers live in a model-free module so integration tests can import them
 # without the plugin-loader double-registration guard tripping (see metrics_query).
 from .metrics_query import (
@@ -242,6 +243,8 @@ async def ingest():
             )).scalar_one_or_none()
             if host is None:
                 return _error(401, "invalid_token")
+            # The agent reached us: that holds even if storing its post fails (#5514).
+            arrivals(current_app)[reg.host_id] = received_at
 
             accepted = 0
             metric_rows: list[PluginMetric] = []

plugins/host_agent/scheduler.py

@@ -15,8 +15,24 @@ from .models import HostAgentRegistration
 logger = logging.getLogger(__name__)
 
 
+def arrivals(app) -> dict[str, datetime]:
+    """{host id: when its agent's newest post reached Steward}, in memory (#5514).
+
+    last_seen_at is written in the ingest's own transaction, so when Steward
+    can't store a post (its storage stalls and the request is dropped) the
+    agent's report is lost with it, and an agent posting every few seconds read
+    as offline. A post that arrived is the agent reaching Steward; storing it
+    is Steward's part. Set before any write, so it holds however slow they are.
+    """
+    got = getattr(app, "_agent_arrivals", None)
+    if got is None:
+        got = app._agent_arrivals = {}
+    return got
+
+
 def seconds_since_seen(regs: Iterable, host_names: dict[str, str],
-                       now: datetime, watching_since: datetime | None = None
+                       now: datetime, watching_since: datetime | None = None,
+                       arrived: dict[str, datetime] | None = None
                        ) -> list[tuple[str, float]]:
     """(host name, seconds since its agent last reported) per registration.
 
@@ -25,12 +41,14 @@ def seconds_since_seen(regs: Iterable, host_names: dict[str, str],
     `watching_since`, when Steward last started watching without a break: an
     agent can't report while Steward is down, so that time is not its silence
     (#5441). An agent that is really gone reads as offline once it has been
-    quiet that long while Steward watched.
+    quiet that long while Steward watched. `arrived` (arrivals(app)) counts a
+    post Steward received but couldn't store as the agent seen (#5514).
     """
     out = []
     for r in regs:
         name = host_names.get(r.host_id)
-        seen = r.last_seen_at or r.created_at
+        seen = max((t for t in (r.last_seen_at or r.created_at, (arrived or {}).get(r.host_id))
+                    if t is not None), default=None)
         if name is None or seen is None:
             continue
         if watching_since is not None:
@@ -53,7 +71,8 @@ async def record_seconds_since_seen(app) -> None:
                 select(Host.id, Host.name).where(Host.id.in_([r.host_id for r in regs]))
             )).all())
             batch = MetricBatch()
-            for name, seconds in seconds_since_seen(regs, host_names, now, watching_since):
+            for name, seconds in seconds_since_seen(regs, host_names, now, watching_since,
+                                                    arrivals(app)):
                 batch.add(source_module="host_agent", resource_name=name,
                           metric_name="seconds_since_seen", value=seconds, recorded_at=now)
             await batch.record(session)