fix(alerts): Steward's own outage no longer reads as every agent going silent (#5441)

After Steward or its database was down, every agent's age and every
series' last sample were old, so all agent-offline rules fired and all
rules went no-data at once, clearing a minute later. With per-account
alert email that would have mailed everyone a dozen notices per restart.

A periodic job now ticks a Watch (steward/core/watch.py) once it has
reached the database; the Watch says since when runs have been unbroken
(process start, or the end of a gap over 120 s). seconds_since_seen and
the no-data sweep count silence from no earlier than that, so a source
must stay quiet while Steward is watching before it counts as silent.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

plugins/host_agent/scheduler.py

@@ -8,6 +8,7 @@ from sqlalchemy import select
 
 from steward.core.alerts import MetricBatch
 from steward.core.scheduler import ScheduledTask
+from steward.core.watch import watch
 from steward.models.hosts import Host
 from .models import HostAgentRegistration
 
@@ -15,11 +16,16 @@ logger = logging.getLogger(__name__)
 
 
 def seconds_since_seen(regs: Iterable, host_names: dict[str, str],
-                       now: datetime) -> list[tuple[str, float]]:
+                       now: datetime, watching_since: datetime | None = None
+                       ) -> list[tuple[str, float]]:
     """(host name, seconds since its agent last reported) per registration.
 
     An agent that has never reported counts from when it was registered. A
-    registration whose host is gone is skipped.
+    registration whose host is gone is skipped. Nothing counts from before
+    `watching_since`, when Steward last started watching without a break: an
+    agent can't report while Steward is down, so that time is not its silence
+    (#5441). An agent that is really gone reads as offline once it has been
+    quiet that long while Steward watched.
     """
     out = []
     for r in regs:
@@ -27,6 +33,8 @@ def seconds_since_seen(regs: Iterable, host_names: dict[str, str],
         seen = r.last_seen_at or r.created_at
         if name is None or seen is None:
             continue
+        if watching_since is not None:
+            seen = max(seen, watching_since)
         out.append((name, max(0.0, (now - seen).total_seconds())))
     return out
 
@@ -38,13 +46,14 @@ async def record_seconds_since_seen(app) -> None:
     async with app.db_sessionmaker() as session:
         async with session.begin():
             regs = (await session.execute(select(HostAgentRegistration))).scalars().all()
+            watching_since = watch(app, "host_agent_seconds_since_seen").tick(now)
             if not regs:
                 return
             host_names = dict((await session.execute(
                 select(Host.id, Host.name).where(Host.id.in_([r.host_id for r in regs]))
             )).all())
             batch = MetricBatch()
-            for name, seconds in seconds_since_seen(regs, host_names, now):
+            for name, seconds in seconds_since_seen(regs, host_names, now, watching_since):
                 batch.add(source_module="host_agent", resource_name=name,
                           metric_name="seconds_since_seen", value=seconds, recorded_at=now)
             await batch.record(session)