fix(alerts): Steward's own outage no longer reads as every agent going silent (#5441)

After Steward or its database was down, every agent's age and every
series' last sample were old, so all agent-offline rules fired and all
rules went no-data at once, clearing a minute later. With per-account
alert email that would have mailed everyone a dozen notices per restart.

A periodic job now ticks a Watch (steward/core/watch.py) once it has
reached the database; the Watch says since when runs have been unbroken
(process start, or the end of a gap over 120 s). seconds_since_seen and
the no-data sweep count silence from no earlier than that, so a source
must stay quiet while Steward is watching before it counts as silent.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

steward/core/alerts.py

@@ -18,6 +18,7 @@ from steward.models.alerts import (
     MaintenanceWindow,
 )
 from steward.core.formatting import human_duration
+from steward.core.watch import watch
 from steward.models.metrics import PluginMetric, PluginMetricLatest
 
 if TYPE_CHECKING:
@@ -297,13 +298,19 @@ def no_data_after(interval: float | None) -> float:
 
 
 def silent_since(last_sample_at: datetime | None, interval: float | None,
-                 watching_since: datetime, now: datetime) -> datetime | None:
+                 watching_since: datetime, now: datetime,
+                 awake_since: datetime | None = None) -> datetime | None:
     """When a series went silent, or None while it is still reporting.
 
     A series that has never had a sample counts from when the rule started
-    watching it (created, changed or re-enabled).
+    watching it (created, changed or re-enabled). Nothing counts from before
+    `awake_since`, when Steward last started watching without a break: no
+    sample can arrive while Steward is down, so its own outage is not the
+    series' silence (#5441).
     """
     last = last_sample_at or watching_since
+    if awake_since is not None:
+        last = max(last, awake_since)
     limit = timedelta(seconds=no_data_after(interval))
     return last + limit if now - last > limit else None
 
@@ -632,6 +639,7 @@ async def check_silence(app: "Quart") -> None:
                 .join(AlertRule, AlertState.rule_id == AlertRule.id)
                 .where(AlertRule.enabled.is_(True), AlertState.no_data_since.is_(None))
             )).all()
+            awake_since = watch(app, "alert_silence").tick(now)
             if not rows:
                 return
             windows = (await session.execute(
@@ -641,7 +649,7 @@ async def check_silence(app: "Quart") -> None:
             for state, rule in rows:
                 watching_since = max(rule.created_at, state.last_transition_at)
                 since = silent_since(state.last_sample_at, state.sample_interval_s,
-                                     watching_since, now)
+                                     watching_since, now, awake_since)
                 if since is None:
                     continue
                 state.no_data_since = since

steward/core/watch.py

@@ -0,0 +1,43 @@
+"""Since when Steward has been watching without a break (#5441).
+
+Silence is measured by Steward: an agent's age is now minus its last report,
+a series is silent once no sample has arrived for a while. While Steward (or
+its database) is down nothing arrives, so on its return every agent and every
+series looks silent at once, and every agent-offline and no-data alert used to
+fire together for an outage that was Steward's own.
+
+A periodic job ticks its Watch each run, once it has reached the database.
+The Watch answers "since when have I been running without a break": since
+the process started, or since the end of a gap longer than GAP_S between
+ticks (the database was down, the loop stalled). Silence is then counted from
+no earlier than that, so a source has to stay quiet while Steward is
+watching before it counts as silent.
+"""
+from __future__ import annotations
+
+from datetime import datetime
+
+# Two missed 60 s ticks. A gap this long means Steward was not watching; a
+# shorter one only delays a real alert by the length of the gap.
+GAP_S = 120.0
+
+
+class Watch:
+    def __init__(self) -> None:
+        self.since: datetime | None = None
+        self.last: datetime | None = None
+
+    def tick(self, now: datetime) -> datetime:
+        """Record a run at `now`; return since when runs have been unbroken."""
+        if self.since is None or self.last is None or (now - self.last).total_seconds() > GAP_S:
+            self.since = now
+        self.last = now
+        return self.since
+
+
+def watch(app, name: str) -> Watch:
+    """The app's Watch for one periodic job, made on first use."""
+    watches = getattr(app, "_watches", None)
+    if watches is None:
+        watches = app._watches = {}
+    return watches.setdefault(name, Watch())