fix(alerts): Steward's own outage no longer reads as every agent going silent (#5441)

After Steward or its database was down, every agent's age and every
series' last sample were old, so all agent-offline rules fired and all
rules went no-data at once, clearing a minute later. With per-account
alert email that would have mailed everyone a dozen notices per restart.

A periodic job now ticks a Watch (steward/core/watch.py) once it has
reached the database; the Watch says since when runs have been unbroken
(process start, or the end of a gap over 120 s). seconds_since_seen and
the no-data sweep count silence from no earlier than that, so a source
must stay quiet while Steward is watching before it counts as silent.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

tests/integration/test_alert_no_data.py

@@ -26,6 +26,7 @@ def test_silence_is_marked_once_and_cleared_by_data(app):
     from sqlalchemy import select, update
     from steward.alerts import service as alert_svc
     from steward.core import alerts
+    from steward.core.watch import watch
     from steward.models.alerts import AlertEvent, AlertRule, AlertState, AlertStateEnum
     from steward.models.metrics import PluginMetric
     from steward.models.users import User, UserRole
@@ -77,6 +78,9 @@ def test_silence_is_marked_once_and_cleared_by_data(app):
         assert st.sample_interval_s == pytest.approx(60)
         assert st.last_sample_at == now - timedelta(minutes=18)
 
+        # Steward has been watching all along (#5441 covers a fresh start).
+        w = watch(app, "alert_silence")
+        w.since, w.last = now - timedelta(days=1), datetime.now(timezone.utc)
         await alerts.check_silence(app)
         await alerts.check_silence(app)  # already marked: nothing more
         await asyncio.sleep(0)
@@ -123,3 +127,60 @@ def test_silence_is_marked_once_and_cleared_by_data(app):
 
     with patch.object(alerts, "_dispatch_notification", fake_dispatch):
         asyncio.run(_go())
+
+
+def test_a_series_quiet_only_while_steward_was_down_is_not_silent(app):
+    """#5441: Steward comes back to a series last sampled 18 minutes ago. Its
+    outage is not the series' silence, so nothing is marked; once Steward has
+    watched for three of the series' gaps with nothing arriving, it is."""
+    from sqlalchemy import select
+    from steward.alerts import service as alert_svc
+    from steward.core import alerts
+    from steward.core.watch import watch
+    from steward.models.alerts import AlertEvent, AlertState
+    from steward.models.users import User, UserRole
+
+    sent = []
+
+    async def fake_dispatch(app_, notif_type, rule, value, event_id, note=None):
+        sent.append((rule.id, notif_type))
+
+    async def _go():
+        tag = uuid.uuid4().hex[:8]
+        res = f"back-{tag}"
+        now = datetime.now(timezone.utc)
+        async with app.db_sessionmaker() as s:
+            async with s.begin():
+                user = User(username=f"bk-{tag}", email=f"bk-{tag}@test", password_hash="x",
+                            role=UserRole.admin)
+                s.add(user)
+                await s.flush()
+                fields = await alert_svc.rule_fields(
+                    s, name=f"back {tag}", source_module="host_agent", resource_name=res,
+                    metric_name="cpu_pct", operator=">", threshold=90, notify_no_data=True)
+                rule_id = (await alert_svc.add_rule(s, fields, created_by=user.id)).id
+                await s.flush()
+                st = (await s.execute(select(AlertState).where(AlertState.rule_id == rule_id))).scalar_one()
+                st.last_sample_at, st.sample_interval_s = now - timedelta(minutes=18), 60.0
+
+        async def marks():
+            async with app.db_sessionmaker() as s:
+                st = (await s.execute(select(AlertState).where(AlertState.rule_id == rule_id))).scalar_one()
+                n = len((await s.execute(select(AlertEvent).where(AlertEvent.rule_id == rule_id))).all())
+            return st.no_data_since, n
+
+        await alerts.check_silence(app)        # the first run since Steward started
+        await asyncio.sleep(0)
+        assert await marks() == (None, 0) and (rule_id, "no_data") not in sent
+
+        # Six quiet minutes later (as if), Steward has watched long enough.
+        w = watch(app, "alert_silence")
+        w.since = now - timedelta(minutes=6)
+        await alerts.check_silence(app)
+        await asyncio.sleep(0)
+        assert await marks() == (w.since + timedelta(seconds=300), 1)
+        assert sent.count((rule_id, "no_data")) == 1
+
+    with patch.object(alerts, "_dispatch_notification", fake_dispatch):
+        asyncio.run(_go())
+