fix(alerts): a series that comes back gets one history event, not two at one instant (#5442)

When a silent series' sample also moved the rule (fired, resolved), the
resume and the transition were two events: the resume from the
pre-evaluation state at ingest time, the transition at the sample's
time. Ties sort by a random id, and the resume's "→ inactive" landed
after "→ firing", so the history read "silent, firing, back to normal"
at once and the episode closed while the alert was firing.

_track_samples now only clears the mark and returns its note; the first
sample's evaluation records one event from no_data to wherever that
sample took the rule ("firing" or "resolved" notification), or back to
the state it was in ("data_resumed", still only when notify_no_data).

Also fixes the #5441 integration test: its rule needs a recorded
resource to target (CI 8878).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

tests/core/test_alert_no_data.py

@@ -2,15 +2,17 @@
 samples, an outage does not become its cadence, and a series counts as silent
 after three usual gaps (at least five minutes; an hour while its cadence is
 unknown)."""
+import asyncio
 from datetime import datetime, timedelta, timezone
 from types import SimpleNamespace as NS
 
 import pytest
 
 from steward.core.alerts import (
-    NO_DATA_FLOOR_S, NO_DATA_UNKNOWN_S, _track_samples, next_interval, no_data_after, silent_since,
+    NO_DATA_FLOOR_S, NO_DATA_UNKNOWN_S, _evaluate_rule, _track_samples, next_interval,
+    no_data_after, silent_since,
 )
-from steward.models.alerts import AlertStateEnum
+from steward.models.alerts import AlertOperator, AlertStateEnum
 
 NOW = datetime(2026, 10, 6, 12, 0, tzinfo=timezone.utc)
 
@@ -80,10 +82,13 @@ class _Session:
     def add(self, row):
         self.added.append(row)
 
+    async def flush(self):
+        pass
+
 
 def _state(**kw):
     base = dict(state=AlertStateEnum.firing, last_sample_at=None, sample_interval_s=None,
-                no_data_since=None)
+                no_data_since=None, consecutive_failure_count=0, consecutive_ok_count=0)
     return NS(**{**base, **kw})
 
 
@@ -92,23 +97,49 @@ def _samples(*ago_values):
 
 
 def test_tracking_learns_the_gap_and_skips_late_samples():
-    st, session = _state(), _Session()
-    assert _track_samples(session, NS(id="r"), st, _samples((180, 1), (120, 1), (60, 1)), NOW) is None
+    st = _state()
+    assert _track_samples(st, _samples((180, 1), (120, 1), (60, 1)), NOW) is None
     assert (st.last_sample_at, st.sample_interval_s) == (NOW - timedelta(seconds=60), 60)
-    _track_samples(session, NS(id="r"), st, _samples((90, 1)), NOW)  # older than the last
+    _track_samples(st, _samples((90, 1)), NOW)  # older than the last
     assert (st.last_sample_at, st.sample_interval_s) == (NOW - timedelta(seconds=60), 60)
-    assert session.added == []
 
 
-def test_a_sample_after_silence_clears_it_and_records_the_resume():
-    st, session = _state(last_sample_at=NOW - timedelta(minutes=20), sample_interval_s=60,
-                         no_data_since=NOW - timedelta(minutes=15)), _Session()
-    event_id, value, note = _track_samples(session, NS(id="r"), st, _samples((0, 42.0)), NOW)
+def test_a_sample_after_silence_clears_it_and_says_so():
+    st = _state(last_sample_at=NOW - timedelta(minutes=20), sample_interval_s=60,
+                no_data_since=NOW - timedelta(minutes=15))
+    note = _track_samples(st, _samples((0, 42.0)), NOW)
     assert st.no_data_since is None and st.state == AlertStateEnum.firing  # still firing
     assert st.sample_interval_s == 90       # the outage counted as three gaps, at most
-    assert value == 42.0 and note == "data resumed after 15m silent"
+    assert note == "data resumed after 15m silent"
+
+
+_RULE = NS(id="r", operator=AlertOperator.gt, threshold=90.0,
+           consecutive_failures_required=1, consecutive_ok_required=1)
+
+
+@pytest.mark.parametrize("before,value,want_event,want_notice", [
+    # Back, and breaching: one event, silent to firing (#5442).
+    (AlertStateEnum.inactive, 95.0, ("no_data", "firing"), "firing"),
+    # Back, and fine: one event, silent back to how it was.
+    (AlertStateEnum.inactive, 10.0, ("no_data", "inactive"), "data_resumed"),
+    # A firing alert that went silent comes back fine: it resolves, from silent.
+    (AlertStateEnum.firing, 10.0, ("no_data", "resolved"), "resolved"),
+    # ... or still breaching: it stays firing, and only the resume is news.
+    (AlertStateEnum.firing, 95.0, ("no_data", "firing"), "data_resumed"),
+])
+def test_a_resumed_series_gets_one_event_for_what_its_first_sample_did(
+        before, value, want_event, want_notice):
+    session, st = _Session(), _state(state=before)
+    said = "data resumed after 15m silent"
+    out = asyncio.run(_evaluate_rule(session, _RULE, st, value, NOW, resumed=said))
     [event] = session.added
-    assert (event.id, event.from_state, event.to_state) == (event_id, "no_data", "firing")
+    assert (event.from_state, event.to_state) == want_event
+    assert event.note == said and event.transitioned_at == NOW
+    assert [(n, e) for n, e, _ in out] == [(want_notice, event.id)]
+    # Without a resume, a sample that changes nothing records nothing.
+    session = _Session()
+    assert asyncio.run(_evaluate_rule(session, _RULE, _state(state=st.state), value, NOW)) == []
+    assert session.added == []
 
 
 def test_muted_by_names_the_window_that_covers_the_rule():

tests/integration/test_alert_no_data.py

@@ -104,13 +104,14 @@ def test_silence_is_marked_once_and_cleared_by_data(app):
         await asyncio.sleep(0)
         st = await state("loud")
         assert st.no_data_since is None and st.state == AlertStateEnum.firing
-        first, *same_instant = await events("loud")
+        # The resume and the firing it leads to are one event, silent to
+        # firing: two events at one instant used to sort either way (#5442).
+        first, back = await events("loud")
         assert first[:2] == ("inactive", "no_data")
-        # The resume and the firing it leads to share the sample's timestamp.
-        assert sorted(e[:2] for e in same_instant) == [("inactive", "firing"),
-                                                       ("no_data", "inactive")]
-        assert any((e[2] or "").startswith("data resumed after ") for e in same_instant)
-        assert [t for _, t, _ in sent if t != "firing"] == ["no_data", "data_resumed"]
+        assert back[:2] == ("no_data", "firing") and back[2].startswith("data resumed after ")
+        assert [t for n, t, _ in sent if n == f"loud {tag}"] == ["no_data", "firing"]
+        # The hushed rule says nothing of silence, but its firing is still news.
+        assert [t for n, t, _ in sent if n == f"hushed {tag}"] == ["firing"]
 
         # A firing alert that goes silent stays firing, marked.
         async with app.db_sessionmaker() as s:
@@ -123,7 +124,7 @@ def test_silence_is_marked_once_and_cleared_by_data(app):
         st = await state("loud")
         assert st.state == AlertStateEnum.firing and st.no_data_since is not None
         assert (await events("loud"))[-1][:2] == ("firing", "no_data")
-        assert len(await events("hushed")) == 3  # disabled: not watched
+        assert len(await events("hushed")) == 2  # disabled: not watched
 
     with patch.object(alerts, "_dispatch_notification", fake_dispatch):
         asyncio.run(_go())
@@ -138,6 +139,7 @@ def test_a_series_quiet_only_while_steward_was_down_is_not_silent(app):
     from steward.core import alerts
     from steward.core.watch import watch
     from steward.models.alerts import AlertEvent, AlertState
+    from steward.models.metrics import PluginMetric
     from steward.models.users import User, UserRole
 
     sent = []
@@ -154,6 +156,9 @@ def test_a_series_quiet_only_while_steward_was_down_is_not_silent(app):
                 user = User(username=f"bk-{tag}", email=f"bk-{tag}@test", password_hash="x",
                             role=UserRole.admin)
                 s.add(user)
+                s.add(PluginMetric(source_module="host_agent", resource_name=res,
+                                   metric_name="cpu_pct", value=1.0,
+                                   recorded_at=now - timedelta(minutes=18)))
                 await s.flush()
                 fields = await alert_svc.rule_fields(
                     s, name=f"back {tag}", source_module="host_agent", resource_name=res,