fix(alerts): a series that comes back gets one history event, not two at one instant (#5442)

When a silent series' sample also moved the rule (fired, resolved), the
resume and the transition were two events: the resume from the
pre-evaluation state at ingest time, the transition at the sample's
time. Ties sort by a random id, and the resume's "→ inactive" landed
after "→ firing", so the history read "silent, firing, back to normal"
at once and the episode closed while the alert was firing.

_track_samples now only clears the mark and returns its note; the first
sample's evaluation records one event from no_data to wherever that
sample took the rule ("firing" or "resolved" notification), or back to
the state it was in ("data_resumed", still only when notify_no_data).

Also fixes the #5441 integration test: its rule needs a recorded
resource to target (CI 8878).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

steward/core/alerts.py

@@ -249,19 +249,18 @@ async def record_metrics(session: AsyncSession, rows: list[PluginMetric], phases
         samples = sorted(by_key[(rule.source_module, rule.resource_name, rule.metric_name)],
                          key=lambda r: r.recorded_at)
         muted = _muted_by(windows, rule)
-        resumed = _track_samples(session, rule, alert_state, samples, now, muted)
-        if resumed and rule.notify_no_data and not muted:
-            event_id, value, note = resumed
-            _after_commit(session, _dispatch_notification, "data_resumed", rule, value,
-                          event_id, note)
+        resumed = _track_samples(alert_state, samples, now)
         for sample in samples:
             notifications = await _evaluate_rule(session, rule, alert_state, sample.value,
-                                                 sample.recorded_at, note=muted)
+                                                 sample.recorded_at, note=muted, resumed=resumed)
+            resumed = None  # said once, with the first sample back
             if muted:
                 continue
-            for notif_type, event_id in notifications:
+            for notif_type, event_id, note in notifications:
+                if notif_type == "data_resumed" and not rule.notify_no_data:
+                    continue
                 _after_commit(session, _dispatch_notification, notif_type, rule, sample.value,
-                              event_id)
+                              event_id, note)
                 # Admin-configured Ansible action runs only on transition-to-firing.
                 if notif_type == "firing" and rule.ansible_action:
                     _after_commit(session, _run_ansible_action, rule, sample.value, event_id)
@@ -315,13 +314,12 @@ def silent_since(last_sample_at: datetime | None, interval: float | None,
     return last + limit if now - last > limit else None
 
 
-def _track_samples(session: AsyncSession, rule: AlertRule, state: AlertState,
-                   samples: list[PluginMetric], now: datetime, muted: str | None = None
-                   ) -> tuple[str, float, str] | None:
+def _track_samples(state: AlertState, samples: list[PluginMetric], now: datetime) -> str | None:
     """Note a rule's new samples (oldest first) for silence tracking. When the
-    series had been marked no_data, clear it and record a resume event,
-    returned as (event_id, value, note) for the caller to notify. `muted` is
-    added to the event's note when a maintenance window holds it back."""
+    series had been marked no_data, clear the mark and return the note saying
+    so ("data resumed after 15m silent"); the first sample's evaluation
+    records it, in one event with whatever that sample does to the rule
+    (#5442)."""
     for sample in samples:
         if state.last_sample_at is not None:
             if sample.recorded_at <= state.last_sample_at:
@@ -332,11 +330,8 @@ def _track_samples(session: AsyncSession, rule: AlertRule, state: AlertState,
     if state.no_data_since is None or not samples:
         return None
     note = f"data resumed after {human_duration((now - state.no_data_since).total_seconds())} silent"
-    value = samples[-1].value
-    event_id = _add_event(session, rule.id, "no_data", state.state.value, value, now,
-                          note=f"{note}; {muted}" if muted else note)
     state.no_data_since = None
-    return event_id, value, note
+    return note
 
 
 def _is_breached(value: float, operator: AlertOperator, threshold: float) -> bool:
@@ -401,26 +396,38 @@ async def _evaluate_rule(
     value: float,
     now: datetime,
     note: str | None = None,
-) -> list[tuple[str, str]]:
-    """Apply one sample to a rule's state. Returns [(notif_type, event_id)];
-    an event it records carries `note`."""
+    resumed: str | None = None,
+) -> list[tuple[str, str, str | None]]:
+    """Apply one sample to a rule's state. Returns [(notif_type, event_id,
+    note)]; an event it records carries `note`.
+
+    `resumed` says this sample ends a silence. The rule then gets one event
+    from no_data: to the state the sample moved it to (a "firing"
+    notification), or back to the state it was in ("data_resumed"). Two
+    events at one instant used to sort either way round, so the history
+    could read "back to inactive" right after "firing" (#5442).
+    """
     from_state = state.state
     step = next_step(from_state, state.consecutive_failure_count, state.consecutive_ok_count,
                      _is_breached(value, rule.operator, rule.threshold),
                      rule.consecutive_failures_required, rule.consecutive_ok_required)
     state.consecutive_failure_count = step.fail_count
     state.consecutive_ok_count = step.ok_count
-    notifications: list[tuple[str, str]] = []
+    notifications: list[tuple[str, str, str | None]] = []
     if step.state != from_state:
         state.state = step.state
         state.last_transition_at = now
         if step.state == AlertStateEnum.inactive:
             state.acknowledged_by = None
             state.acknowledged_at = None
+    said = "; ".join(n for n in (resumed, note) if n) or None
     if step.event:
-        event_id = _add_event(session, rule.id, from_state.value, step.event, value, now,
-                              note=note)
-        notifications.append((step.event, event_id))
+        event_id = _add_event(session, rule.id, "no_data" if resumed else from_state.value,
+                              step.event, value, now, note=said)
+        notifications.append((step.event, event_id, None))
+    elif resumed:
+        event_id = _add_event(session, rule.id, "no_data", state.state.value, value, now, note=said)
+        notifications.append(("data_resumed", event_id, resumed))
     await session.flush()
     return notifications