fix(anomalies): a burst within what a metric reaches in most hours opens nothing (#6727)

#6727 "A metric that bursts now and then opens anomalies at hours whose usual
peak reads 0". usual_peak is the p90 of a slot's hourly max, so a metric that
bursts in under about 1 hour in 10 reads 0 there however often it bursts across
the day. #6723 "An anomaly opens on a burst of minutes inside the metric's usual
peak, so most rows aren't outside the norm" therefore still opened on those.

usual_anywhere is the median, across the hours of the day, of each hour's
highest peak. Height is now judged against the greater of it and the slot's
usual peak. Bursts at any hour lift the median; a job at a time of its own
raises a few hours and leaves it low, so the same level at another time of day
is still an anomaly, as System 182 rules. A held level still opens.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

steward/core/anomalies.py

@@ -4,10 +4,10 @@ Every minute the last WINDOW of each learned key is averaged and compared
 with the normal for the slot it falls in (steward.core.norms). A level above
 the slot's usual hold by DEPART times, and by at least the metric's floor, has
 departed. That opens an occurrence once it is also outside what the normal
-records in height or in time: higher than the slot's usual peak, or held, with
-most of the last HELD above the line. A burst of a few minutes within the
-usual peak lifts the average for a while and opens nothing. The occurrence ends
-when the level comes back under the line.
+records in height or in time: higher than the metric usually goes (Norm.
+usual_height), or held, with most of the last HELD above the line. A burst of
+a few minutes within that lifts the average for a while and opens nothing. The
+occurrence ends when the level comes back under the line.
 
 What the operator ruled (System 182 "Metric Norms and Anomalies"):
 - A departure is a warning, marked as a metric anomaly, and stays "to review"
@@ -32,7 +32,7 @@ from steward.timeline.record import anomaly_changed
 
 WINDOW = timedelta(minutes=15)   # the level judged is this window's average
 DEPART = 1.25                    # times the slot's usual hold that counts as a departure
-HELD = timedelta(hours=1)        # a level within the usual peak opens once the median of this is out
+HELD = timedelta(hours=1)        # a level within its usual height opens once the median of this is out
 # An occurrence that comes back within one window of ending continues the
 # same occurrence: a level wobbling around the line is one event, not ten.
 REJOIN = WINDOW
@@ -169,7 +169,7 @@ async def detect(session, now: datetime | None = None) -> dict:
     out_now = {key for key, level in levels.items()
                if key in norms and departs(level, norms[key], LEARNED[(key[0], key[2])].least_departure)}
     held = await held_levels(session, [key for key in out_now
-                                       if key not in open_now and levels[key] <= norms[key].usual_peak], now)
+                                       if key not in open_now and levels[key] <= norms[key].usual_height], now)
     for key, level in levels.items():
         norm = norms.get(key)
         out = key in out_now
@@ -182,7 +182,7 @@ async def detect(session, now: datetime | None = None) -> dict:
                 _close(anomaly, span, now)
                 changed.append(anomaly)
                 closed += 1
-        elif out and level > norm.usual_peak:
+        elif out and level > norm.usual_height:
             changed.append(await _occur(session, key, level, norm, now, WINDOW))
             opened += 1
         elif out and departs(held.get(key, 0.0), norm, LEARNED[(key[0], key[2])].least_departure):

steward/core/norms.py

@@ -11,7 +11,12 @@ What a slot's normal is:
   (plugin_metrics_hourly.value_p95, #5738), at its 90th percentile across
   past days. A two-minute spike doesn't raise it; an hour spent high does.
 - usual_peak: the same for each hour's max, so a page can say how high the
-  slot usually goes.
+  slot usually goes. Most hours of a metric that bursts now and then see no
+  burst, so this can read 0 for a slot such a metric bursts in as much as any.
+- usual_anywhere: how high the metric has gone in most hours of the day, the
+  median across the 24 hours of each one's highest peak. Bursts that land at
+  any hour raise it. A job at a time of its own raises a few hours and leaves
+  the median alone, so the same level at another time still stands out.
 
 The slots:
 - An hour of the day (24 slots) once a key has DAY_MIN_DAYS of history.
@@ -116,6 +121,13 @@ class Norm:
     usual_hold: float
     usual_peak: float
     days_seen: int
+    usual_anywhere: float = 0.0
+
+    @property
+    def usual_height(self) -> float:
+        """The highest this metric usually goes at this time: a level past it
+        is news however briefly it lasted."""
+        return max(self.usual_peak, self.usual_anywhere)
 
 
 def slot_of(at: datetime, kind: str) -> int:
@@ -171,6 +183,12 @@ kinds AS (
     SELECT s, r, m, days, CASE WHEN first <= :week_ready THEN 'week' ELSE 'day' END AS kind
     FROM keyed WHERE days >= CAST(:day_min_days AS int)
 ),
+anywhere AS (
+    SELECT s, r, m, percentile_cont(0.5) WITHIN GROUP (ORDER BY highest) AS peak
+    FROM (SELECT s, r, m, max(peak) AS highest FROM hours
+          GROUP BY s, r, m, EXTRACT(HOUR FROM b AT TIME ZONE 'UTC')) by_hour
+    GROUP BY s, r, m
+),
 slotted AS (
     SELECT h.s, h.r, h.m, k.kind, k.days, h.hold, h.peak,
            CASE k.kind
@@ -183,13 +201,15 @@ slotted AS (
     WHERE k.kind = 'week' OR h.b >= :day_since
 )
 INSERT INTO metric_norms (source_module, resource_name, metric_name, slot, kind,
-                          usual_hold, usual_peak, days_seen, updated_at)
-SELECT s, r, m, (slot + n.off + slots) % slots, kind,
-       percentile_cont(CAST(:usual AS float8)) WITHIN GROUP (ORDER BY hold),
-       percentile_cont(CAST(:usual AS float8)) WITHIN GROUP (ORDER BY peak),
-       days, :now
-FROM slotted CROSS JOIN generate_series(-CAST(:neighbours AS int), CAST(:neighbours AS int)) AS n(off)
-GROUP BY s, r, m, (slot + n.off + slots) % slots, kind, days
+                          usual_hold, usual_peak, usual_anywhere, days_seen, updated_at)
+SELECT l.s, l.r, l.m, (l.slot + n.off + l.slots) % l.slots, l.kind,
+       percentile_cont(CAST(:usual AS float8)) WITHIN GROUP (ORDER BY l.hold),
+       percentile_cont(CAST(:usual AS float8)) WITHIN GROUP (ORDER BY l.peak),
+       a.peak, l.days, :now
+FROM slotted l
+JOIN anywhere a ON a.s = l.s AND a.r = l.r AND a.m = l.m
+CROSS JOIN generate_series(-CAST(:neighbours AS int), CAST(:neighbours AS int)) AS n(off)
+GROUP BY l.s, l.r, l.m, (l.slot + n.off + l.slots) % l.slots, l.kind, l.days, a.peak
 """).bindparams(bindparam("keys", type_=ARRAY(String)),
                bindparam("starts", type_=ARRAY(DateTime(timezone=True))),
                bindparam("parts", type_=ARRAY(Boolean)))
@@ -235,7 +255,7 @@ async def norms_at(session, keys: list[tuple[str, str, str]],
                | ((MetricNorm.kind == "week") & (MetricNorm.slot == week)))
     )).scalars().all()
     return {(n.source_module, n.resource_name, n.metric_name):
-            Norm(n.kind, n.slot, n.usual_hold, n.usual_peak, n.days_seen) for n in rows}
+            Norm(n.kind, n.slot, n.usual_hold, n.usual_peak, n.days_seen, n.usual_anywhere) for n in rows}
 
 
 async def run_norms(app) -> None: