feat(metrics): each rolled-up hour keeps the level it held, its p95 (#5738)

plugin_metrics_hourly gains value_p95 beside value_avg and value_max. A
two-minute backup spike raises the hour's peak and leaves its p95 low; an
hour spent at that level raises both. Learned normals (milestone 513) need
that difference to call a recurring spike usual without calling the same
level held at another hour usual too. Hours rolled up before this keep a
NULL p95 and read as their average.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

steward/core/metrics_retention.py

@@ -40,7 +40,7 @@ async def rollup_plugin_metrics(
     """Roll up + prune plugin_metrics. Returns a counts dict for logging.
 
     1. Aggregate plugin_metrics older than the (hour-aligned) raw window into
-       plugin_metrics_hourly (avg/max per source/resource/metric/hour), upserting
+       plugin_metrics_hourly (avg/max/p95 per source/resource/metric/hour), upserting
        so a re-run is idempotent, then delete those raw rows.
     2. Prune rolled-up rows older than the rollup window.
     """
@@ -67,6 +67,7 @@ async def rollup_plugin_metrics(
             hour.label("bucket"),
             func.avg(PluginMetric.value).label("value_avg"),
             func.max(PluginMetric.value).label("value_max"),
+            func.percentile_cont(0.95).within_group(PluginMetric.value).label("value_p95"),
             func.count().label("sample_count"),
         )
         .where(PluginMetric.recorded_at < raw_cutoff)
@@ -78,17 +79,19 @@ async def rollup_plugin_metrics(
     for r in (await session.execute(agg)).all():
         avg_v = float(r.value_avg or 0.0)
         max_v = float(r.value_max or 0.0)
+        p95_v = float(r.value_p95) if r.value_p95 is not None else avg_v
         cnt = int(r.sample_count or 0)
         await session.execute(
             pg_insert(PluginMetricHourly)
             .values(
                 source_module=r.source_module, resource_name=r.resource_name,
                 metric_name=r.metric_name, bucket=r.bucket,
-                value_avg=avg_v, value_max=max_v, sample_count=cnt,
+                value_avg=avg_v, value_max=max_v, value_p95=p95_v, sample_count=cnt,
             )
             .on_conflict_do_update(
                 constraint="uq_plugin_metrics_hourly_bucket",
-                set_={"value_avg": avg_v, "value_max": max_v, "sample_count": cnt},
+                set_={"value_avg": avg_v, "value_max": max_v, "value_p95": p95_v,
+                      "sample_count": cnt},
             )
         )
         rolled += 1

steward/migrations/versions/0044_hourly_p95.py

@@ -0,0 +1,26 @@
+"""Each hour of the metric rollup keeps the level it held, its 95th percentile,
+beside its average and peak (#5738), so a learned normal can tell a brief
+spike (a high peak, a low p95) from a level the metric stayed at.
+
+Revision ID: 0044_hourly_p95
+Revises: 0043_outbox_alert_words
+Create Date: 2026-10-14
+"""
+from typing import Sequence, Union
+from alembic import op
+import sqlalchemy as sa
+
+revision: str = "0044_hourly_p95"
+down_revision: Union[str, None] = "0043_outbox_alert_words"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    # Hours rolled up before this have no p95 to recover (their raw samples
+    # are gone), so they stay NULL and read as their average.
+    op.add_column("plugin_metrics_hourly", sa.Column("value_p95", sa.Float, nullable=True))
+
+
+def downgrade() -> None:
+    op.drop_column("plugin_metrics_hourly", "value_p95")

steward/models/metrics.py

@@ -70,6 +70,10 @@ class PluginMetricHourly(Base):
     bucket: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
     value_avg: Mapped[float] = mapped_column(Float, nullable=False, default=0.0)
     value_max: Mapped[float] = mapped_column(Float, nullable=False, default=0.0)
+    # The level the hour held for all but its briefest 5% (#5738): a spike of
+    # a minute or two raises value_max and leaves this low. NULL on hours
+    # rolled up before it was kept; read those as value_avg.
+    value_p95: Mapped[float | None] = mapped_column(Float, nullable=True)
     sample_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
 
     __table_args__ = (