feat(metrics): each rolled-up hour keeps the level it held, its p95 (#5738)

plugin_metrics_hourly gains value_p95 beside value_avg and value_max. A
two-minute backup spike raises the hour's peak and leaves its p95 low; an
hour spent at that level raises both. Learned normals (milestone 513) need
that difference to call a recurring spike usual without calling the same
level held at another hour usual too. Hours rolled up before this keep a
NULL p95 and read as their average.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

tests/integration/test_hourly_p95.py

@@ -0,0 +1,62 @@
+"""Integration: each rolled-up hour keeps the level it held (#5738).
+
+A learned normal has to tell a two-minute backup spike from an hour spent at
+that level. Both have the same peak; only the hour's 95th percentile differs,
+so the rollup must keep it beside the average and the peak.
+Requires STEWARD_DATABASE_URL.
+"""
+from __future__ import annotations
+
+import asyncio
+import os
+import uuid
+from datetime import datetime, timedelta, timezone
+
+import pytest
+
+pytestmark = pytest.mark.integration
+
+
+@pytest.fixture
+def app():
+    if not os.environ.get("STEWARD_DATABASE_URL"):
+        pytest.skip("needs Postgres")
+    from steward.app import create_app
+    return create_app(testing=False)
+
+
+def test_a_brief_spike_raises_the_peak_and_a_held_level_raises_the_p95(app):
+    from sqlalchemy import select
+    from steward.core.metrics_retention import rollup_plugin_metrics
+    from steward.models.metrics import PluginMetric, PluginMetricHourly
+
+    now = datetime.now(timezone.utc)
+    res = f"p95-{uuid.uuid4().hex[:8]}"
+    spike_hour = (now - timedelta(days=10)).replace(minute=0, second=0, microsecond=0)
+    held_hour = spike_hour + timedelta(hours=1)
+
+    def minute(hour, m, metric, value):
+        return PluginMetric(source_module="test", resource_name=res, metric_name=metric,
+                            value=value, recorded_at=hour + timedelta(minutes=m))
+
+    async def _go():
+        async with app.db_sessionmaker() as s:
+            async with s.begin():
+                # One sample a minute. The spike hour sits at 10 and touches 95
+                # for two minutes; the held hour sits at 90 bar a two-minute dip.
+                s.add_all([minute(spike_hour, m, "cpu_pct", 95.0 if m in (2, 3) else 10.0)
+                           for m in range(60)])
+                s.add_all([minute(held_hour, m, "cpu_pct", 10.0 if m in (2, 3) else 90.0)
+                           for m in range(60)])
+            async with s.begin():
+                await rollup_plugin_metrics(s, raw_days=7, rollup_days=90, now=now)
+            rows = (await s.execute(
+                select(PluginMetricHourly)
+                .where(PluginMetricHourly.resource_name == res)
+                .order_by(PluginMetricHourly.bucket))).scalars().all()
+            return [(r.bucket, r.value_max, r.value_p95, r.value_avg) for r in rows]
+
+    (b1, spike_max, spike_p95, _), (b2, held_max, held_p95, _) = asyncio.run(_go())
+    assert (b1, b2) == (spike_hour, held_hour)
+    assert spike_max == 95.0 and abs(spike_p95 - 10.0) < 0.001
+    assert held_max == 90.0 and abs(held_p95 - 90.0) < 0.001