feat(mcp): understand and judge a metric anomaly: what happened, what was usual, what else went on (#5837)

get_anomaly gives each occurrence against what is usual at that hour, the
metric around the newest one with its normal, what else happened on the
resource's host around each occurrence (alert transitions, Ansible runs,
other anomalies there and a count elsewhere, plus plugins' own events
through a new mcp_events hook: the host agent's maintenance jobs and
downtime, Docker's container starts, deaths and OOM kills), and how earlier
anomalies on the same metric were judged.

get_metric_normal reads the learned slots and the level that would be
flagged. review_anomaly takes one id or several, all or nothing, with a
note. get_anomaly_settings and (admin) set_anomaly_settings read and change
the review window and each normal's floor, as Settings → Thresholds does.

metric_series.points takes an optional `until`; core.alerts gains
host_of_resource; floor_setting names its floor.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

steward/core/alerts.py

@@ -84,13 +84,19 @@ def is_heartbeat(rule) -> bool:
         _HEARTBEAT.source_module, _HEARTBEAT.metric_name)
 
 
+def host_of_resource(source_module: str, resource: str) -> str | None:
+    """The host whose agent carries this series, or None for a source no
+    agent feeds (a monitor, a Traefik router)."""
+    host_of = _AGENT_FED.get(source_module)
+    return host_of(resource) if host_of is not None else None
+
+
 def host_behind(rule) -> str | None:
     """The host whose agent feeds `rule`, or None: a rule on another source,
     or the heartbeat itself (agent offline is the one that speaks)."""
-    host_of = _AGENT_FED.get(rule.source_module)
-    if host_of is None or is_heartbeat(rule):
+    if is_heartbeat(rule):
         return None
-    return host_of(rule.resource_name)
+    return host_of_resource(rule.source_module, rule.resource_name)
 
 
 async def offline_agents(session) -> set[str]:

steward/core/metric_series.py

@@ -143,8 +143,9 @@ async def backfill_latest(sessionmaker, log) -> int:
 
 
 async def points(db, source_module: str, resource: str, metric: str, since: datetime,
-                 *, target: int = 120) -> list[tuple[datetime, float]]:
-    """(time, average) points for one metric since `since`, oldest first.
+                 *, target: int = 120, until: datetime | None = None) -> list[tuple[datetime, float]]:
+    """(time, average) points for one metric since `since` (up to `until`,
+    else now), oldest first.
 
     The part of the range older than the raw window comes from the hourly
     rollup; the rest is bucket-averaged in SQL (buckets no wider than an hour
@@ -167,14 +168,17 @@ async def points(db, source_module: str, resource: str, metric: str, since: date
                 PluginMetricHourly.resource_name == resource,
                 PluginMetricHourly.metric_name == metric,
                 PluginMetricHourly.bucket >= since,
-                PluginMetricHourly.bucket < raw_cutoff,
+                PluginMetricHourly.bucket < (min(raw_cutoff, until) if until else raw_cutoff),
             )
             .order_by(PluginMetricHourly.bucket)
         )
         out.extend((b, float(v)) for b, v in hourly)
 
     raw_since = max(since, raw_cutoff)
-    width_s = bucket_seconds(raw_since, target)
+    if until is not None and raw_since >= until:
+        return out
+    width_s = (bucket_seconds(raw_since, target) if until is None
+               else max(60, int((until - raw_since).total_seconds() / target)))
     if since < raw_cutoff:
         width_s = min(width_s, 3600)
     bucket = func.date_bin(
@@ -189,6 +193,7 @@ async def points(db, source_module: str, resource: str, metric: str, since: date
             PluginMetric.resource_name == resource,
             PluginMetric.metric_name == metric,
             PluginMetric.recorded_at >= raw_since,
+            *((PluginMetric.recorded_at < until,) if until is not None else ()),
         )
         .group_by(bucket)
         .order_by(bucket)

steward/core/norms.py

@@ -72,6 +72,11 @@ class Learned:
     def least_departure(self) -> float:
         return float(self.floor() if callable(self.floor) else self.floor)
 
+    @property
+    def floor_name(self) -> str | None:
+        """The NORM_FLOORS setting the floor is read from, if it is one."""
+        return getattr(self.floor, "floor_name", None)
+
 
 LEARNED: dict[tuple[str, str], Learned] = {}
 
@@ -87,9 +92,14 @@ def learn_normal(source_module: str, metric_name: str, floor: float | Callable[[
 
 def floor_setting(app, name: str) -> Callable[[], float]:
     """A floor read from Settings → Thresholds (norms.floor.<name>) at each
-    pass, so a saved change applies at once."""
+    pass, so a saved change applies at once. The function carries the
+    floor's name (`floor_name`), so a reader can say which setting it is."""
     from steward.core.settings import floor_of
-    return lambda: floor_of(app.config.get("THRESHOLDS", {}), name)
+
+    def read() -> float:
+        return floor_of(app.config.get("THRESHOLDS", {}), name)
+    read.floor_name = name
+    return read
 
 
 @dataclass(frozen=True)