feat(anomalies): a metric out of its normal is an anomaly to review (#5740)

#5740 "A metric that departs from its normal is recorded as an anomaly to
review, and is findable after it ends", milestone 513.

- Every minute each learned key's last 15 minutes is judged against its
  hour's normal: above the usual hold x1.25 and by the metric's floor opens
  an occurrence, which ends when the level comes back.
- metric_anomalies holds one row per item to review; a repeat while it
  still waits joins it as an occurrence (metric_anomaly_spans).
- An ended row nobody reviews within anomalies.review_days (14, Settings ->
  Thresholds) lapses; an ongoing one never does.
- Learning skips every hour an occurrence covers unless its anomaly was
  marked normal: no normal is learned without review.
- Alerts -> Anomalies lists those to review first, then the reviewed and
  lapsed under day dividers; marking is operator/admin and audited.
- The Overview widget and get_overview warn of open ones and count those
  to review; the host page shows its own; the weekly report lists the
  week's and the lapsed; MCP list_anomalies and review_anomaly.
- Reviewed and lapsed rows are pruned at data.retention_days, and kept
  in backups like alert history.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

steward/mcp/tools/__init__.py

@@ -12,9 +12,9 @@ each argument's empty value means, and what comes back.
 """
 from __future__ import annotations
 
-from . import alerts, audit, automation, hosts, metrics, monitors, overview, session
+from . import alerts, anomalies, audit, automation, hosts, metrics, monitors, overview, session
 
-_MODULES = (session, overview, hosts, monitors, alerts, metrics, audit, automation)
+_MODULES = (session, overview, hosts, monitors, alerts, anomalies, metrics, audit, automation)
 
 
 def register_all(register_tool) -> None:

steward/mcp/tools/anomalies.py

@@ -0,0 +1,96 @@
+"""Metric anomaly tools (#5740): metrics that left what is normal for them at
+that hour, and marking each normal or real."""
+from __future__ import annotations
+
+from steward.core import anomalies as anom
+from steward.mcp._context import current_caller
+from steward.models.users import UserRole
+
+from ._audit import audit_mcp, mcp_username
+from ._shape import bounded, clamp_limit, db_session, iso, rnd
+
+LIST_MAX = 200
+
+
+def _row(v: anom.AnomalyView) -> dict:
+    a, last = v.anomaly, v.last_span
+    return {
+        "anomaly_id": a.id,
+        "source_module": a.source_module,
+        "resource": a.resource_name,
+        "metric": a.metric_name,
+        "review": a.review,
+        "ongoing": v.ongoing,
+        "started_at": iso(a.started_at),
+        "ended_at": iso(a.ended_at),
+        "occurrences": a.occurrences,
+        "last_occurrence": {"started_at": iso(last.started_at), "ended_at": iso(last.ended_at),
+                            "peak_level": rnd(last.peak_level, 3)} if last else None,
+        "peak_level": rnd(a.peak_level, 3),
+        "peak_at": iso(a.peak_at),
+        "usual_hold": rnd(a.usual_hold, 3),
+        "usual_peak": rnd(a.usual_peak, 3),
+        "reviewed_by": a.reviewed_by,
+        "reviewed_at": iso(a.reviewed_at),
+        "note": a.note,
+        "page": v.href,
+    }
+
+
+async def list_anomalies(review: str = "to_review", limit: int = 50) -> dict:
+    """Metric anomalies: a learned metric whose 15-minute level went well
+    above what is normal for it at that hour (its usual hold ×1.25, and by
+    more than the metric's floor).
+
+    review: "to_review" (default) — waiting for someone to mark them; "normal"
+    — accepted, their hours are learned; "real" — a real problem, not learned;
+    "lapsed" — ended and nobody reviewed them within the review window
+    (Settings → Thresholds), so they are not learned; "" for all of them.
+
+    One row covers every repeat on a resource and metric while it waits for
+    review: `occurrences` counts them and `last_occurrence` is the latest.
+    `peak_level` is the highest 15-minute level any occurrence held, against
+    `usual_hold` (the level that hour usually holds) and `usual_peak` (how high
+    it usually goes), all in the metric's own unit. `ongoing` is true while an
+    occurrence is still going. To-review rows come first, newest occurrence
+    first. review_anomaly marks one.
+    """
+    if review and review not in anom.REVIEWS:
+        raise ValueError(f"review must be one of {', '.join(anom.REVIEWS)}, or empty for all")
+    limit = clamp_limit(limit, 50, LIST_MAX)
+    async with db_session() as db:
+        rows = await anom.list_anomalies(db, (review,) if review else None, limit=LIST_MAX)
+        views = await anom.views(db, rows)
+        to_review = await anom.to_review_count(db)
+    out = bounded([_row(v) for v in views], limit,
+                  key="anomalies", more="Pass a larger `limit`, or a `review` state.")
+    out["to_review"] = to_review
+    return out
+
+
+async def review_anomaly(anomaly_id: str, review: str, note: str = "") -> dict:
+    """Mark a metric anomaly "normal" or "real" (list_anomalies gives ids).
+
+    normal: this level is usual for this resource at that time; its hours are
+    learned into the metric's normal, so it stops being flagged once learned.
+    real: it was a real problem; its hours stay out of what is learned.
+    note: optional, kept with the review (say what it was).
+    A lapsed anomaly may still be marked. Audited as "<user> via MCP".
+    """
+    caller = current_caller()
+    async with db_session() as db, db.begin():
+        try:
+            a = await anom.mark(db, anomaly_id, review, mcp_username(caller.username), note)
+        except LookupError:
+            raise ValueError(f"No anomaly {anomaly_id!r}; list_anomalies shows them.") from None
+        detail = {"source": a.source_module, "resource": a.resource_name,
+                  "metric": a.metric_name, "note": a.note}
+        result = {"anomaly_id": a.id, "review": a.review, "reviewed_by": a.reviewed_by,
+                  "reviewed_at": iso(a.reviewed_at), "note": a.note}
+    await audit_mcp(f"metric_anomaly.marked_{review}", "metric_anomaly", anomaly_id, detail)
+    return result
+
+
+def register(register_tool) -> None:
+    register_tool(list_anomalies, role=UserRole.viewer, read_only=True)
+    register_tool(review_anomaly, role=UserRole.operator, read_only=False)

steward/mcp/tools/overview.py

@@ -7,6 +7,7 @@ from datetime import timedelta
 from sqlalchemy import select
 
 from steward.alerts import service as alert_svc
+from steward.core import anomalies as anom
 from steward.hosts import service as host_svc
 from steward.mcp._context import mcp_app
 from steward.models.ansible import AnsibleRun, AnsibleRunStatus
@@ -140,6 +141,22 @@ def run_findings(runs) -> list[dict]:
     return out
 
 
+def anomaly_findings(open_now) -> list[dict]:
+    """Metric anomalies going on now (#5740): a warning each."""
+    from steward.core.formatting import metric_label, metric_value
+    out = []
+    for a in open_now:
+        out.append(finding(
+            "warning", "metric_anomaly", f"{a.resource_name} {metric_label(a.metric_name)}",
+            f"Metric anomaly: {metric_label(a.metric_name)} on {a.resource_name} held "
+            f"{metric_value(a.metric_name, a.peak_level)} where "
+            f"{metric_value(a.metric_name, a.usual_hold)} is usual for that hour"
+            + (f" ({a.occurrences} times since {iso(a.started_at)})" if a.occurrences > 1 else ""),
+            since=a.last_started_at, ids={"anomaly_id": a.id},
+            next=f'list_anomalies(review="{a.review}")'))
+    return out
+
+
 async def _plugin_findings(server) -> tuple[list[dict], list[str]]:
     found, unavailable = [], []
     for plugin, source in sorted((getattr(server, "overview_sources", None) or {}).items()):
@@ -162,12 +179,15 @@ async def get_overview() -> dict:
     still-breaching acknowledged alerts, TLS certificates near expiry, host
     agents that stopped reporting, hosts over their CPU, memory or disk
     limits (all cutoffs are set under Settings → Thresholds), Ansible runs that failed
-    in the last 24h, and anything enabled plugins add (unhealthy containers,
-    for one). Each finding carries the ids to follow and `next`, the call
-    that shows its evidence.
+    in the last 24h, metrics going on now well above what is normal for them
+    at this hour (metric anomalies), and anything enabled plugins add
+    (unhealthy containers, for one). Each finding carries the ids to follow
+    and `next`, the call that shows its evidence.
 
-    `counts` sizes the fleet, and `active_maintenance` names windows
-    currently silencing alerts. `all_clear` is true when nothing is wrong.
+    `counts` sizes the fleet; `counts.anomalies_to_review` is how many metric
+    anomalies wait for someone to mark them normal or real (list_anomalies).
+    `active_maintenance` names windows currently silencing alerts.
+    `all_clear` is true when nothing is wrong.
     """
     app = mcp_app()
     now = now_utc()
@@ -187,12 +207,15 @@ async def get_overview() -> dict:
             .limit(20)
         )).scalars())
         windows = await alert_svc.list_maintenance_windows(db)
+        open_anomalies = await anom.open_anomalies(db)
+        to_review = await anom.to_review_count(db)
 
     thresholds = app.config.get("THRESHOLDS") or {}
     findings = (monitor_findings(monitors, states, host_names, now, thresholds.get("tls_days"))
                 + alert_findings(active)
                 + agent_findings(hosts, agent, thresholds)
-                + run_findings(runs))
+                + run_findings(runs)
+                + anomaly_findings(open_anomalies))
     plugin_found, unavailable = await _plugin_findings(getattr(app, "mcp_server", None))
     findings = rank(findings + plugin_found)
 
@@ -208,6 +231,7 @@ async def get_overview() -> dict:
             "alerts_active": len(active),
             "critical": sum(1 for f in findings if f["severity"] == "critical"),
             "warning": sum(1 for f in findings if f["severity"] == "warning"),
+            "anomalies_to_review": to_review,
         },
         "active_maintenance": [
             {"window_id": w.id, "name": w.name, "end_at": iso(w.end_at),