feat(anomalies): a metric out of its normal is an anomaly to review (#5740)

#5740 "A metric that departs from its normal is recorded as an anomaly to
review, and is findable after it ends", milestone 513.

- Every minute each learned key's last 15 minutes is judged against its
  hour's normal: above the usual hold x1.25 and by the metric's floor opens
  an occurrence, which ends when the level comes back.
- metric_anomalies holds one row per item to review; a repeat while it
  still waits joins it as an occurrence (metric_anomaly_spans).
- An ended row nobody reviews within anomalies.review_days (14, Settings ->
  Thresholds) lapses; an ongoing one never does.
- Learning skips every hour an occurrence covers unless its anomaly was
  marked normal: no normal is learned without review.
- Alerts -> Anomalies lists those to review first, then the reviewed and
  lapsed under day dividers; marking is operator/admin and audited.
- The Overview widget and get_overview warn of open ones and count those
  to review; the host page shows its own; the weekly report lists the
  week's and the lapsed; MCP list_anomalies and review_anomaly.
- Reviewed and lapsed rows are pruned at data.retention_days, and kept
  in backups like alert history.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

steward/models/__init__.py

@@ -6,7 +6,7 @@ from .password_resets import PasswordReset
 from .hosts import Host
 from .monitors import Monitor, MonitorResult, MonitorType
 from .metrics import PluginMetric
-from .norms import MetricNorm
+from .norms import MetricNorm, MetricAnomaly, MetricAnomalySpan
 from .alerts import AlertRule, AlertState, AlertEvent, AlertOperator, AlertStateEnum
 from .ansible import AnsibleRun, AnsibleRunStatus
 from .ansible_inventory import AnsibleTarget, AnsibleGroup, ansible_target_groups
@@ -19,7 +19,7 @@ __all__ = [
     "Base", "User", "ApiToken", "UserInvite", "PasswordReset",
     "Host",
     "Monitor", "MonitorResult", "MonitorType",
-    "PluginMetric", "MetricNorm",
+    "PluginMetric", "MetricNorm", "MetricAnomaly", "MetricAnomalySpan",
     "AlertRule", "AlertState", "AlertEvent", "AlertOperator", "AlertStateEnum",
     "AnsibleRun", "AnsibleRunStatus",
     "AnsibleTarget", "AnsibleGroup", "ansible_target_groups",

steward/models/norms.py

@@ -1,8 +1,9 @@
 from __future__ import annotations
 from datetime import datetime
-from sqlalchemy import DateTime, Float, Integer, SmallInteger, String
+from sqlalchemy import DateTime, Float, ForeignKey, Index, Integer, SmallInteger, String, Text
 from sqlalchemy.orm import Mapped, mapped_column
 from .base import Base
+from steward.core.ids import time_ordered_id
 
 
 class MetricNorm(Base):
@@ -25,3 +26,55 @@ class MetricNorm(Base):
     usual_peak: Mapped[float] = mapped_column(Float, nullable=False)
     days_seen: Mapped[int] = mapped_column(Integer, nullable=False)
     updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
+
+
+class MetricAnomaly(Base):
+    """A metric that departed from its normal (#5740): one item to review.
+
+    A departure that recurs while the item is still to review joins it as
+    another occurrence (a MetricAnomalySpan) rather than adding a row, so the
+    list stays one row per resource and metric. `review` is to_review until
+    someone marks it normal (its hours are learned) or real (they are not);
+    an ended item nobody reviewed within anomalies.review_days becomes lapsed,
+    and is not learned either."""
+    __tablename__ = "metric_anomalies"
+
+    id: Mapped[str] = mapped_column(String(36), primary_key=True, default=time_ordered_id)
+    source_module: Mapped[str] = mapped_column(String(64), nullable=False)
+    resource_name: Mapped[str] = mapped_column(String(255), nullable=False)
+    metric_name: Mapped[str] = mapped_column(String(128), nullable=False)
+    # The first occurrence's start, the latest's start, and the latest's end:
+    # NULL while an occurrence is still going.
+    started_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
+    last_started_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
+    ended_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
+    occurrences: Mapped[int] = mapped_column(Integer, nullable=False, default=1)
+    # The highest 15-minute level any occurrence held, and the normal for the
+    # hour it was reached in.
+    peak_level: Mapped[float] = mapped_column(Float, nullable=False)
+    peak_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
+    usual_hold: Mapped[float] = mapped_column(Float, nullable=False)
+    usual_peak: Mapped[float] = mapped_column(Float, nullable=False)
+    review: Mapped[str] = mapped_column(String(12), nullable=False, default="to_review")
+    reviewed_by: Mapped[str | None] = mapped_column(String(64), nullable=True)
+    reviewed_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
+    note: Mapped[str | None] = mapped_column(Text, nullable=True)
+
+    __table_args__ = (
+        Index("ix_metric_anomalies_key", "source_module", "resource_name", "metric_name"),
+        Index("ix_metric_anomalies_review_last", "review", "last_started_at"),
+    )
+
+
+class MetricAnomalySpan(Base):
+    """One occurrence of an anomaly: when the metric was out of its normal.
+    Learning skips the hours a span covers unless its anomaly was marked
+    normal, and a resource's chart shades them."""
+    __tablename__ = "metric_anomaly_spans"
+
+    id: Mapped[str] = mapped_column(String(36), primary_key=True, default=time_ordered_id)
+    anomaly_id: Mapped[str] = mapped_column(
+        String(36), ForeignKey("metric_anomalies.id", ondelete="CASCADE"), nullable=False, index=True)
+    started_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
+    ended_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
+    peak_level: Mapped[float] = mapped_column(Float, nullable=False)