bvandeusenandClaude Opus 5.5
5569f9934e
#5740 "A metric that departs from its normal is recorded as an anomaly to review, and is findable after it ends", milestone 513. - Every minute each learned key's last 15 minutes is judged against its hour's normal: above the usual hold x1.25 and by the metric's floor opens an occurrence, which ends when the level comes back. - metric_anomalies holds one row per item to review; a repeat while it still waits joins it as an occurrence (metric_anomaly_spans). - An ended row nobody reviews within anomalies.review_days (14, Settings -> Thresholds) lapses; an ongoing one never does. - Learning skips every hour an occurrence covers unless its anomaly was marked normal: no normal is learned without review. - Alerts -> Anomalies lists those to review first, then the reviewed and lapsed under day dividers; marking is operator/admin and audited. - The Overview widget and get_overview warn of open ones and count those to review; the host page shows its own; the weekly report lists the week's and the lapsed; MCP list_anomalies and review_anomaly. - Reviewed and lapsed rows are pruned at data.retention_days, and kept in backups like alert history. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
81 lines
4.5 KiB
Python
81 lines
4.5 KiB
Python
from __future__ import annotations
|
||
from datetime import datetime
|
||
from sqlalchemy import DateTime, Float, ForeignKey, Index, Integer, SmallInteger, String, Text
|
||
from sqlalchemy.orm import Mapped, mapped_column
|
||
from .base import Base
|
||
from steward.core.ids import time_ordered_id
|
||
|
||
|
||
class MetricNorm(Base):
|
||
"""What one metric usually does in one time slot (#5739), learned from
|
||
its own history by steward.core.norms.rebuild_norms.
|
||
|
||
`kind` says what a slot is: "day" (0–23, the UTC hour of day) or "week"
|
||
(0–167, Monday 00:00 UTC first). A key has slots of one kind at a time.
|
||
The whole table is rebuilt each pass, so a row is never edited."""
|
||
__tablename__ = "metric_norms"
|
||
|
||
source_module: Mapped[str] = mapped_column(String(64), primary_key=True)
|
||
resource_name: Mapped[str] = mapped_column(String(255), primary_key=True)
|
||
metric_name: Mapped[str] = mapped_column(String(128), primary_key=True)
|
||
slot: Mapped[int] = mapped_column(SmallInteger, primary_key=True)
|
||
kind: Mapped[str] = mapped_column(String(8), nullable=False)
|
||
# The level the slot's hours usually held, and how high they usually
|
||
# peaked: the 90th percentile, across past days, of each hour's p95 and max.
|
||
usual_hold: Mapped[float] = mapped_column(Float, nullable=False)
|
||
usual_peak: Mapped[float] = mapped_column(Float, nullable=False)
|
||
days_seen: Mapped[int] = mapped_column(Integer, nullable=False)
|
||
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
|
||
|
||
|
||
class MetricAnomaly(Base):
|
||
"""A metric that departed from its normal (#5740): one item to review.
|
||
|
||
A departure that recurs while the item is still to review joins it as
|
||
another occurrence (a MetricAnomalySpan) rather than adding a row, so the
|
||
list stays one row per resource and metric. `review` is to_review until
|
||
someone marks it normal (its hours are learned) or real (they are not);
|
||
an ended item nobody reviewed within anomalies.review_days becomes lapsed,
|
||
and is not learned either."""
|
||
__tablename__ = "metric_anomalies"
|
||
|
||
id: Mapped[str] = mapped_column(String(36), primary_key=True, default=time_ordered_id)
|
||
source_module: Mapped[str] = mapped_column(String(64), nullable=False)
|
||
resource_name: Mapped[str] = mapped_column(String(255), nullable=False)
|
||
metric_name: Mapped[str] = mapped_column(String(128), nullable=False)
|
||
# The first occurrence's start, the latest's start, and the latest's end:
|
||
# NULL while an occurrence is still going.
|
||
started_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
|
||
last_started_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
|
||
ended_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
|
||
occurrences: Mapped[int] = mapped_column(Integer, nullable=False, default=1)
|
||
# The highest 15-minute level any occurrence held, and the normal for the
|
||
# hour it was reached in.
|
||
peak_level: Mapped[float] = mapped_column(Float, nullable=False)
|
||
peak_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
|
||
usual_hold: Mapped[float] = mapped_column(Float, nullable=False)
|
||
usual_peak: Mapped[float] = mapped_column(Float, nullable=False)
|
||
review: Mapped[str] = mapped_column(String(12), nullable=False, default="to_review")
|
||
reviewed_by: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
||
reviewed_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
|
||
note: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||
|
||
__table_args__ = (
|
||
Index("ix_metric_anomalies_key", "source_module", "resource_name", "metric_name"),
|
||
Index("ix_metric_anomalies_review_last", "review", "last_started_at"),
|
||
)
|
||
|
||
|
||
class MetricAnomalySpan(Base):
|
||
"""One occurrence of an anomaly: when the metric was out of its normal.
|
||
Learning skips the hours a span covers unless its anomaly was marked
|
||
normal, and a resource's chart shades them."""
|
||
__tablename__ = "metric_anomaly_spans"
|
||
|
||
id: Mapped[str] = mapped_column(String(36), primary_key=True, default=time_ordered_id)
|
||
anomaly_id: Mapped[str] = mapped_column(
|
||
String(36), ForeignKey("metric_anomalies.id", ondelete="CASCADE"), nullable=False, index=True)
|
||
started_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
|
||
ended_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
|
||
peak_level: Mapped[float] = mapped_column(Float, nullable=False)
|