bvandeusenandClaude Opus 5.5
5569f9934e
#5740 "A metric that departs from its normal is recorded as an anomaly to review, and is findable after it ends", milestone 513. - Every minute each learned key's last 15 minutes is judged against its hour's normal: above the usual hold x1.25 and by the metric's floor opens an occurrence, which ends when the level comes back. - metric_anomalies holds one row per item to review; a repeat while it still waits joins it as an occurrence (metric_anomaly_spans). - An ended row nobody reviews within anomalies.review_days (14, Settings -> Thresholds) lapses; an ongoing one never does. - Learning skips every hour an occurrence covers unless its anomaly was marked normal: no normal is learned without review. - Alerts -> Anomalies lists those to review first, then the reviewed and lapsed under day dividers; marking is operator/admin and audited. - The Overview widget and get_overview warn of open ones and count those to review; the host page shows its own; the weekly report lists the week's and the lapsed; MCP list_anomalies and review_anomaly. - Reviewed and lapsed rows are pruned at data.retention_days, and kept in backups like alert history. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
97 lines
4.3 KiB
Python
97 lines
4.3 KiB
Python
"""Metric anomaly tools (#5740): metrics that left what is normal for them at
|
||
that hour, and marking each normal or real."""
|
||
from __future__ import annotations
|
||
|
||
from steward.core import anomalies as anom
|
||
from steward.mcp._context import current_caller
|
||
from steward.models.users import UserRole
|
||
|
||
from ._audit import audit_mcp, mcp_username
|
||
from ._shape import bounded, clamp_limit, db_session, iso, rnd
|
||
|
||
LIST_MAX = 200
|
||
|
||
|
||
def _row(v: anom.AnomalyView) -> dict:
|
||
a, last = v.anomaly, v.last_span
|
||
return {
|
||
"anomaly_id": a.id,
|
||
"source_module": a.source_module,
|
||
"resource": a.resource_name,
|
||
"metric": a.metric_name,
|
||
"review": a.review,
|
||
"ongoing": v.ongoing,
|
||
"started_at": iso(a.started_at),
|
||
"ended_at": iso(a.ended_at),
|
||
"occurrences": a.occurrences,
|
||
"last_occurrence": {"started_at": iso(last.started_at), "ended_at": iso(last.ended_at),
|
||
"peak_level": rnd(last.peak_level, 3)} if last else None,
|
||
"peak_level": rnd(a.peak_level, 3),
|
||
"peak_at": iso(a.peak_at),
|
||
"usual_hold": rnd(a.usual_hold, 3),
|
||
"usual_peak": rnd(a.usual_peak, 3),
|
||
"reviewed_by": a.reviewed_by,
|
||
"reviewed_at": iso(a.reviewed_at),
|
||
"note": a.note,
|
||
"page": v.href,
|
||
}
|
||
|
||
|
||
async def list_anomalies(review: str = "to_review", limit: int = 50) -> dict:
|
||
"""Metric anomalies: a learned metric whose 15-minute level went well
|
||
above what is normal for it at that hour (its usual hold ×1.25, and by
|
||
more than the metric's floor).
|
||
|
||
review: "to_review" (default) — waiting for someone to mark them; "normal"
|
||
— accepted, their hours are learned; "real" — a real problem, not learned;
|
||
"lapsed" — ended and nobody reviewed them within the review window
|
||
(Settings → Thresholds), so they are not learned; "" for all of them.
|
||
|
||
One row covers every repeat on a resource and metric while it waits for
|
||
review: `occurrences` counts them and `last_occurrence` is the latest.
|
||
`peak_level` is the highest 15-minute level any occurrence held, against
|
||
`usual_hold` (the level that hour usually holds) and `usual_peak` (how high
|
||
it usually goes), all in the metric's own unit. `ongoing` is true while an
|
||
occurrence is still going. To-review rows come first, newest occurrence
|
||
first. review_anomaly marks one.
|
||
"""
|
||
if review and review not in anom.REVIEWS:
|
||
raise ValueError(f"review must be one of {', '.join(anom.REVIEWS)}, or empty for all")
|
||
limit = clamp_limit(limit, 50, LIST_MAX)
|
||
async with db_session() as db:
|
||
rows = await anom.list_anomalies(db, (review,) if review else None, limit=LIST_MAX)
|
||
views = await anom.views(db, rows)
|
||
to_review = await anom.to_review_count(db)
|
||
out = bounded([_row(v) for v in views], limit,
|
||
key="anomalies", more="Pass a larger `limit`, or a `review` state.")
|
||
out["to_review"] = to_review
|
||
return out
|
||
|
||
|
||
async def review_anomaly(anomaly_id: str, review: str, note: str = "") -> dict:
|
||
"""Mark a metric anomaly "normal" or "real" (list_anomalies gives ids).
|
||
|
||
normal: this level is usual for this resource at that time; its hours are
|
||
learned into the metric's normal, so it stops being flagged once learned.
|
||
real: it was a real problem; its hours stay out of what is learned.
|
||
note: optional, kept with the review (say what it was).
|
||
A lapsed anomaly may still be marked. Audited as "<user> via MCP".
|
||
"""
|
||
caller = current_caller()
|
||
async with db_session() as db, db.begin():
|
||
try:
|
||
a = await anom.mark(db, anomaly_id, review, mcp_username(caller.username), note)
|
||
except LookupError:
|
||
raise ValueError(f"No anomaly {anomaly_id!r}; list_anomalies shows them.") from None
|
||
detail = {"source": a.source_module, "resource": a.resource_name,
|
||
"metric": a.metric_name, "note": a.note}
|
||
result = {"anomaly_id": a.id, "review": a.review, "reviewed_by": a.reviewed_by,
|
||
"reviewed_at": iso(a.reviewed_at), "note": a.note}
|
||
await audit_mcp(f"metric_anomaly.marked_{review}", "metric_anomaly", anomaly_id, detail)
|
||
return result
|
||
|
||
|
||
def register(register_tool) -> None:
|
||
register_tool(list_anomalies, role=UserRole.viewer, read_only=True)
|
||
register_tool(review_anomaly, role=UserRole.operator, read_only=False)
|