feat(anomalies): a metric out of its normal is an anomaly to review (#5740)
#5740 "A metric that departs from its normal is recorded as an anomaly to review, and is findable after it ends", milestone 513. - Every minute each learned key's last 15 minutes is judged against its hour's normal: above the usual hold x1.25 and by the metric's floor opens an occurrence, which ends when the level comes back. - metric_anomalies holds one row per item to review; a repeat while it still waits joins it as an occurrence (metric_anomaly_spans). - An ended row nobody reviews within anomalies.review_days (14, Settings -> Thresholds) lapses; an ongoing one never does. - Learning skips every hour an occurrence covers unless its anomaly was marked normal: no normal is learned without review. - Alerts -> Anomalies lists those to review first, then the reviewed and lapsed under day dividers; marking is operator/admin and audited. - The Overview widget and get_overview warn of open ones and count those to review; the host page shows its own; the weekly report lists the week's and the lapsed; MCP list_anomalies and review_anomaly. - Reviewed and lapsed rows are pruned at data.retention_days, and kept in backups like alert history. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -2,6 +2,7 @@ from __future__ import annotations
|
||||
from datetime import datetime, timezone
|
||||
from quart import Blueprint, render_template, request, redirect, url_for, current_app, session
|
||||
from steward.core.audit import log_audit
|
||||
from steward.core.settings import get_setting
|
||||
from steward.core import outbox
|
||||
from sqlalchemy import func, select
|
||||
from steward.auth.middleware import require_role
|
||||
@@ -279,6 +280,58 @@ async def alert_history():
|
||||
return await render_template("alerts/history.html", **ctx)
|
||||
|
||||
|
||||
@alerts_bp.get("/anomalies")
|
||||
@require_role(UserRole.viewer)
|
||||
async def anomalies():
|
||||
"""Metric anomalies (#5740): those to review, newest occurrence first,
|
||||
then the reviewed and lapsed ones under day dividers."""
|
||||
from steward.audit.service import mark_day_groups
|
||||
from steward.core import anomalies as anom
|
||||
from steward.core import formatting as fmt
|
||||
async with current_app.db_sessionmaker() as db:
|
||||
to_review = await anom.views(db, await anom.list_anomalies(db, (anom.TO_REVIEW,)))
|
||||
past = await anom.views(db, await anom.list_anomalies(
|
||||
db, (anom.NORMAL, anom.REAL, anom.LAPSED), limit=PAGE_SIZE * 4))
|
||||
days = int(await get_setting(db, "anomalies.review_days") or 14)
|
||||
tz = fmt.zone(session.get("tz"))
|
||||
to_review = [{"v": v} for v in to_review]
|
||||
past = [{"v": v} for v in past]
|
||||
at = lambda item: item["v"].anomaly.last_started_at # noqa: E731
|
||||
mark_day_groups(to_review, tz, at=at)
|
||||
mark_day_groups(past, tz, at=at)
|
||||
return await render_template("alerts/anomalies.html", to_review=to_review, past=past,
|
||||
review_days=days)
|
||||
|
||||
|
||||
@alerts_bp.post("/anomalies/<anomaly_id>/review")
|
||||
@require_role(UserRole.operator)
|
||||
async def review_anomaly(anomaly_id: str):
|
||||
"""Mark an anomaly normal (its hours are learned) or real (they are not),
|
||||
with an optional note. Audited."""
|
||||
from steward.core import anomalies as anom
|
||||
form = await request.form
|
||||
review = form.get("review", "")
|
||||
username = session.get("username", "")
|
||||
async with current_app.db_sessionmaker() as db:
|
||||
async with db.begin():
|
||||
try:
|
||||
a = await anom.mark(db, anomaly_id, review, username, form.get("note"))
|
||||
except LookupError:
|
||||
return "Not found", 404
|
||||
except ValueError as exc:
|
||||
return f"Bad request: {exc}", 400
|
||||
detail = {"source": a.source_module, "resource": a.resource_name,
|
||||
"metric": a.metric_name, "note": a.note}
|
||||
await log_audit(current_app, session.get("user_id"), username,
|
||||
f"metric_anomaly.marked_{review}", entity_type="metric_anomaly",
|
||||
entity_id=anomaly_id, detail=detail)
|
||||
back = form.get("next", "")
|
||||
# Only back to a page of this site.
|
||||
if not back.startswith("/") or back.startswith("//"):
|
||||
back = url_for("alerts.anomalies")
|
||||
return redirect(back)
|
||||
|
||||
|
||||
# How many of one rule's transitions the history verdict reads ("fired N
|
||||
# times, X in all"); a rule that flapped more often than this in the range
|
||||
# is summed over its newest ones.
|
||||
|
||||
@@ -104,13 +104,17 @@ PLUGIN_PAGES = {"traefik": "/plugins/traefik/", "unifi": "/plugins/unifi/",
|
||||
|
||||
|
||||
async def resource_url(db, rule: AlertRule) -> str | None:
|
||||
"""Where the thing a rule watches is shown, or None when nothing shows it.
|
||||
"""Where the thing a rule watches is shown, or None when nothing shows it."""
|
||||
return await resource_url_for(db, rule.source_module, rule.resource_name)
|
||||
|
||||
|
||||
async def resource_url_for(db, source: str, resource: str) -> str | None:
|
||||
"""Where a (source, resource) of the metric store is shown, or None.
|
||||
|
||||
host_agent resources are the host's name, or "<host>:<mount|nic|core>";
|
||||
docker's are "<host>/<container>"; a monitor's is its name under its type.
|
||||
"""
|
||||
from steward.models.hosts import Host
|
||||
source, resource = rule.source_module, rule.resource_name
|
||||
if source == "host_agent":
|
||||
name = resource.split(":", 1)[0]
|
||||
host_id = (await db.execute(
|
||||
|
||||
Reference in New Issue
Block a user