feat(mcp): understand and judge a metric anomaly: what happened, what was usual, what else went on (#5837)
CI / lint (push) Successful in 2s
CI / unit (push) Successful in 1m30s
CI / integration (push) Successful in 1m34s
CI / publish (push) Successful in 1m8s

get_anomaly gives each occurrence against what is usual at that hour, the
metric around the newest one with its normal, what else happened on the
resource's host around each occurrence (alert transitions, Ansible runs,
other anomalies there and a count elsewhere, plus plugins' own events
through a new mcp_events hook: the host agent's maintenance jobs and
downtime, Docker's container starts, deaths and OOM kills), and how earlier
anomalies on the same metric were judged.

get_metric_normal reads the learned slots and the level that would be
flagged. review_anomaly takes one id or several, all or nothing, with a
note. get_anomaly_settings and (admin) set_anomaly_settings read and change
the review window and each normal's floor, as Settings → Thresholds does.

metric_series.points takes an optional `until`; core.alerts gains
host_of_resource; floor_setting names its floor.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
2026-10-15 13:17:34 -04:00
co-authored by Claude Opus 5.5
parent 8feec4dc77
commit 0945d78290
14 changed files with 881 additions and 34 deletions
+9 -3
View File
@@ -84,13 +84,19 @@ def is_heartbeat(rule) -> bool:
_HEARTBEAT.source_module, _HEARTBEAT.metric_name)
def host_of_resource(source_module: str, resource: str) -> str | None:
"""The host whose agent carries this series, or None for a source no
agent feeds (a monitor, a Traefik router)."""
host_of = _AGENT_FED.get(source_module)
return host_of(resource) if host_of is not None else None
def host_behind(rule) -> str | None:
"""The host whose agent feeds `rule`, or None: a rule on another source,
or the heartbeat itself (agent offline is the one that speaks)."""
host_of = _AGENT_FED.get(rule.source_module)
if host_of is None or is_heartbeat(rule):
if is_heartbeat(rule):
return None
return host_of(rule.resource_name)
return host_of_resource(rule.source_module, rule.resource_name)
async def offline_agents(session) -> set[str]:
+9 -4
View File
@@ -143,8 +143,9 @@ async def backfill_latest(sessionmaker, log) -> int:
async def points(db, source_module: str, resource: str, metric: str, since: datetime,
*, target: int = 120) -> list[tuple[datetime, float]]:
"""(time, average) points for one metric since `since`, oldest first.
*, target: int = 120, until: datetime | None = None) -> list[tuple[datetime, float]]:
"""(time, average) points for one metric since `since` (up to `until`,
else now), oldest first.
The part of the range older than the raw window comes from the hourly
rollup; the rest is bucket-averaged in SQL (buckets no wider than an hour
@@ -167,14 +168,17 @@ async def points(db, source_module: str, resource: str, metric: str, since: date
PluginMetricHourly.resource_name == resource,
PluginMetricHourly.metric_name == metric,
PluginMetricHourly.bucket >= since,
PluginMetricHourly.bucket < raw_cutoff,
PluginMetricHourly.bucket < (min(raw_cutoff, until) if until else raw_cutoff),
)
.order_by(PluginMetricHourly.bucket)
)
out.extend((b, float(v)) for b, v in hourly)
raw_since = max(since, raw_cutoff)
width_s = bucket_seconds(raw_since, target)
if until is not None and raw_since >= until:
return out
width_s = (bucket_seconds(raw_since, target) if until is None
else max(60, int((until - raw_since).total_seconds() / target)))
if since < raw_cutoff:
width_s = min(width_s, 3600)
bucket = func.date_bin(
@@ -189,6 +193,7 @@ async def points(db, source_module: str, resource: str, metric: str, since: date
PluginMetric.resource_name == resource,
PluginMetric.metric_name == metric,
PluginMetric.recorded_at >= raw_since,
*((PluginMetric.recorded_at < until,) if until is not None else ()),
)
.group_by(bucket)
.order_by(bucket)
+12 -2
View File
@@ -72,6 +72,11 @@ class Learned:
def least_departure(self) -> float:
return float(self.floor() if callable(self.floor) else self.floor)
@property
def floor_name(self) -> str | None:
"""The NORM_FLOORS setting the floor is read from, if it is one."""
return getattr(self.floor, "floor_name", None)
LEARNED: dict[tuple[str, str], Learned] = {}
@@ -87,9 +92,14 @@ def learn_normal(source_module: str, metric_name: str, floor: float | Callable[[
def floor_setting(app, name: str) -> Callable[[], float]:
"""A floor read from Settings → Thresholds (norms.floor.<name>) at each
pass, so a saved change applies at once."""
pass, so a saved change applies at once. The function carries the
floor's name (`floor_name`), so a reader can say which setting it is."""
from steward.core.settings import floor_of
return lambda: floor_of(app.config.get("THRESHOLDS", {}), name)
def read() -> float:
return floor_of(app.config.get("THRESHOLDS", {}), name)
read.floor_name = name
return read
@dataclass(frozen=True)
+10 -1
View File
@@ -79,6 +79,9 @@ class StewardMCPServer(MCPServer):
self.tool_plugins: dict[str, str] = {}
# plugin name -> its `mcp_overview(db)`: findings it adds to get_overview.
self.overview_sources: dict[str, object] = {}
# plugin name -> its `mcp_events(db, host_id, host_name, since, until)`:
# what it knows happened on a host, for get_anomaly.
self.event_sources: dict[str, object] = {}
# ── registration ─────────────────────────────────────────────────────────
def register_tool(self, fn, *, role: UserRole, read_only: bool, plugin: str | None = None) -> None:
@@ -103,7 +106,9 @@ class StewardMCPServer(MCPServer):
def register_plugin(self, name: str, module) -> None:
"""Add a plugin's tools via its optional `register_mcp_tools(register)`,
and its optional `mcp_overview(db)` as a get_overview findings source.
its optional `mcp_overview(db)` as a get_overview findings source, and
its optional `mcp_events(db, host_id, host_name, since, until)` as a
source of what happened on a host (get_anomaly).
Idempotent per plugin (hot-reload drops its previous tools first) and
tolerant: a raising hook is logged and leaves the plugin with no tools,
@@ -114,6 +119,10 @@ class StewardMCPServer(MCPServer):
overview = getattr(module, "mcp_overview", None)
if callable(overview):
self.overview_sources[name] = overview
self.event_sources.pop(name, None)
events = getattr(module, "mcp_events", None)
if callable(events):
self.event_sources[name] = events
hook = getattr(module, "register_mcp_tools", None)
if hook is None:
return
+170
View File
@@ -0,0 +1,170 @@
"""What else was going on: events on a host, or one resource, in a window
(#5837), so an anomaly comes with its circumstances.
Core gathers alert transitions, Ansible runs and other metric anomalies. A
plugin adds its own through `mcp_events(db, host_id, host_name, since, until)`
in its __init__.py (the host agent its maintenance jobs and downtime, Docker
its container events); each returns a list of `event(...)`.
"""
from __future__ import annotations
import logging
from dataclasses import dataclass
from datetime import datetime
from sqlalchemy import and_, or_, select
from steward.alerts.service import watches_host
from steward.core.alerts import host_of_resource
from steward.mcp._context import mcp_app
from steward.models.alerts import AlertEvent, AlertRule
from steward.models.ansible import AnsibleRun
from steward.models.ansible_inventory import AnsibleTarget
from steward.models.hosts import Host
from steward.models.monitors import Monitor
from steward.models.norms import MetricAnomaly, MetricAnomalySpan
from ._shape import db_session, iso
logger = logging.getLogger(__name__)
PER_SOURCE = 40 # events any one source contributes to one window
def event(at: datetime | None, kind: str, what: str, *, until: datetime | None = None,
subject: str | None = None, ids: dict | None = None) -> dict:
"""One thing that happened. `until` for something that lasted (a run, a
reboot); `subject` names what it happened to when that isn't the host."""
return {"at": iso(at), "until": iso(until), "kind": kind, "what": what,
"subject": subject, "ids": ids or {}}
@dataclass(frozen=True)
class Place:
"""Where a metric's resource lives: its host, when it has one."""
source: str
resource: str
host_id: str | None = None
host_name: str | None = None
async def place_of(db, source: str, resource: str) -> Place:
"""The host behind a resource: an agent-fed series names it ("nas:/",
"nas/web"), a monitor may be pinned to one; a Traefik router has none."""
name = host_of_resource(source, resource)
host = None
if name:
host = (await db.execute(select(Host).where(Host.name == name).limit(1))).scalar_one_or_none()
else:
monitor = (await db.execute(select(Monitor).where(
Monitor.type == source, Monitor.name == resource).limit(1))).scalar_one_or_none()
if monitor is not None and monitor.host_id:
host = await db.get(Host, monitor.host_id)
return Place(source, resource, host.id if host else None, host.name if host else None)
def _overlaps(start, end):
"""A row lasting from `start` to `end` (None: still going) overlaps the window."""
def clause(since, until):
return and_(start <= until, or_(end.is_(None), end >= since))
return clause
async def alert_events(db, place: Place, since: datetime, until: datetime) -> list[dict]:
"""Alert transitions on the resource itself, or anything its host
carries: its agent series, its containers, its monitors."""
monitor_keys = set()
if place.host_id:
monitor_keys = {(t, n) for t, n in (await db.execute(
select(Monitor.type, Monitor.name).where(Monitor.host_id == place.host_id))).all()}
rows = (await db.execute(
select(AlertEvent, AlertRule).join(AlertRule, AlertRule.id == AlertEvent.rule_id)
.where(AlertEvent.transitioned_at >= since, AlertEvent.transitioned_at <= until)
.order_by(AlertEvent.transitioned_at).limit(500))).all()
out = []
for ev, rule in rows:
mine = (rule.source_module, rule.resource_name) == (place.source, place.resource)
if not mine and not (place.host_name and watches_host(rule, place.host_name, monitor_keys)):
continue
note = f" ({ev.note})" if ev.note else ""
out.append(event(ev.transitioned_at, "alert", f"{rule.name}: {ev.from_state} → {ev.to_state}{note}",
subject=rule.resource_name, ids={"rule_id": rule.id}))
return out[:PER_SOURCE]
async def ansible_runs(db, place: Place, since: datetime, until: datetime) -> list[dict]:
"""Ansible runs that touched the host while the window was open (by the
hosts in their recap: a run still going hasn't said yet)."""
if not place.host_id:
return []
names = {place.host_name} | set((await db.execute(
select(AnsibleTarget.name).where(AnsibleTarget.host_id == place.host_id))).scalars())
runs = (await db.execute(
select(AnsibleRun).where(_overlaps(AnsibleRun.started_at, AnsibleRun.finished_at)(since, until))
.order_by(AnsibleRun.started_at).limit(200))).scalars()
out = []
for r in runs:
if not names & set(((r.results or {}).get("hosts") or {}).keys()):
continue
check = " in check mode" if (r.params or {}).get("check") else ""
out.append(event(r.started_at, "ansible_run",
f"Ansible {r.playbook_path}{check}: {r.status.value}",
until=r.finished_at, ids={"run_id": r.id}))
return out[:PER_SOURCE]
def _same_host(source: str, resource: str, place: Place) -> bool:
if (source, resource) == (place.source, place.resource):
return True
return bool(place.host_name) and host_of_resource(source, resource) == place.host_name
async def other_anomalies(db, place: Place, anomaly_id: str, since: datetime,
until: datetime) -> tuple[list[dict], int]:
"""Other anomalies going on in the window: those on the same host as
events, and how many there were anywhere else (many at once points at
something shared: the network, storage, Steward itself)."""
rows = (await db.execute(
select(MetricAnomalySpan, MetricAnomaly)
.join(MetricAnomaly, MetricAnomaly.id == MetricAnomalySpan.anomaly_id)
.where(MetricAnomaly.id != anomaly_id,
_overlaps(MetricAnomalySpan.started_at, MetricAnomalySpan.ended_at)(since, until))
.order_by(MetricAnomalySpan.started_at).limit(500))).all()
here, elsewhere = [], 0
for span, a in rows:
if _same_host(a.source_module, a.resource_name, place):
here.append(event(span.started_at, "anomaly",
f"{a.resource_name} {a.metric_name} anomaly (peak {round(span.peak_level, 3)}, "
f"usual {round(a.usual_hold, 3)}; {a.review})",
until=span.ended_at, subject=a.resource_name, ids={"anomaly_id": a.id}))
else:
elsewhere += 1
return here[:PER_SOURCE], elsewhere
async def plugin_events(place: Place, since: datetime, until: datetime) -> list[dict]:
"""What enabled plugins know happened on the host (`mcp_events`), each
in its own session so one that fails can't spoil the others' reads."""
server = getattr(mcp_app(), "mcp_server", None)
if server is None or not place.host_id:
return []
out = []
for plugin, source in sorted((getattr(server, "event_sources", None) or {}).items()):
if not server.plugin_enabled(plugin):
continue
try:
async with db_session() as db:
out.extend((await source(db, place.host_id, place.host_name, since, until))[:PER_SOURCE])
except Exception:
logger.exception("MCP events source of plugin %r failed", plugin)
return out
async def around(db, place: Place, anomaly_id: str, since: datetime,
until: datetime) -> tuple[list[dict], int]:
"""Everything above for one window, oldest first, and the count of
anomalies elsewhere."""
anomalies, elsewhere = await other_anomalies(db, place, anomaly_id, since, until)
found = (await alert_events(db, place, since, until) + await ansible_runs(db, place, since, until)
+ anomalies + await plugin_events(place, since, until))
return sorted(found, key=lambda e: e["at"] or ""), elsewhere
+318 -16
View File
@@ -2,14 +2,25 @@
that hour, and marking each normal or real."""
from __future__ import annotations
from datetime import timedelta
from sqlalchemy import select
from steward.core import anomalies as anom
from steward.mcp._context import current_caller
from steward.core import metric_series
from steward.core.norms import LEARNED, norms_at, slot_of
from steward.core.settings import (DEFAULTS, NORM_FLOORS, floor_of, get_all_settings, set_setting,
to_thresholds_cfg)
from steward.mcp._context import current_caller, mcp_app
from steward.models.norms import MetricAnomaly, MetricAnomalySpan, MetricNorm
from steward.models.users import UserRole
from . import _events as events
from ._audit import audit_mcp, mcp_username
from ._shape import bounded, clamp_limit, db_session, iso, rnd
from ._shape import bounded, clamp_limit, db_session, iso, now_utc, rnd
LIST_MAX = 200
REVIEW_MAX = 100 # anomalies one review_anomaly call may mark
def _row(v: anom.AnomalyView) -> dict:
@@ -68,29 +79,320 @@ async def list_anomalies(review: str = "to_review", limit: int = 50) -> dict:
return out
async def review_anomaly(anomaly_id: str, review: str, note: str = "") -> dict:
"""Mark a metric anomaly "normal" or "real" (list_anomalies gives ids).
async def review_anomaly(anomaly_ids: list[str] | str, review: str, note: str = "") -> dict:
"""Mark metric anomalies "normal" or "real": one id, or several judged
the same way (list_anomalies gives ids; get_anomaly shows the evidence).
normal: this level is usual for this resource at that time; its hours are
learned into the metric's normal, so it stops being flagged once learned.
real: it was a real problem; its hours stay out of what is learned.
note: optional, kept with the review (say what it was).
A lapsed anomaly may still be marked. Audited as "<user> via MCP".
note: say why (what it was, what you checked); kept with each review, so
the next look at this resource and metric sees the precedent.
A lapsed anomaly may still be marked. All or nothing: an unknown id marks
none. Audited as "<user> via MCP", one row per anomaly.
"""
ids = [anomaly_ids] if isinstance(anomaly_ids, str) else list(anomaly_ids or [])
ids = list(dict.fromkeys(i.strip() for i in ids if i and i.strip()))
if not ids:
raise ValueError("Name at least one anomaly id; list_anomalies shows them.")
if len(ids) > REVIEW_MAX:
raise ValueError(f"At most {REVIEW_MAX} anomalies at once.")
caller = current_caller()
marked = []
async with db_session() as db, db.begin():
for anomaly_id in ids:
try:
a = await anom.mark(db, anomaly_id, review, mcp_username(caller.username), note)
except LookupError:
raise ValueError(f"No anomaly {anomaly_id!r}; list_anomalies shows them. "
"Nothing was marked.") from None
marked.append(({"source": a.source_module, "resource": a.resource_name,
"metric": a.metric_name, "note": a.note},
{"anomaly_id": a.id, "resource": a.resource_name, "metric": a.metric_name,
"review": a.review}))
reviewed_by, reviewed_at, kept = a.reviewed_by, a.reviewed_at, a.note
for (detail, row) in marked:
await audit_mcp(f"metric_anomaly.marked_{review}", "metric_anomaly", row["anomaly_id"], detail)
return {"marked": [row for _d, row in marked], "review": review, "reviewed_by": reviewed_by,
"reviewed_at": iso(reviewed_at), "note": kept}
# ── One anomaly, understood ─────────────────────────────────────────────────
AROUND = timedelta(minutes=30) # what else was going on: this long either side of an occurrence
SPANS_MAX = 20
async def _norm_map(db, key: tuple[str, str, str]) -> tuple[str | None, dict[int, MetricNorm]]:
"""The key's learned slots, {slot: row}, of the kind it uses."""
rows = list((await db.execute(select(MetricNorm).where(
MetricNorm.source_module == key[0], MetricNorm.resource_name == key[1],
MetricNorm.metric_name == key[2]))).scalars())
kind = "week" if any(r.kind == "week" for r in rows) else "day" if rows else None
return kind, {r.slot: r for r in rows if r.kind == kind}
def _usual_at(kind, slots: dict, at) -> MetricNorm | None:
return slots.get(slot_of(at, kind)) if kind else None
def series_pad(started_at, ended_at, now) -> timedelta:
"""How far either side of an occurrence its series reaches: as long as
the occurrence, at least an hour and at most six."""
held = (ended_at or now) - started_at
return min(max(held, timedelta(hours=1)), timedelta(hours=6))
async def get_anomaly(anomaly_id: str, occurrences: int = 5) -> dict:
"""One metric anomaly with what is needed to judge it: each occurrence
against what is usual at that hour, the metric around it, what else was
going on, and how earlier ones on the same metric were judged.
occurrences: how many of its occurrences to show, newest first (default
5, max 20).
- `anomaly`: the list_anomalies row, including `usual_hold` and
`usual_peak` as they stood when it was first flagged.
- `place`: the host the resource lives on (`host`, `host_id`), when it
has one: an agent series, a container, a monitor pinned to a host.
- `occurrences[]`: `started_at`, `ended_at`, `held_s`, `peak_level`,
and `usual` for that hour as Steward has learned it now (`usual_hold`,
`usual_peak`, `flag_above`), plus `around`: alert transitions, Ansible
runs, other anomalies on the host, and what plugins saw (container
starts, deaths and OOM kills; maintenance jobs; reboots and agent
stops), from 30 minutes before to 30 minutes after; and
`anomalies_elsewhere`, how many anomalies on other hosts overlapped it.
Many at once points at something shared (network, storage, Steward).
- `series`: the newest shown occurrence with time either side (as long
as it lasted, 1 to 6 hours), as [time, average, usual_hold at that
hour] points.
- `precedent`: earlier anomalies on this resource and metric, how they
were judged and the note left.
Judge it with review_anomaly: "normal" when it is how this resource
behaves at that time (its hours are then learned), "real" when it was a
problem. Say why in the note.
"""
limit = clamp_limit(occurrences, 5, SPANS_MAX)
now = now_utc()
async with db_session() as db:
a = await db.get(MetricAnomaly, (anomaly_id or "").strip())
if a is None:
raise ValueError(f"No anomaly {anomaly_id!r}; list_anomalies shows them.")
[view] = await anom.views(db, [a])
key = (a.source_module, a.resource_name, a.metric_name)
spans = list((await db.execute(
select(MetricAnomalySpan).where(MetricAnomalySpan.anomaly_id == a.id)
.order_by(MetricAnomalySpan.started_at.desc()))).scalars())
kind, slots = await _norm_map(db, key)
learned = LEARNED.get((a.source_module, a.metric_name))
floor = learned.least_departure if learned else None
place = await events.place_of(db, a.source_module, a.resource_name)
shown = []
for span in spans[:limit]:
usual = _usual_at(kind, slots, span.started_at)
near, elsewhere = await events.around(db, place, a.id, span.started_at - AROUND,
(span.ended_at or now) + AROUND)
shown.append({
"started_at": iso(span.started_at), "ended_at": iso(span.ended_at),
"ongoing": span.ended_at is None,
"held_s": int(((span.ended_at or now) - span.started_at).total_seconds()),
"peak_level": rnd(span.peak_level, 3),
"usual": ({"slot": slot_label(kind, usual.slot), "usual_hold": rnd(usual.usual_hold, 3),
"usual_peak": rnd(usual.usual_peak, 3),
"flag_above": rnd(flag_above(usual.usual_hold, floor), 3) if floor is not None else None}
if usual else None),
"around": near,
"anomalies_elsewhere": elsewhere,
})
series = None
if spans:
newest = spans[0]
pad = series_pad(newest.started_at, newest.ended_at, now)
frm, to = newest.started_at - pad, min((newest.ended_at or now) + pad, now)
pts = await metric_series.points(db, *key, frm, target=60, until=to)
series = {"from": iso(frm), "to": iso(to), "columns": ["at", "value", "usual_hold"],
"points": [[iso(t), rnd(v, 3),
rnd(u.usual_hold, 3) if (u := _usual_at(kind, slots, t)) else None]
for t, v in pts]}
earlier = list((await db.execute(
select(MetricAnomaly).where(
MetricAnomaly.source_module == a.source_module, MetricAnomaly.resource_name == a.resource_name,
MetricAnomaly.metric_name == a.metric_name, MetricAnomaly.id != a.id)
.order_by(MetricAnomaly.last_started_at.desc()).limit(10))).scalars())
return {
"anomaly": _row(view),
"place": {"host": place.host_name, "host_id": place.host_id},
"floor": rnd(floor, 3) if floor is not None else None,
"depart_factor": anom.DEPART,
"occurrences": shown,
"occurrences_total": len(spans),
"truncated": len(spans) > limit,
"series": series,
"precedent": [{"anomaly_id": e.id, "review": e.review, "note": e.note, "reviewed_by": e.reviewed_by,
"started_at": iso(e.started_at), "ended_at": iso(e.ended_at),
"occurrences": e.occurrences, "peak_level": rnd(e.peak_level, 3),
"usual_hold": rnd(e.usual_hold, 3)} for e in earlier],
"next": (f'get_metric_normal(source_module="{a.source_module}", resource="{a.resource_name}", '
f'metric="{a.metric_name}")'),
}
# ── The learned normal ──────────────────────────────────────────────────────
_DAYS = ("Mon", "Tue", "Wed", "Thu", "Fri", "Sat", "Sun")
def slot_label(kind: str, slot: int) -> str:
"""A slot as a reader names it: "03:00 UTC", or "Sun 03:00 UTC"."""
if kind == "week":
return f"{_DAYS[slot // 24]} {slot % 24:02d}:00 UTC"
return f"{slot:02d}:00 UTC"
def flag_above(norm_hold: float, floor: float) -> float:
"""The 15-minute level that would open an anomaly in a slot whose usual
hold is `norm_hold` (anomalies.departs)."""
return max(norm_hold * anom.DEPART, norm_hold + floor)
def _learned_or_refuse(source: str, metric: str):
learned = LEARNED.get((source, metric))
if learned is None:
known = ", ".join(sorted(f"{s}/{m}" for s, m in LEARNED))
raise ValueError(f"{source}/{metric} doesn't learn a normal. These do: {known}.")
return learned
async def get_metric_normal(source_module: str, resource: str, metric: str) -> dict:
"""What Steward has learned is normal for one resource's metric, slot by
slot: the yardstick an anomaly is judged against.
source_module, resource, metric: as list_anomalies or query_metrics name
them (e.g. "host_agent", "VDNT-Docker03", "cpu_pct").
`kind` is "day" (24 UTC hours, once 5 days of history exist) or "week"
(168 hours from Monday 00:00 UTC, once 21 days do). Each slot gives
`usual_hold` (the level its hours usually held: each hour's p95, at the
90th percentile across past days), `usual_peak` (how high they usually
went), `days_seen`, and `flag_above`: the 15-minute average that would
open an anomaly there (usual hold ×1.25, and at least `floor` above it).
`floor` is the smallest departure worth calling an anomaly, and
`floor_setting` the Settings → Thresholds floor it comes from. Hours an
anomaly covers are learned only once someone marks it normal.
`learned: false` with no slots means too little history yet; the fixed
cutoffs judge it meanwhile.
"""
learned = _learned_or_refuse(source_module, metric)
async with db_session() as db:
rows = list((await db.execute(
select(MetricNorm)
.where(MetricNorm.source_module == source_module, MetricNorm.resource_name == resource,
MetricNorm.metric_name == metric)
.order_by(MetricNorm.kind, MetricNorm.slot))).scalars())
now = now_utc()
current = (await norms_at(db, [(source_module, resource, metric)], now)).get(
(source_module, resource, metric))
floor = learned.least_departure
kinds = {r.kind for r in rows}
kind = "week" if "week" in kinds else "day" if rows else None
slots = [{"slot": r.slot, "label": slot_label(r.kind, r.slot), "usual_hold": rnd(r.usual_hold, 3),
"usual_peak": rnd(r.usual_peak, 3), "days_seen": r.days_seen,
"flag_above": rnd(flag_above(r.usual_hold, floor), 3)}
for r in rows if r.kind == kind]
out = {
"source_module": source_module, "resource": resource, "metric": metric,
"learned": bool(slots), "kind": kind,
"floor": rnd(floor, 3), "floor_setting": learned.floor_name,
"only_while": ({"metric": learned.gate[0],
"at_least": rnd(learned.gate[1]() if callable(learned.gate[1]) else learned.gate[1], 3)}
if learned.gate else None),
"now": ({"slot": current.slot, "label": slot_label(current.kind, current.slot),
"usual_hold": rnd(current.usual_hold, 3), "usual_peak": rnd(current.usual_peak, 3),
"flag_above": rnd(flag_above(current.usual_hold, floor), 3)} if current else None),
"slots": slots,
"updated_at": iso(max((r.updated_at for r in rows), default=None)),
}
return out
# ── Sensitivity (Settings → Thresholds) ─────────────────────────────────────
def _floors_view(thresholds: dict) -> list[dict]:
uses: dict[str, list[str]] = {}
for (source, metric), learned in sorted(LEARNED.items()):
if learned.floor_name:
uses.setdefault(learned.floor_name, []).append(f"{source}/{metric}")
return [{"name": name, "value": floor_of(thresholds, name),
"default": float(DEFAULTS[f"norms.floor.{name}"]), "metrics": uses.get(name, [])}
for name in NORM_FLOORS]
async def get_anomaly_settings() -> dict:
"""How sensitive anomaly detection is, as Settings → Thresholds sets it.
`review_days`: an ended anomaly nobody reviewed within this many days
lapses (leaves the to-review list, kept as not reviewed, never learned).
`floors`: per kind of metric, the smallest departure from its normal
worth calling an anomaly, in the metric's own unit (% points, ms), with
the metrics each one applies to. A level must also be ×`depart_factor`
its slot's usual hold, averaged over `window_minutes`. Raising a floor
flags less; lowering it flags more. set_anomaly_settings (admin) changes
the review window and floors.
"""
async with db_session() as db:
settings = await get_all_settings(db)
return {"review_days": int(settings.get("anomalies.review_days") or 14),
"depart_factor": anom.DEPART,
"window_minutes": int(anom.WINDOW.total_seconds() // 60),
"floors": _floors_view(to_thresholds_cfg(settings))}
async def set_anomaly_settings(review_days: int = 0, floors: dict[str, float] | None = None) -> dict:
"""Change how sensitive anomaly detection is, as Settings → Thresholds
does. Takes effect on the next minute's pass; audited.
review_days: days an ended anomaly waits for review before it lapses
(at least 1); 0 leaves it as it is.
floors: {name: value} for any of get_anomaly_settings' floor names, in
that floor's unit, 0 or more. A floor not named is left as it is.
Returns the settings as they now stand.
"""
floors = dict(floors or {})
unknown = sorted(set(floors) - set(NORM_FLOORS))
if unknown:
raise ValueError(f"No floor named {', '.join(unknown)}; they are {', '.join(NORM_FLOORS)}.")
changes: dict = {}
for name, value in floors.items():
try:
a = await anom.mark(db, anomaly_id, review, mcp_username(caller.username), note)
except LookupError:
raise ValueError(f"No anomaly {anomaly_id!r}; list_anomalies shows them.") from None
detail = {"source": a.source_module, "resource": a.resource_name,
"metric": a.metric_name, "note": a.note}
result = {"anomaly_id": a.id, "review": a.review, "reviewed_by": a.reviewed_by,
"reviewed_at": iso(a.reviewed_at), "note": a.note}
await audit_mcp(f"metric_anomaly.marked_{review}", "metric_anomaly", anomaly_id, detail)
return result
value = float(value)
except (TypeError, ValueError):
raise ValueError(f"Floor {name} must be a number.") from None
if value < 0 or value != value:
raise ValueError(f"Floor {name} must be 0 or more.")
changes[f"norms.floor.{name}"] = value
if review_days:
if int(review_days) < 1:
raise ValueError("review_days is at least 1.")
changes["anomalies.review_days"] = int(review_days)
if not changes:
raise ValueError("Nothing to change: pass review_days or floors.")
app = mcp_app()
async with db_session() as db:
async with db.begin():
before = await get_all_settings(db)
for key, value in changes.items():
await set_setting(db, key, value)
fresh = await get_all_settings(db)
app.config["THRESHOLDS"] = to_thresholds_cfg(fresh)
await audit_mcp("settings.saved", "settings", "anomalies",
{"section": "anomalies",
"changes": {k: {"from": before.get(k), "to": v} for k, v in changes.items()}})
return await get_anomaly_settings()
def register(register_tool) -> None:
register_tool(list_anomalies, role=UserRole.viewer, read_only=True)
for fn in (list_anomalies, get_anomaly, get_metric_normal, get_anomaly_settings):
register_tool(fn, role=UserRole.viewer, read_only=True)
register_tool(review_anomaly, role=UserRole.operator, read_only=False)
register_tool(set_anomaly_settings, role=UserRole.admin, read_only=False)