feat(anomalies): a metric out of its normal is an anomaly to review (#5740)

#5740 "A metric that departs from its normal is recorded as an anomaly to
review, and is findable after it ends", milestone 513.

- Every minute each learned key's last 15 minutes is judged against its
  hour's normal: above the usual hold x1.25 and by the metric's floor opens
  an occurrence, which ends when the level comes back.
- metric_anomalies holds one row per item to review; a repeat while it
  still waits joins it as an occurrence (metric_anomaly_spans).
- An ended row nobody reviews within anomalies.review_days (14, Settings ->
  Thresholds) lapses; an ongoing one never does.
- Learning skips every hour an occurrence covers unless its anomaly was
  marked normal: no normal is learned without review.
- Alerts -> Anomalies lists those to review first, then the reviewed and
  lapsed under day dividers; marking is operator/admin and audited.
- The Overview widget and get_overview warn of open ones and count those
  to review; the host page shows its own; the weekly report lists the
  week's and the lapsed; MCP list_anomalies and review_anomaly.
- Reviewed and lapsed rows are pruned at data.retention_days, and kept
  in backups like alert history.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

steward/alerts/routes.py

@@ -2,6 +2,7 @@ from __future__ import annotations
 from datetime import datetime, timezone
 from quart import Blueprint, render_template, request, redirect, url_for, current_app, session
 from steward.core.audit import log_audit
+from steward.core.settings import get_setting
 from steward.core import outbox
 from sqlalchemy import func, select
 from steward.auth.middleware import require_role
@@ -279,6 +280,58 @@ async def alert_history():
     return await render_template("alerts/history.html", **ctx)
 
 
+@alerts_bp.get("/anomalies")
+@require_role(UserRole.viewer)
+async def anomalies():
+    """Metric anomalies (#5740): those to review, newest occurrence first,
+    then the reviewed and lapsed ones under day dividers."""
+    from steward.audit.service import mark_day_groups
+    from steward.core import anomalies as anom
+    from steward.core import formatting as fmt
+    async with current_app.db_sessionmaker() as db:
+        to_review = await anom.views(db, await anom.list_anomalies(db, (anom.TO_REVIEW,)))
+        past = await anom.views(db, await anom.list_anomalies(
+            db, (anom.NORMAL, anom.REAL, anom.LAPSED), limit=PAGE_SIZE * 4))
+        days = int(await get_setting(db, "anomalies.review_days") or 14)
+    tz = fmt.zone(session.get("tz"))
+    to_review = [{"v": v} for v in to_review]
+    past = [{"v": v} for v in past]
+    at = lambda item: item["v"].anomaly.last_started_at  # noqa: E731
+    mark_day_groups(to_review, tz, at=at)
+    mark_day_groups(past, tz, at=at)
+    return await render_template("alerts/anomalies.html", to_review=to_review, past=past,
+                                 review_days=days)
+
+
+@alerts_bp.post("/anomalies/<anomaly_id>/review")
+@require_role(UserRole.operator)
+async def review_anomaly(anomaly_id: str):
+    """Mark an anomaly normal (its hours are learned) or real (they are not),
+    with an optional note. Audited."""
+    from steward.core import anomalies as anom
+    form = await request.form
+    review = form.get("review", "")
+    username = session.get("username", "")
+    async with current_app.db_sessionmaker() as db:
+        async with db.begin():
+            try:
+                a = await anom.mark(db, anomaly_id, review, username, form.get("note"))
+            except LookupError:
+                return "Not found", 404
+            except ValueError as exc:
+                return f"Bad request: {exc}", 400
+            detail = {"source": a.source_module, "resource": a.resource_name,
+                      "metric": a.metric_name, "note": a.note}
+    await log_audit(current_app, session.get("user_id"), username,
+                    f"metric_anomaly.marked_{review}", entity_type="metric_anomaly",
+                    entity_id=anomaly_id, detail=detail)
+    back = form.get("next", "")
+    # Only back to a page of this site.
+    if not back.startswith("/") or back.startswith("//"):
+        back = url_for("alerts.anomalies")
+    return redirect(back)
+
+
 # How many of one rule's transitions the history verdict reads ("fired N
 # times, X in all"); a rule that flapped more often than this in the range
 # is summed over its newest ones.

steward/alerts/service.py

@@ -104,13 +104,17 @@ PLUGIN_PAGES = {"traefik": "/plugins/traefik/", "unifi": "/plugins/unifi/",
 
 
 async def resource_url(db, rule: AlertRule) -> str | None:
-    """Where the thing a rule watches is shown, or None when nothing shows it.
+    """Where the thing a rule watches is shown, or None when nothing shows it."""
+    return await resource_url_for(db, rule.source_module, rule.resource_name)
+
+
+async def resource_url_for(db, source: str, resource: str) -> str | None:
+    """Where a (source, resource) of the metric store is shown, or None.
 
     host_agent resources are the host's name, or "<host>:<mount|nic|core>";
     docker's are "<host>/<container>"; a monitor's is its name under its type.
     """
     from steward.models.hosts import Host
-    source, resource = rule.source_module, rule.resource_name
     if source == "host_agent":
         name = resource.split(":", 1)[0]
         host_id = (await db.execute(

steward/app.py

@@ -449,6 +449,10 @@ def _register_core_tasks(app: Quart) -> None:
         from .core.norms import run_norms
         await run_norms(app)
 
+    async def run_metric_anomalies():
+        from .core.anomalies import run_anomalies
+        await run_anomalies(app)
+
     async def run_email_outbox():
         from .core.outbox import send_due
         await send_due(app)
@@ -520,6 +524,14 @@ def _register_core_tasks(app: Quart) -> None:
             interval_seconds=3600,
             run_on_startup=False,
         ),
+        # Each learned metric's last 15 minutes against its normal (#5740):
+        # opens, extends and ends anomalies, and lapses unreviewed ones.
+        ScheduledTask(
+            name="metric_anomalies",
+            coro_factory=run_metric_anomalies,
+            interval_seconds=60,
+            run_on_startup=False,
+        ),
     ])
 
     # Pull Ansible git sources as each falls due. One task reads the sources

steward/audit/service.py

@@ -265,13 +265,15 @@ def detail_facts(detail: dict, scopes: dict[str, str]) -> list[dict]:
     return facts
 
 
-def mark_day_groups(items: list[dict], tz, group: str = "", now=None) -> str:
+def mark_day_groups(items: list[dict], tz, group: str = "", now=None,
+                    at=lambda item: item["event"].timestamp) -> str:
     """Give each newest-first entry its recency group (formatting.day_group,
     in the reader's zone) and whether it opens one, so the page draws a
     divider there. `group` is the one the previous page ended in: a group
-    split across pages is opened once. Returns the group this page ends in."""
+    split across pages is opened once. `at` reads an entry's moment.
+    Returns the group this page ends in."""
     for item in items:
-        item["group"] = day_group(item["event"].timestamp, tz, now)
+        item["group"] = day_group(at(item), tz, now)
         item["opens_group"] = item["group"] != group
         group = item["group"]
     return group

steward/backup/contents.py

@@ -25,6 +25,9 @@ KEPT: frozenset[str] = frozenset({
     "unifi_known_clients",
     # History kept by the ruling.
     "audit_events", "alert_events",
+    # Metric anomalies and their reviews (#5740): what someone marked normal
+    # or real is a decision, kept like alert history.
+    "metric_anomalies", "metric_anomaly_spans",
 })
 
 # Rows that are measurements, logs or in-flight work: left out.

steward/core/anomalies.py

@@ -0,0 +1,316 @@
+"""A metric that departs from its normal is an anomaly to review (#5740).
+
+Every minute the last WINDOW of each learned key is averaged and compared
+with the normal for the slot it falls in (steward.core.norms). A level above
+the slot's usual hold by DEPART times, and by at least the metric's floor,
+opens an occurrence; the occurrence ends when the level comes back.
+
+What the operator ruled (System 182 "Metric Norms and Anomalies"):
+- A departure is a warning, marked as a metric anomaly, and stays "to review"
+  after it ends until someone marks it normal (its hours are learned) or real
+  (they are not).
+- The list never becomes a backlog. An occurrence while the key's earlier
+  anomaly is still to review joins it, so one row covers every repeat. An
+  ended anomaly nobody reviewed within anomalies.review_days lapses: it leaves
+  the to-review list and stays in history as not reviewed. A lapsed anomaly
+  is not learned, as no normal is learned without review.
+"""
+from __future__ import annotations
+
+from dataclasses import dataclass
+from datetime import datetime, timedelta, timezone
+
+from sqlalchemy import ARRAY, String, bindparam, case, delete, func, or_, select, text, update
+
+from steward.core.norms import LEARNED, Norm, norms_at
+from steward.models.norms import MetricAnomaly, MetricAnomalySpan
+
+WINDOW = timedelta(minutes=15)   # the level judged is this window's average
+DEPART = 1.25                    # times the slot's usual hold that counts as a departure
+# An occurrence that comes back within one window of ending continues the
+# same occurrence: a level wobbling around the line is one event, not ten.
+REJOIN = WINDOW
+
+TO_REVIEW, NORMAL, REAL, LAPSED = "to_review", "normal", "real", "lapsed"
+REVIEWS = (TO_REVIEW, NORMAL, REAL, LAPSED)
+MARKS = (NORMAL, REAL)
+
+
+def departs(level: float, norm: Norm, floor: float) -> bool:
+    """Whether `level` is out of `norm`: above its usual hold by DEPART times
+    and by at least `floor`, so a 0.1% → 0.3% share is never news."""
+    return level > norm.usual_hold * DEPART and level - norm.usual_hold >= floor
+
+
+# Each learned key reporting in the window, with the window's average. Keys
+# come from plugin_metric_latest (one row each), and each average is a range
+# read of the series index, so the pass never scans the raw table.
+_LEVELS = text("""
+SELECT l.source_module, l.resource_name, l.metric_name, v.level
+FROM plugin_metric_latest l
+CROSS JOIN LATERAL (
+    SELECT avg(p.value) AS level FROM plugin_metrics p
+    WHERE p.source_module = l.source_module AND p.resource_name = l.resource_name
+      AND p.metric_name = l.metric_name AND p.recorded_at >= :since
+      AND p.recorded_at <= :now
+) v
+WHERE l.source_module || '/' || l.metric_name = ANY(:keys)
+  AND l.recorded_at >= :since AND v.level IS NOT NULL
+""").bindparams(bindparam("keys", type_=ARRAY(String)))
+
+
+async def levels_now(session, now: datetime) -> dict[tuple[str, str, str], float]:
+    """{(source, resource, metric): its average over the last WINDOW}."""
+    keys = [f"{s}/{m}" for s, m in LEARNED]
+    if not keys:
+        return {}
+    rows = await session.execute(_LEVELS, {"keys": keys, "since": now - WINDOW, "now": now})
+    return {(s, r, m): float(level) for s, r, m, level in rows}
+
+
+async def detect(session, now: datetime | None = None) -> dict:
+    """One pass: open, extend and close occurrences. Runs inside the caller's
+    transaction. Returns {"opened", "closed"} counts."""
+    now = now or datetime.now(timezone.utc)
+    levels = await levels_now(session, now)
+    norms = await norms_at(session, list(levels), now)
+    open_now = {
+        (a.source_module, a.resource_name, a.metric_name): (span, a)
+        for span, a in (await session.execute(
+            select(MetricAnomalySpan, MetricAnomaly)
+            .join(MetricAnomaly, MetricAnomaly.id == MetricAnomalySpan.anomaly_id)
+            .where(MetricAnomalySpan.ended_at.is_(None)))).all()
+    }
+    opened = closed = 0
+    for key, level in levels.items():
+        norm = norms.get(key)
+        learned = LEARNED.get((key[0], key[2]))
+        out = norm is not None and learned is not None and departs(level, norm, learned.floor)
+        current = open_now.pop(key, None)
+        if current is not None:
+            span, anomaly = current
+            if out:
+                _raise_peak(anomaly, span, level, norm, now)
+            else:
+                _close(anomaly, span, now)
+                closed += 1
+        elif out:
+            await _occur(session, key, level, norm, now)
+            opened += 1
+    # An open occurrence whose key stopped reporting, or lost its normal,
+    # can't be judged any more: it ends now.
+    for span, anomaly in open_now.values():
+        _close(anomaly, span, now)
+        closed += 1
+    return {"opened": opened, "closed": closed}
+
+
+def _raise_peak(anomaly: MetricAnomaly, span: MetricAnomalySpan, level: float,
+                norm: Norm, now: datetime) -> None:
+    span.peak_level = max(span.peak_level, level)
+    if level > anomaly.peak_level:
+        anomaly.peak_level, anomaly.peak_at = level, now
+        anomaly.usual_hold, anomaly.usual_peak = norm.usual_hold, norm.usual_peak
+
+
+def _close(anomaly: MetricAnomaly, span: MetricAnomalySpan, now: datetime) -> None:
+    span.ended_at = now
+    anomaly.ended_at = now
+
+
+async def _occur(session, key, level: float, norm: Norm, now: datetime) -> None:
+    """A departure on a key with no open occurrence: join the key's anomaly
+    still to review, or open a new one."""
+    source, resource, metric = key
+    start = now - WINDOW
+    anomaly = (await session.execute(
+        select(MetricAnomaly)
+        .where(MetricAnomaly.source_module == source, MetricAnomaly.resource_name == resource,
+               MetricAnomaly.metric_name == metric, MetricAnomaly.review == TO_REVIEW)
+        .order_by(MetricAnomaly.last_started_at.desc()).limit(1))).scalar_one_or_none()
+    if anomaly is None:
+        anomaly = MetricAnomaly(
+            source_module=source, resource_name=resource, metric_name=metric,
+            started_at=start, last_started_at=start, ended_at=None, occurrences=1,
+            peak_level=level, peak_at=now, usual_hold=norm.usual_hold,
+            usual_peak=norm.usual_peak, review=TO_REVIEW)
+        session.add(anomaly)
+        await session.flush()
+        session.add(MetricAnomalySpan(anomaly_id=anomaly.id, started_at=start, peak_level=level))
+        return
+    last = (await session.execute(
+        select(MetricAnomalySpan).where(MetricAnomalySpan.anomaly_id == anomaly.id)
+        .order_by(MetricAnomalySpan.started_at.desc()).limit(1))).scalar_one_or_none()
+    if last is not None and last.ended_at is not None and now - last.ended_at <= REJOIN:
+        last.ended_at = None
+        span = last
+    else:
+        span = MetricAnomalySpan(anomaly_id=anomaly.id, started_at=start, peak_level=level)
+        session.add(span)
+        anomaly.occurrences += 1
+        anomaly.last_started_at = start
+    anomaly.ended_at = None
+    _raise_peak(anomaly, span, level, norm, now)
+
+
+async def lapse(session, now: datetime, review_days: int) -> int:
+    """Ended anomalies still to review after `review_days` lapse. One still
+    going never does. reviewed_at records when it lapsed. Returns how many."""
+    result = await session.execute(
+        update(MetricAnomaly)
+        .where(MetricAnomaly.review == TO_REVIEW, MetricAnomaly.ended_at.is_not(None),
+               MetricAnomaly.ended_at < now - timedelta(days=review_days))
+        .values(review=LAPSED, reviewed_at=now))
+    return result.rowcount or 0
+
+
+async def prune(session, before: datetime) -> int:
+    """Reviewed and lapsed anomalies whose last occurrence ended before
+    `before` are deleted (data.retention_days), their spans with them."""
+    result = await session.execute(
+        delete(MetricAnomaly)
+        .where(MetricAnomaly.review != TO_REVIEW, MetricAnomaly.ended_at.is_not(None),
+               MetricAnomaly.ended_at < before))
+    return result.rowcount or 0
+
+
+async def mark(session, anomaly_id: str, review: str, by: str,
+               note: str | None = None, now: datetime | None = None) -> MetricAnomaly:
+    """Mark an anomaly normal or real. A lapsed one may still be marked.
+    Raises LookupError when there is no such anomaly, ValueError on a bad mark."""
+    if review not in MARKS:
+        raise ValueError(f"review must be one of {', '.join(MARKS)}")
+    anomaly = await session.get(MetricAnomaly, anomaly_id)
+    if anomaly is None:
+        raise LookupError(anomaly_id)
+    anomaly.review = review
+    anomaly.reviewed_by = by
+    anomaly.reviewed_at = now or datetime.now(timezone.utc)
+    anomaly.note = (note or "").strip() or None
+    return anomaly
+
+
+async def run_anomalies(app) -> None:
+    """The minute pass: judge every learned key, then lapse what has waited
+    out the review window."""
+    from steward.core.settings import get_setting
+    now = datetime.now(timezone.utc)
+    async with app.db_sessionmaker() as session:
+        async with session.begin():
+            await detect(session, now)
+            days = int(await get_setting(session, "anomalies.review_days") or 14)
+            await lapse(session, now, max(1, days))
+
+
+# ── Reading them ─────────────────────────────────────────────────────────────
+
+@dataclass
+class AnomalyView:
+    """One anomaly as the pages, the report and MCP show it."""
+    anomaly: MetricAnomaly
+    last_span: MetricAnomalySpan | None
+    href: str | None = None
+
+    @property
+    def ongoing(self) -> bool:
+        return self.anomaly.ended_at is None
+
+    @property
+    def link(self) -> str | None:
+        """The resource's page over a range that takes in the latest
+        occurrence (the pages offer ranges back from now, not a window)."""
+        if not self.href:
+            return None
+        reach = datetime.now(timezone.utc) - self.anomaly.last_started_at
+        key = "24h" if reach < timedelta(hours=20) else "7d" if reach < timedelta(days=6) else "30d"
+        return f"{self.href}?range={key}"
+
+
+def _ordered():
+    """To review first, then the rest; newest occurrence first within each."""
+    return (case((MetricAnomaly.review == TO_REVIEW, 0), else_=1),
+            MetricAnomaly.last_started_at.desc())
+
+
+async def views(session, rows: list[MetricAnomaly], *, links: bool = True) -> list[AnomalyView]:
+    """Each anomaly with its latest occurrence and the page that shows its
+    resource."""
+    if not rows:
+        return []
+    from steward.alerts.service import resource_url_for
+    latest = {}
+    for span in (await session.execute(
+            select(MetricAnomalySpan)
+            .where(MetricAnomalySpan.anomaly_id.in_([a.id for a in rows]))
+            .order_by(MetricAnomalySpan.started_at))).scalars():
+        latest[span.anomaly_id] = span
+    hrefs: dict[tuple[str, str], str | None] = {}
+    out = []
+    for a in rows:
+        key = (a.source_module, a.resource_name)
+        if links and key not in hrefs:
+            hrefs[key] = await resource_url_for(session, *key)
+        out.append(AnomalyView(a, latest.get(a.id), hrefs.get(key)))
+    return out
+
+
+async def list_anomalies(session, reviews=None, *, since: datetime | None = None,
+                         source: str | None = None, resource: str | None = None,
+                         limit: int = 200) -> list[MetricAnomaly]:
+    """Anomalies in any of `reviews` (all when None), to review first, newest
+    occurrence first. `since` keeps those with an occurrence after it, or
+    still going."""
+    q = select(MetricAnomaly)
+    if reviews:
+        q = q.where(MetricAnomaly.review.in_(list(reviews)))
+    if since is not None:
+        q = q.where(or_(MetricAnomaly.ended_at.is_(None), MetricAnomaly.ended_at >= since))
+    if source:
+        q = q.where(MetricAnomaly.source_module == source)
+    if resource:
+        # host_agent's parts ("host:/var") belong to the host's page too.
+        q = q.where(or_(MetricAnomaly.resource_name == resource,
+                        MetricAnomaly.resource_name.like(f"{resource}:%")))
+    return list((await session.execute(q.order_by(*_ordered()).limit(limit))).scalars())
+
+
+async def open_anomalies(session) -> list[MetricAnomaly]:
+    """Anomalies with an occurrence going on now that still need attention:
+    to review, or marked real. One marked normal is the new normal being
+    learned, and is not a warning."""
+    return list((await session.execute(
+        select(MetricAnomaly)
+        .where(MetricAnomaly.ended_at.is_(None), MetricAnomaly.review.in_((TO_REVIEW, REAL)))
+        .order_by(MetricAnomaly.last_started_at))).scalars())
+
+
+async def to_review_count(session) -> int:
+    return int((await session.execute(
+        select(func.count()).select_from(MetricAnomaly)
+        .where(MetricAnomaly.review == TO_REVIEW))).scalar_one())
+
+
+async def spans_for(session, source: str, resource: str, metric: str | None,
+                    since: datetime) -> list[MetricAnomalySpan]:
+    """Occurrences on one resource (and metric) since `since`, for shading
+    its chart."""
+    q = (select(MetricAnomalySpan)
+         .join(MetricAnomaly, MetricAnomaly.id == MetricAnomalySpan.anomaly_id)
+         .where(MetricAnomaly.source_module == source, MetricAnomaly.resource_name == resource,
+                or_(MetricAnomalySpan.ended_at.is_(None), MetricAnomalySpan.ended_at >= since)))
+    if metric:
+        q = q.where(MetricAnomaly.metric_name == metric)
+    return list((await session.execute(q.order_by(MetricAnomalySpan.started_at))).scalars())
+
+
+async def week_summary(session, since: datetime) -> dict:
+    """For the weekly report: anomalies with an occurrence since `since`, and
+    those that lapsed since then."""
+    seen = await list_anomalies(session, since=since, limit=50)
+    lapsed = list((await session.execute(
+        select(MetricAnomaly).where(MetricAnomaly.review == LAPSED,
+                                    MetricAnomaly.reviewed_at >= since)
+        .order_by(MetricAnomaly.last_started_at.desc()).limit(50))).scalars())
+    seen_ids = {a.id for a in seen}
+    return {"seen": seen, "lapsed": [a for a in lapsed if a.id not in seen_ids]}
+

steward/core/cleanup.py

@@ -8,6 +8,7 @@ from typing import TYPE_CHECKING
 
 from sqlalchemy import delete
 
+from steward.core.anomalies import prune as prune_anomalies
 from steward.models.alerts import AlertEvent
 from steward.models.ansible import AnsibleRun
 from steward.models.audit import AuditEvent
@@ -76,6 +77,11 @@ async def run_cleanup(app: "Quart") -> dict:
                 removed.append([f"{table}: older than {days} days", result.rowcount or 0])
                 if result.rowcount:
                     logger.info("Pruned %d rows from %s", result.rowcount, table)
+            # Anomalies go by when their last occurrence ended, and only once
+            # reviewed or lapsed: one still to review is never pruned (#5740).
+            pruned = await prune_anomalies(session, cutoff)
+            removed.append([f"metric_anomalies: reviewed, ended over {data_days} days ago",
+                            pruned])
 
         files = await asyncio.to_thread(prune_run_logs, ARTIFACT_DIR, cutoff)
         removed.append([f"Ansible run log files: older than {data_days} days", files])

steward/core/formatting.py

@@ -207,6 +207,12 @@ STATES: dict[str, tuple[str, str]] = {
     "failed": ("crit", "Failed"),
     "interrupted": ("warn", "Interrupted"),
     "cancelled": ("dim", "Cancelled"),
+    # A metric anomaly (#5740), and where its review stands.
+    "anomaly": ("warn", "Metric anomaly"),
+    "to_review": ("warn", "To review"),
+    "normal": ("ok", "Marked normal"),
+    "real": ("crit", "Real problem"),
+    "lapsed": ("dim", "Not reviewed"),
 }
 
 

steward/core/mail.py

@@ -258,6 +258,22 @@ def burst_email(cfg: dict, alerts: list[dict], to: Recipient) -> tuple[str, str,
     return f"[Steward] {len(facts)} alerts: {said}", text, html
 
 
+def anomaly_facts(a: dict, tz: tzinfo) -> dict:
+    """A metric anomaly in the weekly report (#5740): its review as a badge,
+    and when it happened in the reader's zone."""
+    from steward.core.formatting import status_info
+    tone, review = status_info(a["review"])
+    if a["occurrences"] > 1:
+        last = (f"since {moment(a['last_started_at'], tz)}, ongoing" if a["ended_at"] is None
+                else f"{moment(a['last_started_at'], tz)} – {moment(a['ended_at'], tz)}")
+        when = f"{a['occurrences']} times since {moment(a['started_at'], tz)}, last {last}"
+    elif a["ended_at"] is None:
+        when = f"ongoing since {moment(a['started_at'], tz)}"
+    else:
+        when = f"{moment(a['started_at'], tz)} – {moment(a['ended_at'], tz)}"
+    return {"name": a["name"], "tone": tone, "review": review, "when": when, "level": a["level"]}
+
+
 def report_email(cfg: dict, data: dict, to: Recipient) -> tuple[str, str, str]:
     """(subject, text, html) for the weekly report built by
     reports.build_report_data, its dates in the recipient's zone."""
@@ -269,8 +285,12 @@ def report_email(cfg: dict, data: dict, to: Recipient) -> tuple[str, str, str]:
               else f"{start.day} {start:%b %Y} – {end.day} {end:%b %Y}")
     alerts = [{**a, "tone": status_info(a["state"])[0], "state": status_info(a["state"])[1]}
               for a in data["active_alerts"]]
+    found = data.get("anomalies") or {}
+    anomalies = {"seen": [anomaly_facts(a, tz) for a in found.get("seen", [])],
+                 "lapsed": [anomaly_facts(a, tz) for a in found.get("lapsed", [])],
+                 "to_review": found.get("to_review", 0)}
     base = (cfg.get("base_url") or "").rstrip("/")
-    text, html = _render("report", d=data, alerts=alerts, period=period,
+    text, html = _render("report", d=data, alerts=alerts, anomalies=anomalies, period=period,
                          sent=moment(data["generated_at"], tz), base=base)
     return f"[Steward] Weekly report, {period}", text, html
 

steward/core/norms.py

@@ -24,6 +24,10 @@ The slots:
 Only metrics registered with learn_normal() are learned: a host agent alone
 reports thousands of per-core, per-disk and per-interface series, and most
 have no use for a normal.
+
+An hour an anomaly covers is learned only once someone marks the anomaly
+normal (steward.core.anomalies): a level that jumps and stays is never
+absorbed on its own, and a lapsed anomaly is never learned.
 """
 from __future__ import annotations
 
@@ -80,11 +84,12 @@ def slot_of(at: datetime, kind: str) -> int:
 
 
 # One statement learns every normal: the hours of each learned key (rolled up,
-# plus the raw window not rolled up yet), which kind of slot its history
-# supports, then each slot's usual hold and peak across past days, every hour
-# counted in its own slot and its neighbours'.
+# plus the raw window not rolled up yet) less those an anomaly covers that
+# nobody marked normal (#5740), which kind of slot its history supports, then
+# each slot's usual hold and peak across past days, every hour counted in its
+# own slot and its neighbours'.
 _REBUILD = text("""
-WITH hours AS (
+WITH all_hours AS (
     SELECT source_module AS s, resource_name AS r, metric_name AS m, bucket AS b,
            COALESCE(value_p95, value_avg) AS hold, value_max AS peak
     FROM plugin_metrics_hourly
@@ -97,6 +102,16 @@ WITH hours AS (
       AND source_module || '/' || metric_name = ANY(:keys)
     GROUP BY 1, 2, 3, 4
 ),
+hours AS (
+    SELECT * FROM all_hours h
+    WHERE NOT EXISTS (
+        SELECT 1 FROM metric_anomaly_spans sp
+        JOIN metric_anomalies a ON a.id = sp.anomaly_id
+        WHERE a.review <> 'normal'
+          AND a.source_module = h.s AND a.resource_name = h.r AND a.metric_name = h.m
+          AND sp.started_at < h.b + interval '1 hour'
+          AND COALESCE(sp.ended_at, :now) > h.b)
+),
 keyed AS (
     SELECT s, r, m, count(DISTINCT date_trunc('day', b)) AS days, min(b) AS first
     FROM hours GROUP BY s, r, m

steward/core/reports.py

@@ -106,6 +106,15 @@ async def build_report_data(app) -> dict:
         hosts_down = sorted(name for mid, name in enabled
                             if latest.get(mid) and not latest[mid][-1].is_up)
 
+        # ── Metric anomalies this week, and those that lapsed (#5740) ─────────
+        from steward.core import anomalies as anom
+        week = await anom.week_summary(db, cutoff_7d)
+        anomalies = {
+            "seen": [_anomaly(a) for a in week["seen"]],
+            "lapsed": [_anomaly(a) for a in week["lapsed"]],
+            "to_review": await anom.to_review_count(db),
+        }
+
     return {
         "generated_at": now,
         "uptime_summary": uptime_summary,
@@ -113,6 +122,21 @@ async def build_report_data(app) -> dict:
         "alert_events": alert_events,
         "top_routers": top_routers,
         "hosts_down": hosts_down,
+        "anomalies": anomalies,
+    }
+
+
+def _anomaly(a) -> dict:
+    """One anomaly for the report; its times are put in each reader's zone
+    when the email is written (mail.report_email)."""
+    from steward.core.formatting import metric_label, metric_value
+    return {
+        "name": f"{a.resource_name} — {metric_label(a.metric_name)}",
+        "review": a.review,
+        "started_at": a.started_at, "last_started_at": a.last_started_at, "ended_at": a.ended_at,
+        "occurrences": a.occurrences,
+        "level": (f"held {metric_value(a.metric_name, a.peak_level)}, usual "
+                  f"{metric_value(a.metric_name, a.usual_hold)}"),
     }
 
 

steward/core/settings.py

@@ -114,6 +114,9 @@ DEFAULTS: dict[str, Any] = {
     # Seconds a host's clock may be from Steward's before it is called out
     # (milestone 490), either way.
     "thresholds.clock_warn_s": 30, "thresholds.clock_crit_s": 300,
+    # Days an ended metric anomaly waits to be reviewed before it lapses:
+    # it leaves the to-review list and is kept as not reviewed (#5740).
+    "anomalies.review_days": 14,
     # Docker time-series retention (rule 25 — tunable, no restart). Raw 30s
     # samples are heavy, so keep a short raw window then roll up to hourly
     # averages kept much longer; lifecycle events are light, keep a month.

steward/hosts/routes.py

@@ -1,13 +1,13 @@
 from __future__ import annotations
 import asyncio
 import uuid
-from datetime import datetime, timezone
+from datetime import datetime, timedelta, timezone
 from quart import Blueprint, make_response, render_template, request, redirect, url_for, current_app, session
 from sqlalchemy import select
 from steward.ansible import executor, sources as ansible_src
 from steward.alerts import service as alert_svc
 from steward.auth.middleware import require_role
-from steward.core import renames
+from steward.core import anomalies as anom, renames
 from steward.core.audit import audit_request, log_audit
 from steward.models.ansible import AnsibleRun, AnsibleRunStatus
 from steward.models.hosts import Host
@@ -131,10 +131,14 @@ async def host_detail(host_id: str):
         linkable_targets = (await db.execute(
             select(AnsibleTarget).where(AnsibleTarget.host_id.is_(None))
             .order_by(AnsibleTarget.name))).scalars().all()
+        # The host's metric anomalies, its disks and interfaces included (#5740).
+        anomalies = await anom.views(db, await anom.list_anomalies(
+            db, source="host_agent", resource=host.name,
+            since=datetime.now(timezone.utc) - timedelta(days=30), limit=10), links=False)
 
     return await render_template(
         "hosts/detail.html",
-        host=host, monitors=monitors, uptime=uptime, alerts=alerts,
+        host=host, monitors=monitors, uptime=uptime, alerts=alerts, anomalies=anomalies,
         linked_target=linked_target, linkable_targets=linkable_targets,
         ansible_sources=_ansible_source_names(),
     )

steward/mcp/tools/__init__.py

@@ -12,9 +12,9 @@ each argument's empty value means, and what comes back.
 """
 from __future__ import annotations
 
-from . import alerts, audit, automation, hosts, metrics, monitors, overview, session
+from . import alerts, anomalies, audit, automation, hosts, metrics, monitors, overview, session
 
-_MODULES = (session, overview, hosts, monitors, alerts, metrics, audit, automation)
+_MODULES = (session, overview, hosts, monitors, alerts, anomalies, metrics, audit, automation)
 
 
 def register_all(register_tool) -> None:

steward/mcp/tools/anomalies.py

@@ -0,0 +1,96 @@
+"""Metric anomaly tools (#5740): metrics that left what is normal for them at
+that hour, and marking each normal or real."""
+from __future__ import annotations
+
+from steward.core import anomalies as anom
+from steward.mcp._context import current_caller
+from steward.models.users import UserRole
+
+from ._audit import audit_mcp, mcp_username
+from ._shape import bounded, clamp_limit, db_session, iso, rnd
+
+LIST_MAX = 200
+
+
+def _row(v: anom.AnomalyView) -> dict:
+    a, last = v.anomaly, v.last_span
+    return {
+        "anomaly_id": a.id,
+        "source_module": a.source_module,
+        "resource": a.resource_name,
+        "metric": a.metric_name,
+        "review": a.review,
+        "ongoing": v.ongoing,
+        "started_at": iso(a.started_at),
+        "ended_at": iso(a.ended_at),
+        "occurrences": a.occurrences,
+        "last_occurrence": {"started_at": iso(last.started_at), "ended_at": iso(last.ended_at),
+                            "peak_level": rnd(last.peak_level, 3)} if last else None,
+        "peak_level": rnd(a.peak_level, 3),
+        "peak_at": iso(a.peak_at),
+        "usual_hold": rnd(a.usual_hold, 3),
+        "usual_peak": rnd(a.usual_peak, 3),
+        "reviewed_by": a.reviewed_by,
+        "reviewed_at": iso(a.reviewed_at),
+        "note": a.note,
+        "page": v.href,
+    }
+
+
+async def list_anomalies(review: str = "to_review", limit: int = 50) -> dict:
+    """Metric anomalies: a learned metric whose 15-minute level went well
+    above what is normal for it at that hour (its usual hold ×1.25, and by
+    more than the metric's floor).
+
+    review: "to_review" (default) — waiting for someone to mark them; "normal"
+    — accepted, their hours are learned; "real" — a real problem, not learned;
+    "lapsed" — ended and nobody reviewed them within the review window
+    (Settings → Thresholds), so they are not learned; "" for all of them.
+
+    One row covers every repeat on a resource and metric while it waits for
+    review: `occurrences` counts them and `last_occurrence` is the latest.
+    `peak_level` is the highest 15-minute level any occurrence held, against
+    `usual_hold` (the level that hour usually holds) and `usual_peak` (how high
+    it usually goes), all in the metric's own unit. `ongoing` is true while an
+    occurrence is still going. To-review rows come first, newest occurrence
+    first. review_anomaly marks one.
+    """
+    if review and review not in anom.REVIEWS:
+        raise ValueError(f"review must be one of {', '.join(anom.REVIEWS)}, or empty for all")
+    limit = clamp_limit(limit, 50, LIST_MAX)
+    async with db_session() as db:
+        rows = await anom.list_anomalies(db, (review,) if review else None, limit=LIST_MAX)
+        views = await anom.views(db, rows)
+        to_review = await anom.to_review_count(db)
+    out = bounded([_row(v) for v in views], limit,
+                  key="anomalies", more="Pass a larger `limit`, or a `review` state.")
+    out["to_review"] = to_review
+    return out
+
+
+async def review_anomaly(anomaly_id: str, review: str, note: str = "") -> dict:
+    """Mark a metric anomaly "normal" or "real" (list_anomalies gives ids).
+
+    normal: this level is usual for this resource at that time; its hours are
+    learned into the metric's normal, so it stops being flagged once learned.
+    real: it was a real problem; its hours stay out of what is learned.
+    note: optional, kept with the review (say what it was).
+    A lapsed anomaly may still be marked. Audited as "<user> via MCP".
+    """
+    caller = current_caller()
+    async with db_session() as db, db.begin():
+        try:
+            a = await anom.mark(db, anomaly_id, review, mcp_username(caller.username), note)
+        except LookupError:
+            raise ValueError(f"No anomaly {anomaly_id!r}; list_anomalies shows them.") from None
+        detail = {"source": a.source_module, "resource": a.resource_name,
+                  "metric": a.metric_name, "note": a.note}
+        result = {"anomaly_id": a.id, "review": a.review, "reviewed_by": a.reviewed_by,
+                  "reviewed_at": iso(a.reviewed_at), "note": a.note}
+    await audit_mcp(f"metric_anomaly.marked_{review}", "metric_anomaly", anomaly_id, detail)
+    return result
+
+
+def register(register_tool) -> None:
+    register_tool(list_anomalies, role=UserRole.viewer, read_only=True)
+    register_tool(review_anomaly, role=UserRole.operator, read_only=False)

steward/mcp/tools/overview.py

@@ -7,6 +7,7 @@ from datetime import timedelta
 from sqlalchemy import select
 
 from steward.alerts import service as alert_svc
+from steward.core import anomalies as anom
 from steward.hosts import service as host_svc
 from steward.mcp._context import mcp_app
 from steward.models.ansible import AnsibleRun, AnsibleRunStatus
@@ -140,6 +141,22 @@ def run_findings(runs) -> list[dict]:
     return out
 
 
+def anomaly_findings(open_now) -> list[dict]:
+    """Metric anomalies going on now (#5740): a warning each."""
+    from steward.core.formatting import metric_label, metric_value
+    out = []
+    for a in open_now:
+        out.append(finding(
+            "warning", "metric_anomaly", f"{a.resource_name} {metric_label(a.metric_name)}",
+            f"Metric anomaly: {metric_label(a.metric_name)} on {a.resource_name} held "
+            f"{metric_value(a.metric_name, a.peak_level)} where "
+            f"{metric_value(a.metric_name, a.usual_hold)} is usual for that hour"
+            + (f" ({a.occurrences} times since {iso(a.started_at)})" if a.occurrences > 1 else ""),
+            since=a.last_started_at, ids={"anomaly_id": a.id},
+            next=f'list_anomalies(review="{a.review}")'))
+    return out
+
+
 async def _plugin_findings(server) -> tuple[list[dict], list[str]]:
     found, unavailable = [], []
     for plugin, source in sorted((getattr(server, "overview_sources", None) or {}).items()):
@@ -162,12 +179,15 @@ async def get_overview() -> dict:
     still-breaching acknowledged alerts, TLS certificates near expiry, host
     agents that stopped reporting, hosts over their CPU, memory or disk
     limits (all cutoffs are set under Settings → Thresholds), Ansible runs that failed
-    in the last 24h, and anything enabled plugins add (unhealthy containers,
-    for one). Each finding carries the ids to follow and `next`, the call
-    that shows its evidence.
+    in the last 24h, metrics going on now well above what is normal for them
+    at this hour (metric anomalies), and anything enabled plugins add
+    (unhealthy containers, for one). Each finding carries the ids to follow
+    and `next`, the call that shows its evidence.
 
-    `counts` sizes the fleet, and `active_maintenance` names windows
-    currently silencing alerts. `all_clear` is true when nothing is wrong.
+    `counts` sizes the fleet; `counts.anomalies_to_review` is how many metric
+    anomalies wait for someone to mark them normal or real (list_anomalies).
+    `active_maintenance` names windows currently silencing alerts.
+    `all_clear` is true when nothing is wrong.
     """
     app = mcp_app()
     now = now_utc()
@@ -187,12 +207,15 @@ async def get_overview() -> dict:
             .limit(20)
         )).scalars())
         windows = await alert_svc.list_maintenance_windows(db)
+        open_anomalies = await anom.open_anomalies(db)
+        to_review = await anom.to_review_count(db)
 
     thresholds = app.config.get("THRESHOLDS") or {}
     findings = (monitor_findings(monitors, states, host_names, now, thresholds.get("tls_days"))
                 + alert_findings(active)
                 + agent_findings(hosts, agent, thresholds)
-                + run_findings(runs))
+                + run_findings(runs)
+                + anomaly_findings(open_anomalies))
     plugin_found, unavailable = await _plugin_findings(getattr(app, "mcp_server", None))
     findings = rank(findings + plugin_found)
 
@@ -208,6 +231,7 @@ async def get_overview() -> dict:
             "alerts_active": len(active),
             "critical": sum(1 for f in findings if f["severity"] == "critical"),
             "warning": sum(1 for f in findings if f["severity"] == "warning"),
+            "anomalies_to_review": to_review,
         },
         "active_maintenance": [
             {"window_id": w.id, "name": w.name, "end_at": iso(w.end_at),

steward/migrations/versions/0046_metric_anomalies.py

@@ -0,0 +1,55 @@
+"""Metrics that departed from their normal, to review (#5740).
+
+Revision ID: 0046_metric_anomalies
+Revises: 0045_metric_norms
+Create Date: 2026-10-14
+"""
+from typing import Sequence, Union
+from alembic import op
+import sqlalchemy as sa
+
+revision: str = "0046_metric_anomalies"
+down_revision: Union[str, None] = "0045_metric_norms"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    op.create_table(
+        "metric_anomalies",
+        sa.Column("id", sa.String(36), primary_key=True),
+        sa.Column("source_module", sa.String(64), nullable=False),
+        sa.Column("resource_name", sa.String(255), nullable=False),
+        sa.Column("metric_name", sa.String(128), nullable=False),
+        sa.Column("started_at", sa.DateTime(timezone=True), nullable=False),
+        sa.Column("last_started_at", sa.DateTime(timezone=True), nullable=False),
+        sa.Column("ended_at", sa.DateTime(timezone=True), nullable=True),
+        sa.Column("occurrences", sa.Integer, nullable=False),
+        sa.Column("peak_level", sa.Float, nullable=False),
+        sa.Column("peak_at", sa.DateTime(timezone=True), nullable=False),
+        sa.Column("usual_hold", sa.Float, nullable=False),
+        sa.Column("usual_peak", sa.Float, nullable=False),
+        sa.Column("review", sa.String(12), nullable=False),
+        sa.Column("reviewed_by", sa.String(64), nullable=True),
+        sa.Column("reviewed_at", sa.DateTime(timezone=True), nullable=True),
+        sa.Column("note", sa.Text, nullable=True),
+    )
+    op.create_index("ix_metric_anomalies_key", "metric_anomalies",
+                    ["source_module", "resource_name", "metric_name"])
+    op.create_index("ix_metric_anomalies_review_last", "metric_anomalies",
+                    ["review", "last_started_at"])
+    op.create_table(
+        "metric_anomaly_spans",
+        sa.Column("id", sa.String(36), primary_key=True),
+        sa.Column("anomaly_id", sa.String(36),
+                  sa.ForeignKey("metric_anomalies.id", ondelete="CASCADE"), nullable=False),
+        sa.Column("started_at", sa.DateTime(timezone=True), nullable=False),
+        sa.Column("ended_at", sa.DateTime(timezone=True), nullable=True),
+        sa.Column("peak_level", sa.Float, nullable=False),
+    )
+    op.create_index("ix_metric_anomaly_spans_anomaly_id", "metric_anomaly_spans", ["anomaly_id"])
+
+
+def downgrade() -> None:
+    op.drop_table("metric_anomaly_spans")
+    op.drop_table("metric_anomalies")

steward/models/__init__.py

@@ -6,7 +6,7 @@ from .password_resets import PasswordReset
 from .hosts import Host
 from .monitors import Monitor, MonitorResult, MonitorType
 from .metrics import PluginMetric
-from .norms import MetricNorm
+from .norms import MetricNorm, MetricAnomaly, MetricAnomalySpan
 from .alerts import AlertRule, AlertState, AlertEvent, AlertOperator, AlertStateEnum
 from .ansible import AnsibleRun, AnsibleRunStatus
 from .ansible_inventory import AnsibleTarget, AnsibleGroup, ansible_target_groups
@@ -19,7 +19,7 @@ __all__ = [
     "Base", "User", "ApiToken", "UserInvite", "PasswordReset",
     "Host",
     "Monitor", "MonitorResult", "MonitorType",
-    "PluginMetric", "MetricNorm",
+    "PluginMetric", "MetricNorm", "MetricAnomaly", "MetricAnomalySpan",
     "AlertRule", "AlertState", "AlertEvent", "AlertOperator", "AlertStateEnum",
     "AnsibleRun", "AnsibleRunStatus",
     "AnsibleTarget", "AnsibleGroup", "ansible_target_groups",

steward/models/norms.py

@@ -1,8 +1,9 @@
 from __future__ import annotations
 from datetime import datetime
-from sqlalchemy import DateTime, Float, Integer, SmallInteger, String
+from sqlalchemy import DateTime, Float, ForeignKey, Index, Integer, SmallInteger, String, Text
 from sqlalchemy.orm import Mapped, mapped_column
 from .base import Base
+from steward.core.ids import time_ordered_id
 
 
 class MetricNorm(Base):
@@ -25,3 +26,55 @@ class MetricNorm(Base):
     usual_peak: Mapped[float] = mapped_column(Float, nullable=False)
     days_seen: Mapped[int] = mapped_column(Integer, nullable=False)
     updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
+
+
+class MetricAnomaly(Base):
+    """A metric that departed from its normal (#5740): one item to review.
+
+    A departure that recurs while the item is still to review joins it as
+    another occurrence (a MetricAnomalySpan) rather than adding a row, so the
+    list stays one row per resource and metric. `review` is to_review until
+    someone marks it normal (its hours are learned) or real (they are not);
+    an ended item nobody reviewed within anomalies.review_days becomes lapsed,
+    and is not learned either."""
+    __tablename__ = "metric_anomalies"
+
+    id: Mapped[str] = mapped_column(String(36), primary_key=True, default=time_ordered_id)
+    source_module: Mapped[str] = mapped_column(String(64), nullable=False)
+    resource_name: Mapped[str] = mapped_column(String(255), nullable=False)
+    metric_name: Mapped[str] = mapped_column(String(128), nullable=False)
+    # The first occurrence's start, the latest's start, and the latest's end:
+    # NULL while an occurrence is still going.
+    started_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
+    last_started_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
+    ended_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
+    occurrences: Mapped[int] = mapped_column(Integer, nullable=False, default=1)
+    # The highest 15-minute level any occurrence held, and the normal for the
+    # hour it was reached in.
+    peak_level: Mapped[float] = mapped_column(Float, nullable=False)
+    peak_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
+    usual_hold: Mapped[float] = mapped_column(Float, nullable=False)
+    usual_peak: Mapped[float] = mapped_column(Float, nullable=False)
+    review: Mapped[str] = mapped_column(String(12), nullable=False, default="to_review")
+    reviewed_by: Mapped[str | None] = mapped_column(String(64), nullable=True)
+    reviewed_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
+    note: Mapped[str | None] = mapped_column(Text, nullable=True)
+
+    __table_args__ = (
+        Index("ix_metric_anomalies_key", "source_module", "resource_name", "metric_name"),
+        Index("ix_metric_anomalies_review_last", "review", "last_started_at"),
+    )
+
+
+class MetricAnomalySpan(Base):
+    """One occurrence of an anomaly: when the metric was out of its normal.
+    Learning skips the hours a span covers unless its anomaly was marked
+    normal, and a resource's chart shades them."""
+    __tablename__ = "metric_anomaly_spans"
+
+    id: Mapped[str] = mapped_column(String(36), primary_key=True, default=time_ordered_id)
+    anomaly_id: Mapped[str] = mapped_column(
+        String(36), ForeignKey("metric_anomalies.id", ondelete="CASCADE"), nullable=False, index=True)
+    started_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
+    ended_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
+    peak_level: Mapped[float] = mapped_column(Float, nullable=False)

steward/settings/routes.py

@@ -155,6 +155,7 @@ _RETENTION_FIELDS = [
     ("docker_logs_retention_days", "docker.logs.retention_days"),
     ("metrics_raw_days", "metrics.retention.raw_days"),
     ("metrics_rollup_days", "metrics.retention.rollup_days"),
+    ("anomalies_review_days", "anomalies.review_days"),
 ]
 
 

steward/status/routes.py

@@ -4,6 +4,7 @@ from sqlalchemy import func, select
 
 from steward.alerts.service import active_alerts, alert_label
 from steward.auth.middleware import require_role
+from steward.core import anomalies as anom
 from steward.core.status import collect_status
 from steward.models.hosts import Host
 from steward.models.users import UserRole
@@ -29,11 +30,16 @@ async def widget():
         total_hosts = (await db.execute(
             select(func.count()).select_from(Host))).scalar()
         alerts = await active_alerts(db)
+        # Metric anomalies going on now are warnings; the ones waiting for a
+        # review are one line linking to the list (#5740).
+        anomalies = await anom.views(db, await anom.open_anomalies(db))
+        to_review = await anom.to_review_count(db)
     up, down, pending = _summarize(entries)
     return await render_template(
         "status/widget.html",
         entries=entries, up=up, down=down, pending=pending,
         total_hosts=total_hosts, alerts=overview_alerts(alerts),
+        anomalies=anomalies, to_review=to_review,
         widget_id=request.args.get("wid", "0"),
     )
 

steward/templates/_anomaly.html

@@ -0,0 +1,72 @@
+{# Metric anomalies (#5740), drawn the same on Alerts → Anomalies, a resource's
+   own page and the overview widget. Import with:
+   {% from "_anomaly.html" import anomaly_row, anomaly_when with context %}
+   (with context: the review buttons read session.user_role). #}
+{% from "_macros.html" import status_badge %}
+
+{# When it happened: "since …" while going, "from – to" once ended; a repeat
+   says how often and when the latest was. #}
+{% macro anomaly_when(v) -%}
+{%- set a = v.anomaly -%}
+{%- if a.occurrences > 1 -%}
+  {{ a.occurrences }} times since {{ a.started_at|when }}, last
+  {% if v.ongoing %}since {{ a.last_started_at|when }}, ongoing{% else %}{{ a.last_started_at|when }} – {{ a.ended_at|when }}{% endif %}
+{%- elif v.ongoing -%}
+  Ongoing since {{ a.started_at|when }}
+{%- else -%}
+  {{ a.started_at|when }} – {{ a.ended_at|when }}
+{%- endif -%}
+{%- endmacro %}
+
+{# Its highest level against what is usual for that hour. #}
+{% macro anomaly_level(a) -%}
+Held {{ metric_value(a.metric_name, a.peak_level) }}; usual for that hour
+{{ metric_value(a.metric_name, a.usual_hold) }}, peaking at {{ metric_value(a.metric_name, a.usual_peak) }}
+{%- endmacro %}
+
+{# The review buttons, or how it was reviewed. `back` is the page to come
+   back to after marking. #}
+{% macro anomaly_review(a, back=none) -%}
+{%- set can_review = session.user_role in ('operator', 'admin') -%}
+{%- if a.review == "normal" -%}
+  <span>Marked normal by {{ a.reviewed_by or "someone" }}, {{ a.reviewed_at|when }}.</span>
+{%- elif a.review == "real" -%}
+  <span>Real problem, marked by {{ a.reviewed_by or "someone" }}, {{ a.reviewed_at|when }}{% if a.note %}: {{ a.note }}{% else %}.{% endif %}</span>
+{%- elif a.review == "lapsed" -%}
+  <span>Not reviewed within the review window; lapsed {{ a.reviewed_at|when }}. Its hours are not learned.</span>
+{%- endif -%}
+{%- if can_review and a.review in ("to_review", "lapsed") %}
+<form method="post" action="/alerts/anomalies/{{ a.id }}/review"
+      style="display:flex;gap:0.4rem;flex-wrap:wrap;align-items:center;margin-top:0.35rem;">
+  {{ csrf_field() }}
+  {% if back %}<input type="hidden" name="next" value="{{ back }}">{% endif %}
+  <input class="input" name="note" placeholder="Note (optional)" aria-label="Note"
+         style="max-width:220px;font-size:0.8rem;padding:0.25rem 0.5rem;">
+  <button class="btn btn-ghost btn-sm" name="review" value="normal"
+          title="Its hours are learned as normal for that time">This is normal</button>
+  <button class="btn btn-sm" name="review" value="real"
+          title="Kept out of what is learned">Real problem</button>
+</form>
+{%- endif -%}
+{%- endmacro %}
+
+{# One anomaly as a row: badge, resource and metric, when, level, review. #}
+{% macro anomaly_row(v, back=none, show_resource=true) -%}
+{%- set a = v.anomaly -%}
+<div class="anomaly-row" style="padding:0.55rem 0;border-bottom:1px solid var(--border);">
+  <div style="display:flex;gap:0.5rem;align-items:baseline;flex-wrap:wrap;">
+    {{ status_badge("anomaly") }}
+    {% if a.review != "to_review" %}{{ status_badge(a.review) }}{% endif %}
+    {% if show_resource %}
+      {% if v.link %}<a href="{{ v.link }}" style="font-weight:500;">{{ a.resource_name }}</a>
+      {% else %}<span style="font-weight:500;">{{ a.resource_name }}</span>{% endif %}
+      <span style="color:var(--text-dim);font-size:0.8rem;">{{ source_label(a.source_module) }}</span>
+    {% endif %}
+    <span>{{ metric_label(a.metric_name) }}</span>
+  </div>
+  <div style="font-size:0.8rem;color:var(--text-muted);margin-top:0.2rem;">
+    {{ anomaly_when(v) }} · {{ anomaly_level(a) }}
+  </div>
+  <div style="font-size:0.8rem;color:var(--text-muted);margin-top:0.2rem;">{{ anomaly_review(a, back) }}</div>
+</div>
+{%- endmacro %}

steward/templates/_anomaly_card.html

@@ -0,0 +1,14 @@
+{# A resource's metric anomalies (#5740), for its own page: `anomalies` are
+   anomalies.views() rows, to review first; `back` is this page's path, so a
+   review returns here. Shown only when there is one. #}
+{% from "_anomaly.html" import anomaly_row with context %}
+{% if anomalies %}
+{% set going = anomalies | selectattr("ongoing") | list %}
+<section class="card{% if going %} problem-card{% endif %}" aria-labelledby="resource-anomalies" style="margin-bottom:1rem;">
+  <h2 id="resource-anomalies" class="section-title" style="margin-bottom:0.35rem;">
+    {% if going %}Out of its normal now{% else %}Metric anomalies, last 30 days{% endif %}
+  </h2>
+  {% for v in anomalies %}{{ anomaly_row(v, back=back, show_resource=v.anomaly.resource_name != base_resource) }}{% endfor %}
+  <div style="font-size:0.78rem;margin-top:0.4rem;"><a href="/alerts/anomalies">All anomalies</a></div>
+</section>
+{% endif %}

steward/templates/alerts/anomalies.html

@@ -0,0 +1,36 @@
+{% extends "base.html" %}
+{% from "_macros.html" import crumbs, page_header %}
+{% from "_anomaly.html" import anomaly_row with context %}
+{% block title %}Metric Anomalies — Steward{% endblock %}
+{% block breadcrumb %}{{ crumbs([("Alerts", "/alerts/"), ("Anomalies", "")]) }}{% endblock %}
+{% block content %}
+{% call page_header("Metric anomalies", "A metric that left what is normal for it at that hour. Mark each normal, so its hours are learned, or a real problem, so they are not.") %}
+    <a class="btn btn-ghost" href="/alerts/history">History</a>
+{% endcall %}
+
+{% macro divided(items) %}
+{% for item in items %}
+  {% if item.opens_group %}<h3 class="table-divider" style="font-size:0.78rem;font-weight:500;color:var(--text);background:var(--bg-elevated);padding:0.35rem 0.75rem;margin:0.75rem -0.75rem 0;border-top:1px solid var(--border-mid);border-bottom:1px solid var(--border-mid);">{{ item.group }}</h3>{% endif %}
+  {{ anomaly_row(item.v, back="/alerts/anomalies") }}
+{% endfor %}
+{% endmacro %}
+
+<div class="card" style="margin-top:1rem;">
+  <h2 class="section-title" style="margin-bottom:0.35rem;">
+    {% if to_review %}{{ to_review|length }} to review{% else %}Nothing to review{% endif %}
+  </h2>
+  <p style="font-size:0.8rem;color:var(--text-muted);margin:0;">
+    An ended anomaly nobody reviews within {{ review_days }} day{{ "s" if review_days != 1 }} lapses:
+    it leaves this list, stays below as not reviewed, and is never learned.
+    The window is in Settings → Thresholds.
+  </p>
+  {{ divided(to_review) }}
+</div>
+
+{% if past %}
+<div class="card" style="margin-top:1rem;">
+  <h2 class="section-title" style="margin-bottom:0.35rem;">Reviewed and lapsed</h2>
+  {{ divided(past) }}
+</div>
+{% endif %}
+{% endblock %}

steward/templates/alerts/list.html

@@ -14,6 +14,7 @@
 {% set rules_off = rules|rejectattr("enabled")|list|length %}
 {% call page_header("Alerts") %}
     <a class="btn btn-ghost" href="/alerts/history">History</a>
+    <a class="btn btn-ghost" href="/alerts/anomalies">Anomalies</a>
     <a class="btn btn-ghost" href="/alerts/maintenance">Maintenance</a>
     {% if can_edit %}<a class="btn" href="/alerts/rules/new">New rule</a>{% endif %}
 {% endcall %}

steward/templates/email/report.html

@@ -43,6 +43,23 @@ This week {{ d.alert_events.fired }} {{ "alert" if d.alert_events.fired == 1 els
 <tr><td style="padding:0 24px;font:14px/1.5 {{ c.font }};color:{{ c.muted }};">No monitors are enabled.</td></tr>
 {% endif %}
 
+{% if anomalies.seen or anomalies.lapsed %}
+{{ section("Metric anomalies this week", base ~ "/alerts/anomalies" if base else "") }}
+{% call rows() %}
+{% for a in anomalies.seen %}
+{{ row([badge(a.tone, a.review), a.name ~ " — " ~ a.when, a.level]) }}
+{% endfor %}
+{% for a in anomalies.lapsed %}
+{{ row([badge(a.tone, a.review), a.name ~ " — lapsed unreviewed; " ~ a.when, a.level]) }}
+{% endfor %}
+{% endcall %}
+{% if anomalies.to_review %}
+<tr><td style="padding:8px 24px 0 24px;font:13px/1.5 {{ c.font }};color:{{ c.muted }};">
+{{ anomalies.to_review }} {{ "anomaly waits" if anomalies.to_review == 1 else "anomalies wait" }} to be marked normal or real.
+</td></tr>
+{% endif %}
+{% endif %}
+
 {% if d.top_routers %}
 {{ section("Busiest Traefik routers, last 24 hours", base ~ "/plugins/traefik/" if base else "") }}
 {% call rows() %}

steward/templates/email/report.txt

@@ -21,6 +21,20 @@ UPTIME OVER 7 DAYS{% if base %} ({{ base }}/monitors/){% endif %}
 {% else %}
 No monitors are enabled.
 {% endfor %}
+{% if anomalies.seen or anomalies.lapsed %}
+
+METRIC ANOMALIES THIS WEEK{% if base %} ({{ base }}/alerts/anomalies){% endif %}
+
+{% for a in anomalies.seen %}
+- {{ a.review }}: {{ a.name }}, {{ a.when }} ({{ a.level }})
+{% endfor %}
+{% for a in anomalies.lapsed %}
+- {{ a.review }}: {{ a.name }}, lapsed unreviewed; {{ a.when }} ({{ a.level }})
+{% endfor %}
+{% if anomalies.to_review %}
+{{ anomalies.to_review }} {{ "anomaly waits" if anomalies.to_review == 1 else "anomalies wait" }} to be marked normal or real.
+{% endif %}
+{% endif %}
 {% if d.top_routers %}
 
 BUSIEST TRAEFIK ROUTERS, LAST 24 HOURS{% if base %} ({{ base }}/plugins/traefik/){% endif %}

steward/templates/hosts/detail.html

@@ -44,6 +44,9 @@
 </section>
 {% endif %}
 
+{# ── Metric anomalies: out of what is normal for this host at that hour ───── #}
+{% with back="/hosts/" ~ host.id, base_resource=host.name %}{% include "_anomaly_card.html" %}{% endwith %}
+
 {# ── Live vitals strip (host_agent fragment) — full width on top, polled ────── #}
 <div id="hv-strip"
      hx-get="/plugins/host_agent/vitals/{{ host.id }}"

steward/templates/settings/thresholds.html

@@ -91,6 +91,18 @@
 </div>
 {% endmacro %}
 
+<div class="card" style="max-width:640px;margin-top:1.5rem;">
+  <h2 class="section-title" style="margin-bottom:0.5rem;">Metric anomalies</h2>
+  <p style="font-size:0.82rem;color:var(--text-muted);margin-bottom:1.25rem;">
+    A metric that leaves what is normal for it at that hour is listed under
+    Alerts → Anomalies to review. One nobody marks normal or real within this
+    window, once it has ended, lapses: it leaves the list and is kept as not
+    reviewed, and its hours are never learned.
+  </p>
+  {{ days("Review window", "anomalies_review_days", "anomalies.review_days",
+          "An anomaly still going never lapses.") }}
+</div>
+
 <div class="card" style="max-width:640px;margin-top:1.5rem;">
   <h2 class="section-title" style="margin-bottom:0.5rem;">Docker data retention</h2>
   <p style="font-size:0.82rem;color:var(--text-muted);margin-bottom:1.25rem;">

steward/templates/status/widget.html

@@ -3,6 +3,7 @@
    widget is the full list of checks; this is the summary. #}
 {% from "_macros.html" import status_badge %}
 {% set firing = alerts | selectattr("label", "in", ["firing", "no_data"]) | list %}
+{% set anomalies = anomalies or [] %}
 <div style="display:flex;flex-wrap:wrap;gap:0.4rem 1.5rem;align-items:baseline;margin-bottom:0.6rem;">
   <span>
     {% if session.user_id %}<a href="/hosts/" style="color:inherit;text-decoration:none;">{% endif %}
@@ -24,7 +25,7 @@
 </div>
 
 {% set problems = entries | rejectattr("status", "equalto", "up") | list %}
-{% if not problems and not alerts %}
+{% if not problems and not alerts and not anomalies %}
 <div class="tone-ok" style="font-size:0.85rem;padding:0.3rem 0;">
   {% if entries %}✓ All {{ up }} monitors are up and no alert is firing.{% else %}No monitors configured, and no alert is firing.{% endif %}
 </div>
@@ -54,6 +55,18 @@
     {% if e.since %}<span style="color:var(--text-dim);font-size:0.75rem;white-space:nowrap;">{% if e.since_at_least %}over {% endif %}{{ e.since|ago }}</span>{% endif %}
   </div>
   {% endfor %}
+  {# Metric anomalies going on now (#5740): a warning each. #}
+  {% for v in anomalies[:4] %}
+  <div style="display:flex;align-items:center;gap:0.5rem;padding:0.2rem 0;font-size:0.82rem;">
+    {{ status_badge("anomaly") }}
+    {% set nm_style = "flex:1;min-width:0;white-space:nowrap;overflow:hidden;text-overflow:ellipsis;" %}
+    {% set nm = v.anomaly.resource_name ~ " · " ~ metric_label(v.anomaly.metric_name) %}
+    {% if session.user_id %}
+    <a href="/alerts/anomalies" title="{{ nm }}: {{ metric_value(v.anomaly.metric_name, v.anomaly.peak_level) }}, usual {{ metric_value(v.anomaly.metric_name, v.anomaly.usual_hold) }}" style="{{ nm_style }}color:inherit;text-decoration:none;">{{ nm }}</a>
+    {% else %}<span title="{{ nm }}" style="{{ nm_style }}">{{ nm }}</span>{% endif %}
+    <span style="color:var(--text-dim);font-size:0.75rem;white-space:nowrap;">{{ v.anomaly.last_started_at|ago }}</span>
+  </div>
+  {% endfor %}
   {# What didn't fit, each linking to its own list (#5510). #}
   {% set more_alerts, more_monitors = alerts|length - 6, problems|length - 8 %}
   {% if more_alerts > 0 or more_monitors > 0 %}
@@ -65,3 +78,9 @@
   {% endif %}
 </div>
 {% endif %}
+{% if to_review %}
+<div style="font-size:0.78rem;color:var(--text-muted);padding-top:0.3rem;">
+  {% if session.user_id %}<a href="/alerts/anomalies">{{ to_review }} metric anomal{{ "y" if to_review == 1 else "ies" }} to review</a>
+  {% else %}{{ to_review }} metric anomal{{ "y" if to_review == 1 else "ies" }} to review{% endif %}
+</div>
+{% endif %}