feat(anomalies): a metric out of its normal is an anomaly to review (#5740)

#5740 "A metric that departs from its normal is recorded as an anomaly to review, and is findable after it ends", milestone 513. - Every minute each learned key's last 15 minutes is judged against its hour's normal: above the usual hold x1.25 and by the metric's floor opens an occurrence, which ends when the level comes back. - metric_anomalies holds one row per item to review; a repeat while it still waits joins it as an occurrence (metric_anomaly_spans). - An ended row nobody reviews within anomalies.review_days (14, Settings -> Thresholds) lapses; an ongoing one never does. - Learning skips every hour an occurrence covers unless its anomaly was marked normal: no normal is learned without review. - Alerts -> Anomalies lists those to review first, then the reviewed and lapsed under day dividers; marking is operator/admin and audited. - The Overview widget and get_overview warn of open ones and count those to review; the host page shows its own; the weekly report lists the week's and the lapsed; MCP list_anomalies and review_anomaly. - Reviewed and lapsed rows are pruned at data.retention_days, and kept in backups like alert history. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-14 13:41:57 -04:00
parent cddf7fc6c4
commit 5569f9934e
34 changed files with 1343 additions and 25 deletions
@@ -0,0 +1,72 @@
+{# Metric anomalies (#5740), drawn the same on Alerts → Anomalies, a resource's
+   own page and the overview widget. Import with:
+   {% from "_anomaly.html" import anomaly_row, anomaly_when with context %}
+   (with context: the review buttons read session.user_role). #}
+{% from "_macros.html" import status_badge %}
+
+{# When it happened: "since …" while going, "from – to" once ended; a repeat
+   says how often and when the latest was. #}
+{% macro anomaly_when(v) -%}
+{%- set a = v.anomaly -%}
+{%- if a.occurrences > 1 -%}
+  {{ a.occurrences }} times since {{ a.started_at|when }}, last
+  {% if v.ongoing %}since {{ a.last_started_at|when }}, ongoing{% else %}{{ a.last_started_at|when }} – {{ a.ended_at|when }}{% endif %}
+{%- elif v.ongoing -%}
+  Ongoing since {{ a.started_at|when }}
+{%- else -%}
+  {{ a.started_at|when }} – {{ a.ended_at|when }}
+{%- endif -%}
+{%- endmacro %}
+
+{# Its highest level against what is usual for that hour. #}
+{% macro anomaly_level(a) -%}
+Held {{ metric_value(a.metric_name, a.peak_level) }}; usual for that hour
+{{ metric_value(a.metric_name, a.usual_hold) }}, peaking at {{ metric_value(a.metric_name, a.usual_peak) }}
+{%- endmacro %}
+
+{# The review buttons, or how it was reviewed. `back` is the page to come
+   back to after marking. #}
+{% macro anomaly_review(a, back=none) -%}
+{%- set can_review = session.user_role in ('operator', 'admin') -%}
+{%- if a.review == "normal" -%}
+  <span>Marked normal by {{ a.reviewed_by or "someone" }}, {{ a.reviewed_at|when }}.</span>
+{%- elif a.review == "real" -%}
+  <span>Real problem, marked by {{ a.reviewed_by or "someone" }}, {{ a.reviewed_at|when }}{% if a.note %}: {{ a.note }}{% else %}.{% endif %}</span>
+{%- elif a.review == "lapsed" -%}
+  <span>Not reviewed within the review window; lapsed {{ a.reviewed_at|when }}. Its hours are not learned.</span>
+{%- endif -%}
+{%- if can_review and a.review in ("to_review", "lapsed") %}
+<form method="post" action="/alerts/anomalies/{{ a.id }}/review"
+      style="display:flex;gap:0.4rem;flex-wrap:wrap;align-items:center;margin-top:0.35rem;">
+  {{ csrf_field() }}
+  {% if back %}<input type="hidden" name="next" value="{{ back }}">{% endif %}
+  <input class="input" name="note" placeholder="Note (optional)" aria-label="Note"
+         style="max-width:220px;font-size:0.8rem;padding:0.25rem 0.5rem;">
+  <button class="btn btn-ghost btn-sm" name="review" value="normal"
+          title="Its hours are learned as normal for that time">This is normal</button>
+  <button class="btn btn-sm" name="review" value="real"
+          title="Kept out of what is learned">Real problem</button>
+</form>
+{%- endif -%}
+{%- endmacro %}
+
+{# One anomaly as a row: badge, resource and metric, when, level, review. #}
+{% macro anomaly_row(v, back=none, show_resource=true) -%}
+{%- set a = v.anomaly -%}
+<div class="anomaly-row" style="padding:0.55rem 0;border-bottom:1px solid var(--border);">
+  <div style="display:flex;gap:0.5rem;align-items:baseline;flex-wrap:wrap;">
+    {{ status_badge("anomaly") }}
+    {% if a.review != "to_review" %}{{ status_badge(a.review) }}{% endif %}
+    {% if show_resource %}
+      {% if v.link %}<a href="{{ v.link }}" style="font-weight:500;">{{ a.resource_name }}</a>
+      {% else %}<span style="font-weight:500;">{{ a.resource_name }}</span>{% endif %}
+      <span style="color:var(--text-dim);font-size:0.8rem;">{{ source_label(a.source_module) }}</span>
+    {% endif %}
+    <span>{{ metric_label(a.metric_name) }}</span>
+  </div>
+  <div style="font-size:0.8rem;color:var(--text-muted);margin-top:0.2rem;">
+    {{ anomaly_when(v) }} · {{ anomaly_level(a) }}
+  </div>
+  <div style="font-size:0.8rem;color:var(--text-muted);margin-top:0.2rem;">{{ anomaly_review(a, back) }}</div>
+</div>
+{%- endmacro %}
@@ -0,0 +1,14 @@
+{# A resource's metric anomalies (#5740), for its own page: `anomalies` are
+   anomalies.views() rows, to review first; `back` is this page's path, so a
+   review returns here. Shown only when there is one. #}
+{% from "_anomaly.html" import anomaly_row with context %}
+{% if anomalies %}
+{% set going = anomalies | selectattr("ongoing") | list %}
+<section class="card{% if going %} problem-card{% endif %}" aria-labelledby="resource-anomalies" style="margin-bottom:1rem;">
+  <h2 id="resource-anomalies" class="section-title" style="margin-bottom:0.35rem;">
+    {% if going %}Out of its normal now{% else %}Metric anomalies, last 30 days{% endif %}
+  </h2>
+  {% for v in anomalies %}{{ anomaly_row(v, back=back, show_resource=v.anomaly.resource_name != base_resource) }}{% endfor %}
+  <div style="font-size:0.78rem;margin-top:0.4rem;"><a href="/alerts/anomalies">All anomalies</a></div>
+</section>
+{% endif %}
@@ -0,0 +1,36 @@
+{% extends "base.html" %}
+{% from "_macros.html" import crumbs, page_header %}
+{% from "_anomaly.html" import anomaly_row with context %}
+{% block title %}Metric Anomalies — Steward{% endblock %}
+{% block breadcrumb %}{{ crumbs([("Alerts", "/alerts/"), ("Anomalies", "")]) }}{% endblock %}
+{% block content %}
+{% call page_header("Metric anomalies", "A metric that left what is normal for it at that hour. Mark each normal, so its hours are learned, or a real problem, so they are not.") %}
+    <a class="btn btn-ghost" href="/alerts/history">History</a>
+{% endcall %}
+
+{% macro divided(items) %}
+{% for item in items %}
+  {% if item.opens_group %}<h3 class="table-divider" style="font-size:0.78rem;font-weight:500;color:var(--text);background:var(--bg-elevated);padding:0.35rem 0.75rem;margin:0.75rem -0.75rem 0;border-top:1px solid var(--border-mid);border-bottom:1px solid var(--border-mid);">{{ item.group }}</h3>{% endif %}
+  {{ anomaly_row(item.v, back="/alerts/anomalies") }}
+{% endfor %}
+{% endmacro %}
+
+<div class="card" style="margin-top:1rem;">
+  <h2 class="section-title" style="margin-bottom:0.35rem;">
+    {% if to_review %}{{ to_review|length }} to review{% else %}Nothing to review{% endif %}
+  </h2>
+  <p style="font-size:0.8rem;color:var(--text-muted);margin:0;">
+    An ended anomaly nobody reviews within {{ review_days }} day{{ "s" if review_days != 1 }} lapses:
+    it leaves this list, stays below as not reviewed, and is never learned.
+    The window is in Settings → Thresholds.
+  </p>
+  {{ divided(to_review) }}
+</div>
+
+{% if past %}
+<div class="card" style="margin-top:1rem;">
+  <h2 class="section-title" style="margin-bottom:0.35rem;">Reviewed and lapsed</h2>
+  {{ divided(past) }}
+</div>
+{% endif %}
+{% endblock %}
@@ -14,6 +14,7 @@
 {% set rules_off = rules|rejectattr("enabled")|list|length %}
 {% call page_header("Alerts") %}
    <a class="btn btn-ghost" href="/alerts/history">History</a>
+    <a class="btn btn-ghost" href="/alerts/anomalies">Anomalies</a>
    <a class="btn btn-ghost" href="/alerts/maintenance">Maintenance</a>
    {% if can_edit %}<a class="btn" href="/alerts/rules/new">New rule</a>{% endif %}
 {% endcall %}
@@ -43,6 +43,23 @@ This week {{ d.alert_events.fired }} {{ "alert" if d.alert_events.fired == 1 els
 <tr><td style="padding:0 24px;font:14px/1.5 {{ c.font }};color:{{ c.muted }};">No monitors are enabled.</td></tr>
 {% endif %}

+{% if anomalies.seen or anomalies.lapsed %}
+{{ section("Metric anomalies this week", base ~ "/alerts/anomalies" if base else "") }}
+{% call rows() %}
+{% for a in anomalies.seen %}
+{{ row([badge(a.tone, a.review), a.name ~ " — " ~ a.when, a.level]) }}
+{% endfor %}
+{% for a in anomalies.lapsed %}
+{{ row([badge(a.tone, a.review), a.name ~ " — lapsed unreviewed; " ~ a.when, a.level]) }}
+{% endfor %}
+{% endcall %}
+{% if anomalies.to_review %}
+<tr><td style="padding:8px 24px 0 24px;font:13px/1.5 {{ c.font }};color:{{ c.muted }};">
+{{ anomalies.to_review }} {{ "anomaly waits" if anomalies.to_review == 1 else "anomalies wait" }} to be marked normal or real.
+</td></tr>
+{% endif %}
+{% endif %}
+
 {% if d.top_routers %}
 {{ section("Busiest Traefik routers, last 24 hours", base ~ "/plugins/traefik/" if base else "") }}
 {% call rows() %}
@@ -21,6 +21,20 @@ UPTIME OVER 7 DAYS{% if base %} ({{ base }}/monitors/){% endif %}
 {% else %}
 No monitors are enabled.
 {% endfor %}
+{% if anomalies.seen or anomalies.lapsed %}
+
+METRIC ANOMALIES THIS WEEK{% if base %} ({{ base }}/alerts/anomalies){% endif %}
+
+{% for a in anomalies.seen %}
+- {{ a.review }}: {{ a.name }}, {{ a.when }} ({{ a.level }})
+{% endfor %}
+{% for a in anomalies.lapsed %}
+- {{ a.review }}: {{ a.name }}, lapsed unreviewed; {{ a.when }} ({{ a.level }})
+{% endfor %}
+{% if anomalies.to_review %}
+{{ anomalies.to_review }} {{ "anomaly waits" if anomalies.to_review == 1 else "anomalies wait" }} to be marked normal or real.
+{% endif %}
+{% endif %}
 {% if d.top_routers %}

 BUSIEST TRAEFIK ROUTERS, LAST 24 HOURS{% if base %} ({{ base }}/plugins/traefik/){% endif %}
@@ -44,6 +44,9 @@
 </section>
 {% endif %}

+{# ── Metric anomalies: out of what is normal for this host at that hour ───── #}
+{% with back="/hosts/" ~ host.id, base_resource=host.name %}{% include "_anomaly_card.html" %}{% endwith %}
+
 {# ── Live vitals strip (host_agent fragment) — full width on top, polled ────── #}
 <div id="hv-strip"
     hx-get="/plugins/host_agent/vitals/{{ host.id }}"
@@ -91,6 +91,18 @@
 </div>
 {% endmacro %}

+<div class="card" style="max-width:640px;margin-top:1.5rem;">
+  <h2 class="section-title" style="margin-bottom:0.5rem;">Metric anomalies</h2>
+  <p style="font-size:0.82rem;color:var(--text-muted);margin-bottom:1.25rem;">
+    A metric that leaves what is normal for it at that hour is listed under
+    Alerts → Anomalies to review. One nobody marks normal or real within this
+    window, once it has ended, lapses: it leaves the list and is kept as not
+    reviewed, and its hours are never learned.
+  </p>
+  {{ days("Review window", "anomalies_review_days", "anomalies.review_days",
+          "An anomaly still going never lapses.") }}
+</div>
+
 <div class="card" style="max-width:640px;margin-top:1.5rem;">
  <h2 class="section-title" style="margin-bottom:0.5rem;">Docker data retention</h2>
  <p style="font-size:0.82rem;color:var(--text-muted);margin-bottom:1.25rem;">
@@ -3,6 +3,7 @@
   widget is the full list of checks; this is the summary. #}
 {% from "_macros.html" import status_badge %}
 {% set firing = alerts | selectattr("label", "in", ["firing", "no_data"]) | list %}
+{% set anomalies = anomalies or [] %}
 <div style="display:flex;flex-wrap:wrap;gap:0.4rem 1.5rem;align-items:baseline;margin-bottom:0.6rem;">
  <span>
    {% if session.user_id %}<a href="/hosts/" style="color:inherit;text-decoration:none;">{% endif %}
@@ -24,7 +25,7 @@
 </div>

 {% set problems = entries | rejectattr("status", "equalto", "up") | list %}
-{% if not problems and not alerts %}
+{% if not problems and not alerts and not anomalies %}
 <div class="tone-ok" style="font-size:0.85rem;padding:0.3rem 0;">
  {% if entries %}✓ All {{ up }} monitors are up and no alert is firing.{% else %}No monitors configured, and no alert is firing.{% endif %}
 </div>
@@ -54,6 +55,18 @@
    {% if e.since %}<span style="color:var(--text-dim);font-size:0.75rem;white-space:nowrap;">{% if e.since_at_least %}over {% endif %}{{ e.since|ago }}</span>{% endif %}
  </div>
  {% endfor %}
+  {# Metric anomalies going on now (#5740): a warning each. #}
+  {% for v in anomalies[:4] %}
+  <div style="display:flex;align-items:center;gap:0.5rem;padding:0.2rem 0;font-size:0.82rem;">
+    {{ status_badge("anomaly") }}
+    {% set nm_style = "flex:1;min-width:0;white-space:nowrap;overflow:hidden;text-overflow:ellipsis;" %}
+    {% set nm = v.anomaly.resource_name ~ " · " ~ metric_label(v.anomaly.metric_name) %}
+    {% if session.user_id %}
+    <a href="/alerts/anomalies" title="{{ nm }}: {{ metric_value(v.anomaly.metric_name, v.anomaly.peak_level) }}, usual {{ metric_value(v.anomaly.metric_name, v.anomaly.usual_hold) }}" style="{{ nm_style }}color:inherit;text-decoration:none;">{{ nm }}</a>
+    {% else %}<span title="{{ nm }}" style="{{ nm_style }}">{{ nm }}</span>{% endif %}
+    <span style="color:var(--text-dim);font-size:0.75rem;white-space:nowrap;">{{ v.anomaly.last_started_at|ago }}</span>
+  </div>
+  {% endfor %}
  {# What didn't fit, each linking to its own list (#5510). #}
  {% set more_alerts, more_monitors = alerts|length - 6, problems|length - 8 %}
  {% if more_alerts > 0 or more_monitors > 0 %}
@@ -65,3 +78,9 @@
  {% endif %}
 </div>
 {% endif %}
+{% if to_review %}
+<div style="font-size:0.78rem;color:var(--text-muted);padding-top:0.3rem;">
+  {% if session.user_id %}<a href="/alerts/anomalies">{{ to_review }} metric anomal{{ "y" if to_review == 1 else "ies" }} to review</a>
+  {% else %}{{ to_review }} metric anomal{{ "y" if to_review == 1 else "ies" }} to review{% endif %}
+</div>
+{% endif %}