feat(norms): hosts, containers and monitor response times are graded against their own normal (#5742)
Host CPU, memory and load per core, container CPU and memory, and every monitor's response time learn a normal per hour (milestone 513, System 182). A reading past its fixed cutoff but within its normal reads healthy and says what is usual at this hour; one well out of it keeps its grade and is a metric anomaly. Disk, a monitor down and an agent offline keep their cutoffs. - Each normal's floor, the smallest departure that counts, is a setting: Settings -> Thresholds -> Normals (norms.floor.*). Defaults: host CPU 10 points, host memory 5, load per core 10, container CPU 10 points of one core, container memory 5, monitor response 100 ms; Traefik's 2-point 4xx and 50 ms p95 floors move there from constants. learn_normal takes a floor function (floor_setting) read at each pass. - learn_normal(parts=False): a host learns, its cores don't. - Ingest records load_pct (load_1m / cores x 100), the figure pages grade. - Shared grading: anomalies.against / graded, worst_reading(normals=), hosts' host_grades, docker usage(thresholds=, norms=), monitors' latency_grade. Applied to host vitals, the Hosts widget, MCP overview, Docker usage, and the monitor list, widget and page. - The monitor and container pages list their anomalies. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -6,12 +6,12 @@ Host (host_id) or watch a free-standing custom destination (host_id = NULL).
|
||||
"""
|
||||
from __future__ import annotations
|
||||
import uuid
|
||||
from datetime import datetime, timezone
|
||||
from datetime import datetime, timedelta, timezone
|
||||
|
||||
from quart import Blueprint, current_app, render_template, request, redirect, url_for
|
||||
|
||||
from steward.auth.middleware import require_role
|
||||
from steward.core import renames
|
||||
from steward.core import anomalies as anom, renames
|
||||
from steward.core.audit import audit_request
|
||||
from steward.models.users import UserRole
|
||||
from steward.models.monitors import Monitor, MonitorType, MONITOR_TYPE_VALUES
|
||||
@@ -125,6 +125,10 @@ async def detail(monitor_id: str):
|
||||
st = (await monitor_svc.monitor_states(db, [m]))[m.id]
|
||||
failures = await monitor_svc.recent_failures(db, m.id)
|
||||
history = await monitor_svc.response_history(db, m.id, datetime.now(timezone.utc))
|
||||
norm = (await monitor_svc.latency_norms(db, [m])).get(m.id)
|
||||
anomalies = await anom.views(db, await anom.list_anomalies(
|
||||
db, source=m.type, resource=m.name,
|
||||
since=datetime.now(timezone.utc) - timedelta(days=30), limit=20), links=False)
|
||||
host = await db.get(Host, m.host_id) if m.host_id else None
|
||||
thresholds = current_app.config.get("THRESHOLDS", {})
|
||||
latest = st["latest"]
|
||||
@@ -136,7 +140,8 @@ async def detail(monitor_id: str):
|
||||
tls_days = (latest.tls_expires_at - datetime.now(timezone.utc)).total_seconds() / 86400.0
|
||||
return await render_template(
|
||||
"monitors/detail.html", monitor=m, host=host, latest=latest, failures=failures,
|
||||
state=monitor_svc.row_state(m, latest, st["uptime"].get("24h"), thresholds),
|
||||
state=monitor_svc.row_state(m, latest, st["uptime"].get("24h"), thresholds, norm),
|
||||
latency=monitor_svc.latency_grade(latest, thresholds, norm), anomalies=anomalies,
|
||||
since=since, since_at_least=since_at_least, uptime=st["uptime"], tls_days=tls_days,
|
||||
heartbeat=[{"state": "up" if r.is_up else "down", "title": _heartbeat_title(m.type, r)}
|
||||
for r in st["recent"]],
|
||||
|
||||
Reference in New Issue
Block a user