feat(norms): hosts, containers and monitor response times are graded against their own normal (#5742)
Host CPU, memory and load per core, container CPU and memory, and every monitor's response time learn a normal per hour (milestone 513, System 182). A reading past its fixed cutoff but within its normal reads healthy and says what is usual at this hour; one well out of it keeps its grade and is a metric anomaly. Disk, a monitor down and an agent offline keep their cutoffs. - Each normal's floor, the smallest departure that counts, is a setting: Settings -> Thresholds -> Normals (norms.floor.*). Defaults: host CPU 10 points, host memory 5, load per core 10, container CPU 10 points of one core, container memory 5, monitor response 100 ms; Traefik's 2-point 4xx and 50 ms p95 floors move there from constants. learn_normal takes a floor function (floor_setting) read at each pass. - learn_normal(parts=False): a host learns, its cores don't. - Ingest records load_pct (load_1m / cores x 100), the figure pages grade. - Shared grading: anomalies.against / graded, worst_reading(normals=), hosts' host_grades, docker usage(thresholds=, norms=), monitors' latency_grade. Applied to host vitals, the Hosts widget, MCP overview, Docker usage, and the monitor list, widget and page. - The monitor and container pages list their anomalies. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -21,6 +21,7 @@ from steward.core.health import ingest as ingest_tally
|
||||
from steward.core.metric_series import points as metric_points
|
||||
from steward.core.settings import get_setting, public_base_url
|
||||
from steward.core.time_range import parse_range, RANGE_OPTIONS
|
||||
from steward.hosts import service as host_svc
|
||||
from steward.models.hosts import Host
|
||||
from steward.models.metrics import PluginMetric
|
||||
from .models import HostAgentRegistration
|
||||
@@ -118,6 +119,10 @@ def _expand_sample_to_metrics(
|
||||
for key in ("1m", "5m", "15m"):
|
||||
if key in load:
|
||||
row(f"load_{key}", host_name, load[key])
|
||||
# Load per core as a %, the figure the pages grade, so it can learn a
|
||||
# normal (#5742): a load of 6 is idle on 32 cores and busy on 4.
|
||||
if "1m" in load and sample.get("cpu_cores"):
|
||||
row("load_pct", host_name, 100.0 * load["1m"] / len(sample["cpu_cores"]))
|
||||
|
||||
if sample.get("uptime_secs") is not None:
|
||||
row("uptime_secs", host_name, sample["uptime_secs"])
|
||||
@@ -725,6 +730,7 @@ async def host_vitals(host_id: str):
|
||||
reg = (await db.execute(select(HostAgentRegistration).where(
|
||||
HostAgentRegistration.host_id == host_id))).scalar_one_or_none()
|
||||
latest = await _latest_metrics_for_host(db, host.name) if reg else {}
|
||||
norms = await host_svc.host_norms(db, host.name) if reg else {}
|
||||
|
||||
# The last 6 h for the at-a-glance sparklines (cpu/mem/load host-level,
|
||||
# disk from the root mount), averaged into ~120 points in SQL, so the
|
||||
@@ -759,13 +765,23 @@ async def host_vitals(host_id: str):
|
||||
reporting = bool(reg) and ls is not None
|
||||
stale = reporting and (datetime.now(timezone.utc) - ls).total_seconds() > _stale_after_seconds()
|
||||
cpu, mem = hostlvl.get("cpu_pct"), hostlvl.get("mem_used_pct")
|
||||
thresholds = current_app.config.get("THRESHOLDS", {})
|
||||
worst = worst_reading((("CPU", cpu, "cpu"), ("Memory", mem, "mem"), ("Disk /", disk_root, "disk"),
|
||||
("Load per core", load_per_core, "load")),
|
||||
current_app.config.get("THRESHOLDS", {}))
|
||||
thresholds, host_svc.host_normals(norms, thresholds))
|
||||
# Each tile's grade; CPU, memory and load within the host's own normal
|
||||
# for the hour read healthy, and say what is usual (#5742).
|
||||
grades = host_svc.host_grades({"cpu": cpu, "mem": mem, "disk": disk_root, "load": load_per_core},
|
||||
norms, thresholds)
|
||||
usual = [f"{label} {value:.0f}% is {grades[kind][1]}"
|
||||
for label, value, kind in (("CPU", cpu, "cpu"), ("Memory", mem, "mem"),
|
||||
("load per core", load_per_core, "load"))
|
||||
if grades[kind][1]]
|
||||
return await render_template(
|
||||
"_host_vitals.html",
|
||||
reg=reg, cpu=cpu, mem=mem, disk_root=disk_root, load_per_core=load_per_core,
|
||||
sparks=sparks, psi=psi, reporting=reporting, stale=stale, worst=worst,
|
||||
grades=grades, usual=usual,
|
||||
clock=clock_words(reg.clock_offset_s if reg else None, current_app.config.get("THRESHOLDS", {})),
|
||||
)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user