diff --git a/docs/architecture.md b/docs/architecture.md index fee63c6..18946c5 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -111,6 +111,22 @@ Every minute, `metric_anomalies` averages each learned key's last 15 minutes and - Learning skips every hour an occurrence covers unless its anomaly was marked `normal`, so a new level is never learned without review. - `data_cleanup` prunes reviewed and lapsed rows at `data.retention_days`; a row still to review is never pruned. +What learns a normal, and the floor each departure must clear (Settings → Thresholds → Normals, `norms.floor.*`; a floor may be a function, `floor_setting()`, read at each pass): + +| Kind | Metric | Floor (default) | +|---|---|---| +| Host CPU | `host_agent` / `cpu_pct` | 10 points | +| Host memory | `host_agent` / `mem_used_pct` | 5 points | +| Host load per core | `host_agent` / `load_pct` (load_1m ÷ cores × 100, recorded at ingest) | 10 points | +| Container CPU | `docker` / `cpu_pct` (100% is one core) | 10 points of one core | +| Container memory | `docker` / `mem_pct` (of its limit, or of host RAM when it has none) | 5 points | +| Monitor response time | `` / `response_ms` | 100 ms | +| Router 4xx share, p95 | `traefik` / `error_rate_4xx_pct`, `latency_p95_ms` | 2 points, 50 ms | + +Host metrics learn with `parts=False`: the host learns, its cores (`:core`) don't. Disk space, a monitor that is down, an agent gone quiet and a router's 5xx share are failures and keep their fixed cutoffs. + +Pages grade by the normal through `anomalies.graded()` (hosts' `host_grades` and `worst_reading(normals=)`, `docker.service.usage(thresholds=, norms=)`, `monitors.service.latency_grade`): a reading past its cutoff but within its normal reads healthy and says what is usual at this hour. The host, monitor, container and Traefik pages list their anomalies with `_anomaly_card.html`. + --- ## Database Session Pattern diff --git a/docs/plugins/traefik.md b/docs/plugins/traefik.md index 50c9c01..1981136 100644 --- a/docs/plugins/traefik.md +++ b/docs/plugins/traefik.md @@ -89,7 +89,7 @@ The full Traefik page at `/plugins/traefik/` reads top to bottom, for the time r A router is graded on its last 15 minutes against Settings → Thresholds. When a problem started is read from the router's history over the range. -Once a router has 5 days of history, its 4xx share and p95 latency are judged against its own normal for that hour (see [Normals and anomalies](../architecture.md)) instead of the fixed cutoffs. One over the cutoff but within its normal is healthy, and the healthy table says it is usual for that router (an error-pages service that answers 4xx all day, a login wall's steady 401s). One out of its normal is a warning that says what is usual, and a metric anomaly to review; the Traefik page lists the routers' anomalies of the last 30 days. A 4xx share over fewer requests in the 15 minutes than Settings → Thresholds allows (30 by default) isn't judged at all. The 5xx share is a failure and always keeps its fixed cutoff. Latency is learned only from #5755's per-scrape figures, so its normal arrives 5 days after that release. +Once a router has 5 days of history, its 4xx share and p95 latency are judged against its own normal for that hour (see [Normals and anomalies](../architecture.md)) instead of the fixed cutoffs. One over the cutoff but within its normal is healthy, and the healthy table says it is usual for that router (an error-pages service that answers 4xx all day, a login wall's steady 401s). One out of its normal is a warning that says what is usual, and a metric anomaly to review; the Traefik page lists the routers' anomalies of the last 30 days. How far out counts is set on Settings → Thresholds → Normals (2 points of 4xx share, 50 ms of p95 by default). A 4xx share over fewer requests in the 15 minutes than Settings → Thresholds allows (30 by default) isn't judged at all. The 5xx share is a failure and always keeps its fixed cutoff. Latency is learned only from #5755's per-scrape figures, so its normal arrives 5 days after that release. --- diff --git a/plugins/docker/__init__.py b/plugins/docker/__init__.py index 38e1c13..641cc0f 100644 --- a/plugins/docker/__init__.py +++ b/plugins/docker/__init__.py @@ -43,6 +43,11 @@ def setup(app: "Quart") -> None: description="Keep each container's logs within the age window and byte cap.", required_role=UserRole.viewer, ) + # Each container's CPU (100% is one core) and memory share learn their + # normal (#5742), each with its floor from Settings → Thresholds. + from steward.core.norms import floor_setting, learn_normal + learn_normal("docker", "cpu_pct", floor_setting(app, "container_cpu_pct")) + learn_normal("docker", "mem_pct", floor_setting(app, "container_mem_pct")) def get_scheduled_tasks() -> list: diff --git a/plugins/docker/routes.py b/plugins/docker/routes.py index a76ff59..91d09a5 100644 --- a/plugins/docker/routes.py +++ b/plugins/docker/routes.py @@ -12,6 +12,7 @@ from sqlalchemy import func, select from steward.auth.middleware import require_role from steward.models.users import UserRole from steward.models.hosts import Host +from steward.core import anomalies as anom from steward.core.formatting import human_bytes, human_duration, seconds_since from steward.core.time_range import parse_range, DEFAULT_RANGE from . import service as docker_svc @@ -159,16 +160,20 @@ async def rows(): host_name = {h.id: h.name for h in hosts.values()} swarm_view = ctx["swarm_view"] capacity = await docker_svc.host_capacity(db, {c.host_id for c in containers}) + norms = await docker_svc.container_norms( + db, {(c.host_id, c.name) for c in containers if not c.service_name}, now) failures = docker_svc.failures_24h( await docker_svc.failure_events(db, now - timedelta(hours=24)), swarm_view["services"], host_name) + thresholds = current_app.config.get("THRESHOLDS", {}) service_rows = [service_row(s, now) for s in swarm_view["services"]] standalone = [c for c in containers if not c.service_name] container_rows = [ container_row( c, label=docker_svc.display_name(c), host=host_name.get(c.host_id, c.host_id), - usage=docker_svc.usage(c, *capacity.get(c.host_id, (None, None))), + usage=docker_svc.usage(c, *capacity.get(c.host_id, (None, None)), thresholds=thresholds, + norms=norms.get((c.host_id, c.name))), ports=json.loads(c.ports_json) if c.ports_json else [], stale=_is_stale(c, now, stale_after, docker_down), exit_failed=docker_svc.exit_is_failure(c.exit_code, c.oom_killed), now=now) @@ -314,6 +319,8 @@ async def widget_resources(): db, {(c.host_id, c.name) for c in busiest}, slots[0], width) hosts = await docker_svc.host_map(db, {c.host_id for c in busiest}) capacity = await docker_svc.host_capacity(db, {c.host_id for c in busiest}) + norms = await docker_svc.container_norms(db, {(c.host_id, c.name) for c in busiest}) + thresholds = current_app.config.get("THRESHOLDS", {}) def bars(c, i, scale, floor): seen = history.get((c.host_id, c.name), {}) @@ -325,7 +332,8 @@ async def widget_resources(): "c": c, "label": docker_svc.display_name(c), "host_name": hosts[c.host_id].name if c.host_id in hosts else c.host_id, - "usage": docker_svc.usage(c, *capacity.get(c.host_id, (None, None)), cpu_pct=cpu_of(c)), + "usage": docker_svc.usage(c, *capacity.get(c.host_id, (None, None)), cpu_pct=cpu_of(c), + thresholds=thresholds, norms=norms.get((c.host_id, c.name))), "cpu_bars": bars(c, 0, 0.01, cpu_floor), # cpu_pct → cores "mem_bars": bars(c, 1, 1.0, mem_floor * 2**30), # bytes; floor in GiB } for c in busiest] @@ -365,7 +373,9 @@ async def host_panel(host_id: str): nodes = list((await db.execute(select(DockerSwarmNode))).scalars()) failures = await docker_svc.recent_failures_by_container(db, now - timedelta(hours=24)) capacity = await docker_svc.host_capacity(db, {host_id}) + norms = await docker_svc.container_norms(db, {(c.host_id, c.name) for c in containers}, now) + thresholds = current_app.config.get("THRESHOLDS", {}) replaced = replaced_tasks(containers, services) node_hostname = {n.node_id: n.hostname for n in nodes if n.hostname} hour_ago = now - timedelta(hours=1) @@ -380,7 +390,8 @@ async def host_panel(host_id: str): rows.append({ "c": c, "label": docker_svc.task_label(c, node_hostname), "problems": problems, "since": docker_svc.problem_since(problems, events, hour_ago), - "usage": docker_svc.usage(c, *capacity.get(host_id, (None, None))), + "usage": docker_svc.usage(c, *capacity.get(host_id, (None, None)), thresholds=thresholds, + norms=norms.get((c.host_id, c.name))), "uptime": _uptime(c.started_at) if c.status == "running" else None, }) shown, folded = host_panel_rows(rows) @@ -445,6 +456,13 @@ async def container_detail(host_id: str, name: str): )).scalar() cpus, mem_total = (await docker_svc.host_capacity(db, [host_id])).get(host_id, (None, None)) \ if c is not None else (None, None) + # Its CPU and memory against their normal, and its anomalies (#5742). + norms = (await docker_svc.container_norms(db, {(host_id, name)})).get((host_id, name)) + resource = docker_svc.container_resource(host.name, name) if host is not None else None + anomalies = await anom.views(db, await anom.list_anomalies( + db, source="docker", resource=resource, + since=datetime.now(timezone.utc) - timedelta(days=30), limit=20), + links=False) if resource else [] if c is None: return await render_template( @@ -460,7 +478,10 @@ async def container_detail(host_id: str, name: str): name=name, label=docker_svc.task_label(c, {c.node_id: node_name} if node_name else None), node_name=node_name, - usage=docker_svc.usage(c, cpus, mem_total), + usage=docker_svc.usage(c, cpus, mem_total, thresholds=current_app.config.get("THRESHOLDS", {}), + norms=norms), + anomalies=anomalies, anomaly_back=f"/plugins/docker/container/{host_id}/{name}", + anomaly_resource=resource, exit_failed=docker_svc.exit_is_failure(c.exit_code, c.oom_killed), ports=json.loads(c.ports_json) if c.ports_json else [], uptime=_uptime(c.started_at) if c.status == "running" else None, @@ -888,6 +909,8 @@ async def service_detail(name: str): s = next((s for s in ctx["swarm_view"]["services"] if s["name"] == name), None) capacity = await docker_svc.host_capacity( db, {r["host_id"] for r in (s["replicas"] if s else []) if not r["ghost"]}) + norms = await docker_svc.container_norms( + db, {(r["host_id"], r["name"]) for r in (s["replicas"] if s else []) if not r["ghost"]}, now) if s is None and not task_containers: return await render_template("docker/service.html", name=name, service=None), 404 @@ -899,7 +922,9 @@ async def service_detail(name: str): for r in (s["replicas"] if s else []): c = None if r["ghost"] else by_key.get((r["host_id"], r["name"])) replicas.append(dict(r, task_id=getattr(c, "task_id", None) if c is not None else None, - usage=docker_svc.usage(c, *capacity.get(r["host_id"], (None, None))) + usage=docker_svc.usage(c, *capacity.get(r["host_id"], (None, None)), + thresholds=current_app.config.get("THRESHOLDS", {}), + norms=norms.get((c.host_id, c.name))) if c is not None else None)) node_host = {c.node_id: c.host_id for c in ctx["containers"] if c.node_id} table = _service_tasks(name, tasks, replicas, node_host, ctx["host_name"]) diff --git a/plugins/docker/service.py b/plugins/docker/service.py index 108e88d..050f5f5 100644 --- a/plugins/docker/service.py +++ b/plugins/docker/service.py @@ -155,7 +155,8 @@ def display_name(c) -> str: return c.name -def usage(c, cpus: int | None, mem_total: int | None, cpu_pct: float | None = None) -> dict: +def usage(c, cpus: int | None, mem_total: int | None, cpu_pct: float | None = None, *, + thresholds: dict | None = None, norms: dict | None = None) -> dict: """A container's CPU and memory in terms a reader can weigh. Docker's CPU% is per core (100% = one core busy), so it is shown as cores @@ -164,11 +165,16 @@ def usage(c, cpus: int | None, mem_total: int | None, cpu_pct: float | None = No is shown in bytes, against a limit only when one is actually set; mem_pct is then a share of that limit, otherwise of the host's memory. `cpu_pct` replaces the container's last sample (an average, say) when given. + + With `thresholds`, cpu_grade and mem_grade are the grades to show: past + the cutoff but within the container's own normal for the hour (`norms`, + {metric: Norm}, container_norms) is "ok", and cpu_usual / mem_usual say + what is usual (#5742). Without, they are None and the page grades. """ cpu = getattr(c, "cpu_pct", None) if cpu_pct is None else cpu_pct limit = getattr(c, "mem_limit_bytes", None) limited = bool(limit and mem_total and limit < mem_total * 0.95) - return { + out = { "cores": None if cpu is None else cpu / 100.0, "cpu_share": None if cpu is None or not cpus else cpu / cpus, "cpus": cpus, @@ -176,7 +182,51 @@ def usage(c, cpus: int | None, mem_total: int | None, cpu_pct: float | None = No "mem_limit": limit if limited else None, "mem_pct": getattr(c, "mem_pct", None), "mem_of": "its limit" if limited else ("the host's memory" if mem_total else None), + "cpu_grade": None, "cpu_usual": None, "mem_grade": None, "mem_usual": None, } + if thresholds is not None: + _grade(out, cpu, thresholds, norms or {}) + return out + + +# Container figures graded against their own normal (#5742): the metric that +# learns it and its floor's setting (norms.floor.*). +NORMED = {"cpu": ("cpu_pct", "container_cpu_pct"), "mem": ("mem_pct", "container_mem_pct")} + + +def _grade(u: dict, cpu: float | None, thresholds: dict, norms: dict) -> None: + """Fill usage()'s grades. CPU is graded as a share of the host but learned + per core (Docker's own figure), so its normal is read in cores.""" + from steward.core.anomalies import departs, graded + from steward.core.formatting import status_class + from steward.core.settings import floor_of + grade = status_class(u["cpu_share"], "cpu", thresholds) + norm = norms.get("cpu_pct") + if grade in ("warn", "crit") and norm is not None and cpu is not None \ + and not departs(cpu, norm, floor_of(thresholds, "container_cpu_pct")): + grade = "ok" + u["cpu_usual"] = f"usual {norm.usual_hold / 100:.2f} cores at this hour" + u["cpu_grade"] = grade + if u["mem_of"]: + u["mem_grade"], u["mem_usual"] = graded( + u["mem_pct"], "mem", thresholds, norms.get("mem_pct"), + floor_of(thresholds, "container_mem_pct"), "%") + else: + u["mem_grade"] = "dim" + + +async def container_norms(db, keys, now: datetime | None = None) -> dict: + """{(host_id, container_name): {metric: Norm}}: each container's CPU and + memory normals for this hour. One without enough history is absent.""" + from steward.core.norms import norms_at + resources = await container_resources(db, keys) + got = await norms_at(db, [("docker", r, metric) for r in resources + for metric, _f in NORMED.values()], + now or datetime.now(timezone.utc)) + out: dict = {} + for (_s, resource, metric), norm in got.items(): + out.setdefault(resources[resource], {})[metric] = norm + return out async def host_capacity(db, host_ids) -> dict[str, tuple[int | None, int | None]]: diff --git a/plugins/docker/templates/docker/_usage.html b/plugins/docker/templates/docker/_usage.html index b270c62..8d0a62a 100644 --- a/plugins/docker/templates/docker/_usage.html +++ b/plugins/docker/templates/docker/_usage.html @@ -3,7 +3,9 @@ Docker counts CPU per core (100% = one core busy), so CPU reads as cores, toned by its share of the host's CPUs when the host's core count is known. Memory reads in bytes, against a limit only when one is actually set; - mem_pct is then a share of that limit, otherwise of the host's memory. -#} + mem_pct is then a share of that limit, otherwise of the host's memory. + Each is toned by usage()'s grade when it has one: within the container's + own normal for the hour reads healthy, and says what is usual (#5742). -#} {% macro cores(n) -%} {{ "%.2f"|format(n) if n < 1 else "%.1f"|format(n) }} {{ "core" if n == 1 else "cores" }} @@ -12,15 +14,15 @@ {% macro cpu(u) -%} {%- if u.cores is none -%}— {%- else -%} -{{ cores(u.cores) }} +{{ cores(u.cores) }} {%- endif -%} {%- endmacro %} {% macro mem(u) -%} {%- if u.mem_bytes is none -%}— {%- else -%} -{{ u.mem_bytes|bytes }}{% if u.mem_limit %} / {{ u.mem_limit|bytes }}{% endif %} +{{ u.mem_bytes|bytes }}{% if u.mem_limit %} / {{ u.mem_limit|bytes }}{% endif %} {%- endif -%} {%- endmacro %} diff --git a/plugins/docker/templates/docker/container_detail.html b/plugins/docker/templates/docker/container_detail.html index 44e9f35..1cd08f7 100644 --- a/plugins/docker/templates/docker/container_detail.html +++ b/plugins/docker/templates/docker/container_detail.html @@ -1,7 +1,8 @@ {# docker/container_detail.html — one container (#5310), shaped like a service's page: a header that says whether it is well and what it is; why it stopped, only while it has; what it moved since it started; its CPU and - memory over time (the one home for what it uses now); its lifecycle. + memory over time (the one home for what it uses now), with its metric + anomalies above when it has any; its lifecycle. A swarm task is titled by its service and slot, the raw name under it. #} {% extends "base.html" %} {% from "_macros.html" import crumbs, status_badge %} @@ -63,6 +64,9 @@ {% endif %} +{# ── Its CPU or memory out of its normal for the hour, and past anomalies (#5742) #} +{% with back=anomaly_back, base_resource=anomaly_resource %}{% include "_anomaly_card.html" %}{% endwith %} + {# ── What it is set up with, and what it moved since it started ─────────── #}
diff --git a/plugins/docker/templates/docker/widget_resources.html b/plugins/docker/templates/docker/widget_resources.html index 970f92e..a3b11bf 100644 --- a/plugins/docker/templates/docker/widget_resources.html +++ b/plugins/docker/templates/docker/widget_resources.html @@ -33,13 +33,13 @@ {%- if session.user_id %}{{ r.label }}{% else %}{{ r.label }}{% endif -%} {{ r.host_name }} - + {{- bars(r.cpu_bars, (cores(r.cpu_bars.peak) ~ " cores") if r.cpu_bars.peak is not none else "", cores(r.cpu_bars.roof) ~ " cores") -}} {% if u.cores is none %}—{% else %}{{ cores(u.cores) }} cores{% endif %} - + {{- bars(r.mem_bars, r.mem_bars.peak|bytes if r.mem_bars.peak is not none else "", r.mem_bars.roof|bytes) -}} {% if u.mem_bytes is none %}—{% else %}{{ u.mem_bytes|bytes }}{% endif %} diff --git a/plugins/host_agent/__init__.py b/plugins/host_agent/__init__.py index b579491..2af95c3 100644 --- a/plugins/host_agent/__init__.py +++ b/plugins/host_agent/__init__.py @@ -11,6 +11,14 @@ _app: "Quart | None" = None def setup(app: "Quart") -> None: global _app _app = app + # A host's CPU, memory and load per core learn their normal (#5742), each + # with its floor from Settings → Thresholds. Parts don't: a host's CPU has + # a normal, each of its cores doesn't. Disks stay on their fixed cutoffs: + # a nearly full disk is a failure whatever its history (System 182). + from steward.core.norms import floor_setting, learn_normal + learn_normal("host_agent", "cpu_pct", floor_setting(app, "host_cpu_pct"), parts=False) + learn_normal("host_agent", "mem_used_pct", floor_setting(app, "host_mem_pct"), parts=False) + learn_normal("host_agent", "load_pct", floor_setting(app, "host_load_pct"), parts=False) def get_scheduled_tasks() -> list: @@ -38,9 +46,10 @@ def get_alert_metrics() -> dict[str, list[str]]: disk_used_pct per mount, resource ":"). seconds_since_seen comes from the scheduler, not the agent: "agent offline" is > 180. clock_off_secs is how far the host's clock is from Steward's, either way - (agent ≥ 1.12.0): "clock drifting" is > 30.""" + (agent ≥ 1.12.0): "clock drifting" is > 30. load_pct is load_1m as a % + of the host's cores.""" return {"host_agent": [ "cpu_pct", "mem_used_pct", "mem_available_bytes", "swap_used_bytes", - "disk_used_pct_worst", "disk_used_pct", "load_1m", "load_5m", "load_15m", + "disk_used_pct_worst", "disk_used_pct", "load_1m", "load_5m", "load_15m", "load_pct", "uptime_secs", "seconds_since_seen", "clock_off_secs", ]} diff --git a/plugins/host_agent/routes.py b/plugins/host_agent/routes.py index 2134773..fa40caa 100644 --- a/plugins/host_agent/routes.py +++ b/plugins/host_agent/routes.py @@ -21,6 +21,7 @@ from steward.core.health import ingest as ingest_tally from steward.core.metric_series import points as metric_points from steward.core.settings import get_setting, public_base_url from steward.core.time_range import parse_range, RANGE_OPTIONS +from steward.hosts import service as host_svc from steward.models.hosts import Host from steward.models.metrics import PluginMetric from .models import HostAgentRegistration @@ -118,6 +119,10 @@ def _expand_sample_to_metrics( for key in ("1m", "5m", "15m"): if key in load: row(f"load_{key}", host_name, load[key]) + # Load per core as a %, the figure the pages grade, so it can learn a + # normal (#5742): a load of 6 is idle on 32 cores and busy on 4. + if "1m" in load and sample.get("cpu_cores"): + row("load_pct", host_name, 100.0 * load["1m"] / len(sample["cpu_cores"])) if sample.get("uptime_secs") is not None: row("uptime_secs", host_name, sample["uptime_secs"]) @@ -725,6 +730,7 @@ async def host_vitals(host_id: str): reg = (await db.execute(select(HostAgentRegistration).where( HostAgentRegistration.host_id == host_id))).scalar_one_or_none() latest = await _latest_metrics_for_host(db, host.name) if reg else {} + norms = await host_svc.host_norms(db, host.name) if reg else {} # The last 6 h for the at-a-glance sparklines (cpu/mem/load host-level, # disk from the root mount), averaged into ~120 points in SQL, so the @@ -759,13 +765,23 @@ async def host_vitals(host_id: str): reporting = bool(reg) and ls is not None stale = reporting and (datetime.now(timezone.utc) - ls).total_seconds() > _stale_after_seconds() cpu, mem = hostlvl.get("cpu_pct"), hostlvl.get("mem_used_pct") + thresholds = current_app.config.get("THRESHOLDS", {}) worst = worst_reading((("CPU", cpu, "cpu"), ("Memory", mem, "mem"), ("Disk /", disk_root, "disk"), ("Load per core", load_per_core, "load")), - current_app.config.get("THRESHOLDS", {})) + thresholds, host_svc.host_normals(norms, thresholds)) + # Each tile's grade; CPU, memory and load within the host's own normal + # for the hour read healthy, and say what is usual (#5742). + grades = host_svc.host_grades({"cpu": cpu, "mem": mem, "disk": disk_root, "load": load_per_core}, + norms, thresholds) + usual = [f"{label} {value:.0f}% is {grades[kind][1]}" + for label, value, kind in (("CPU", cpu, "cpu"), ("Memory", mem, "mem"), + ("load per core", load_per_core, "load")) + if grades[kind][1]] return await render_template( "_host_vitals.html", reg=reg, cpu=cpu, mem=mem, disk_root=disk_root, load_per_core=load_per_core, sparks=sparks, psi=psi, reporting=reporting, stale=stale, worst=worst, + grades=grades, usual=usual, clock=clock_words(reg.clock_offset_s if reg else None, current_app.config.get("THRESHOLDS", {})), ) diff --git a/plugins/host_agent/templates/_host_vitals.html b/plugins/host_agent/templates/_host_vitals.html index 843cb09..f34b29f 100644 --- a/plugins/host_agent/templates/_host_vitals.html +++ b/plugins/host_agent/templates/_host_vitals.html @@ -1,6 +1,7 @@ {# Host health header — live polled into the host hub: live or stale, the worst reading against Settings → Thresholds, last seen, then CPU / memory / disk / - load tiles graded by the same cutoffs. Renders nothing until the agent + load tiles graded by the same cutoffs (`grades`; CPU, memory and load within + the host's own normal for the hour read healthy, and `usual` says so). Renders nothing until the agent reports (the Agent panel below then shows provisioning). #} {% from "_macros.html" import status_badge, stat_tile %} {% macro pct(v) %}{{ '%.0f%%'|format(v) if v is not none else '—' }}{% endmacro %} @@ -11,6 +12,7 @@ {%- if stale %}No report since {{ reg.last_seen_at|ago }}; the readings below are from then. {%- elif worst %}{{ worst[0] }} {{ pct(worst[1]) }} is past its {{ "critical" if worst[2] == "crit" else "warning" }} threshold. + {%- elif usual %}Every reading is within its thresholds or its usual level for this hour. {%- else %}Every reading is within its thresholds.{% endif -%} @@ -20,11 +22,14 @@
- {{ stat_tile("CPU", pct(cpu), state=status_class(cpu, 'cpu'), hint=sparks.cpu|safe) }} - {{ stat_tile("Memory", pct(mem), state=status_class(mem, 'mem'), hint=sparks.mem|safe) }} - {{ stat_tile("Disk /", pct(disk_root), state=status_class(disk_root, 'disk'), hint=sparks.disk|safe) }} - {{ stat_tile("Load per core", pct(load_per_core), state=status_class(load_per_core, 'load'), hint=sparks.load|safe) }} + {{ stat_tile("CPU", pct(cpu), state=grades.cpu[0], hint=sparks.cpu|safe) }} + {{ stat_tile("Memory", pct(mem), state=grades.mem[0], hint=sparks.mem|safe) }} + {{ stat_tile("Disk /", pct(disk_root), state=grades.disk[0], hint=sparks.disk|safe) }} + {{ stat_tile("Load per core", pct(load_per_core), state=grades.load[0], hint=sparks.load|safe) }}
+ {% if usual and not stale %} +

{{ usual | join("; ") | capitalize }}.

+ {% endif %} {% if psi.cpu is not none or psi.mem is not none or psi.io is not none %}

Pressure (10 s): cpu {{ pct(psi.cpu) }} · mem {{ pct(psi.mem) }} · io {{ pct(psi.io) }} diff --git a/plugins/traefik/__init__.py b/plugins/traefik/__init__.py index 6785af6..06b0623 100644 --- a/plugins/traefik/__init__.py +++ b/plugins/traefik/__init__.py @@ -26,12 +26,12 @@ def setup(app: "Quart") -> None: # learned from the scrape-by-scrape figures only: before #5755 it was a # percentile of every response since Traefik started. from datetime import datetime, timezone - from steward.core.norms import learn_normal - from .routes import FLOOR_4XX, FLOOR_P95, STATE_WINDOW, min_requests + from steward.core.norms import floor_setting, learn_normal + from .routes import STATE_WINDOW, min_requests gate = ("request_rate", lambda: min_requests(app.config.get("THRESHOLDS", {})) / STATE_WINDOW.total_seconds()) - learn_normal("traefik", "error_rate_4xx_pct", FLOOR_4XX, gate=gate) - learn_normal("traefik", "latency_p95_ms", FLOOR_P95, gate=gate, + learn_normal("traefik", "error_rate_4xx_pct", floor_setting(app, "router_4xx_pct"), gate=gate) + learn_normal("traefik", "latency_p95_ms", floor_setting(app, "router_p95_ms"), gate=gate, since=datetime(2026, 10, 14, 18, 47, tzinfo=timezone.utc)) diff --git a/plugins/traefik/routes.py b/plugins/traefik/routes.py index 54a8eda..32645c0 100644 --- a/plugins/traefik/routes.py +++ b/plugins/traefik/routes.py @@ -14,6 +14,7 @@ from steward.core.time_range import parse_range, DEFAULT_RANGE, bucket_seconds, from steward.core import anomalies as anom from steward.core.formatting import cert_left, status_class from steward.core.norms import norms_at +from steward.core.settings import floor_of from steward.models.metrics import PluginMetricLatest from .models import TraefikAccessSummary, TraefikCert, TraefikGlobalStat, TraefikMetric, TraefikStatusCode from .scheduler import router_domains @@ -56,12 +57,11 @@ def error_cutoffs(thresholds: dict) -> tuple[float, float]: # Each router's 4xx share and p95 are judged against its own normal once it # has one (#5741, System 182): an error-pages service answers 4xx all day and -# that is its normal, not a problem. Departing by less than the floor is never -# news. A share over fewer than min_requests() in the window is not graded: one -# 404 of three requests is 33%. The 5xx share is a failure and keeps its -# fixed cutoff whatever the history or traffic (the operator's ruling). -FLOOR_4XX = 2.0 # percentage points -FLOOR_P95 = 50.0 # ms +# that is its normal, not a problem. Departing by less than the floor +# (Settings → Thresholds, norms.floor.router_*) is never news. A share over +# fewer than min_requests() in the window is not graded: one 404 of three +# requests is 33%. The 5xx share is a failure and keeps its fixed cutoff +# whatever the history or traffic (the operator's ruling). NORMED = ("error_rate_4xx_pct", "latency_p95_ms") @@ -71,14 +71,6 @@ def min_requests(thresholds: dict) -> float: return float(thresholds.get("router_requests", {}).get("warn", 30)) -def _against(value: float, norm, floor: float, unit: str) -> tuple[bool, str]: - """(departs, what is usual) for a reading over its fixed cutoff.""" - if norm is None: - return True, "" - usual = f"usual {norm.usual_hold:.1f}{unit}" if unit == "%" else f"usual {norm.usual_hold:.0f}{unit}" - return anom.departs(value, norm, floor), f"{usual} at this hour" - - def router_state(reading, thresholds: dict, over: str = "", norms: dict | None = None) -> tuple[str, str | None]: """(grade, why) for a router's reading (recent_window's, or one scrape's): crit while it serves 5xx past its cutoff, warn on a p95 past the latency @@ -93,14 +85,16 @@ def router_state(reading, thresholds: dict, over: str = "", norms: dict | None = usual = [] p95 = reading.latency_p95_ms if p95 is not None and status_class(p95, "latency", thresholds) == "crit": - out, said = _against(p95, norms.get("latency_p95_ms"), FLOOR_P95, " ms") + out, said = anom.against(p95, norms.get("latency_p95_ms"), + floor_of(thresholds, "router_p95_ms"), " ms") if out: return "warn", f"p95 latency is {p95:.0f} ms{over}" + (f", {said}" if said else "") usual.append(f"p95 {p95:.0f} ms is {said}") requests = getattr(reading, "requests", None) e4 = reading.error_rate_4xx_pct or 0 if e4 > error_4xx and (requests is None or requests >= min_requests(thresholds)): - out, said = _against(e4, norms.get("error_rate_4xx_pct"), FLOOR_4XX, "%") + out, said = anom.against(e4, norms.get("error_rate_4xx_pct"), + floor_of(thresholds, "router_4xx_pct"), "%") if out: return "warn", f"{e4:.1f}% of responses are 4xx{over}" + (f", {said}" if said else "") usual.append(f"4xx {e4:.1f}% is {said}") diff --git a/steward/app.py b/steward/app.py index 335a941..ed897b2 100644 --- a/steward/app.py +++ b/steward/app.py @@ -129,6 +129,10 @@ def create_app( app.jinja_env.globals["threshold_style"] = ( lambda value, kind: _ts_for(value, kind, app.config.get("THRESHOLDS", {})) ) + # `grade_style(grade)` — the same styling for a grade worked out already + # (one graded against its own normal, #5742). + from .core.settings import grade_style as _grade_style + app.jinja_env.globals["grade_style"] = _grade_style # One way to write a size, a rate, a span and a moment, and to grade a # reading ('ok' / 'warn' / 'crit' / 'unknown') against the same cutoffs. @@ -445,6 +449,11 @@ def _register_core_tasks(app: Quart) -> None: from .backup.job import tick await tick(app) + # Monitor response times learn their normal (#5742); plugins register + # their own metrics from setup(). + from .monitors.service import learn_response_times + learn_response_times(app) + async def run_metric_norms(): from .core.norms import run_norms await run_norms(app) diff --git a/steward/core/anomalies.py b/steward/core/anomalies.py index 213096c..c034d31 100644 --- a/steward/core/anomalies.py +++ b/steward/core/anomalies.py @@ -51,6 +51,33 @@ def departs(level: float, norm: Norm, floor: float) -> bool: return level > norm.usual_hold * DEPART and level - norm.usual_hold >= floor +def against(value: float, norm: Norm | None, floor: float, unit: str) -> tuple[bool, str]: + """(departs, what is usual) for a page's reading past its fixed cutoff + (#5741, #5742). Without a normal it departs, and the cutoff stands; within + one it is usual for that resource, and the page says so. `unit` is "%" + (one decimal) or " ms" (whole).""" + if norm is None: + return True, "" + usual = f"{norm.usual_hold:.1f}{unit}" if unit == "%" else f"{norm.usual_hold:.0f}{unit}" + return departs(value, norm, floor), f"usual {usual} at this hour" + + +def graded(value: float | None, kind: str, thresholds: dict, norm: Norm | None, + floor: float, unit: str) -> tuple[str, str | None]: + """(status class, why) for a behaviour reading on a page (#5742): graded + against Settings → Thresholds, except that one past its cutoff but within + its own normal for the hour is "ok", and `why` says what is usual. Only + for behaviour (CPU, memory, load, latency); a failure keeps its cutoff + whatever its history (System 182).""" + from steward.core.formatting import status_class + cls = status_class(value, kind, thresholds) + if cls in ("warn", "crit") and norm is not None: + out, said = against(value, norm, floor, unit) + if not out: + return "ok", said + return cls, None + + # Each learned key reporting in the window, with the window's average. Keys # come from plugin_metric_latest (one row each), and each average is a range # read of the series index, so the pass never scans the raw table. @@ -64,18 +91,22 @@ CROSS JOIN LATERAL ( AND p.recorded_at <= :now ) v WHERE l.source_module || '/' || l.metric_name = ANY(:keys) + AND (strpos(l.resource_name, ':') = 0 OR NOT l.source_module || '/' || l.metric_name = ANY(:whole)) AND l.recorded_at >= :since AND v.level IS NOT NULL -""").bindparams(bindparam("keys", type_=ARRAY(String))) +""").bindparams(bindparam("keys", type_=ARRAY(String)), bindparam("whole", type_=ARRAY(String))) async def levels_now(session, now: datetime) -> dict[tuple[str, str, str], float]: """{(source, resource, metric): its average over the last WINDOW}, for - every learned metric and every metric one is gated on.""" + every learned metric and every metric one is gated on. A metric learned + without its parts reads its whole resources only.""" keys = sorted({f"{s}/{m}" for s, m in LEARNED} | {f"{k.source_module}/{k.gate[0]}" for k in LEARNED.values() if k.gate}) if not keys: return {} - rows = await session.execute(_LEVELS, {"keys": keys, "since": now - WINDOW, "now": now}) + whole = sorted(f"{k.source_module}/{k.metric_name}" for k in LEARNED.values() if not k.parts) + rows = await session.execute(_LEVELS, {"keys": keys, "whole": whole, + "since": now - WINDOW, "now": now}) return {(s, r, m): float(level) for s, r, m, level in rows} @@ -98,7 +129,7 @@ async def detect(session, now: datetime | None = None) -> dict: for key, level in levels.items(): norm = norms.get(key) learned = LEARNED.get((key[0], key[2])) - out = norm is not None and learned is not None and departs(level, norm, learned.floor) + out = norm is not None and learned is not None and departs(level, norm, learned.least_departure) current = open_now.pop(key, None) if current is not None: span, anomaly = current diff --git a/steward/core/formatting.py b/steward/core/formatting.py index d6d32b8..1101555 100644 --- a/steward/core/formatting.py +++ b/steward/core/formatting.py @@ -233,16 +233,23 @@ def required(value: Any, what: str) -> Any: _SEVERITY = {"crit": 2, "warn": 1} -def worst_reading(readings, thresholds: dict): +def worst_reading(readings, thresholds: dict, normals: dict | None = None): """The reading furthest past its cutoff, as (label, value, grade), or None when every reading is within its cutoffs or missing. `readings` is (label, value, kind) triples, graded like status_class; the first of equally bad readings wins, so list them in the order a reader - should hear about them.""" + should hear about them. `normals` ({kind: (Norm, floor)}) are the + resource's own normals for this hour: a reading past its cutoff but + within its normal is not past it (#5742).""" + from steward.core.anomalies import graded + normals = normals or {} worst, rank = None, 0 for label, value, kind in readings: - grade = status_class(value, kind, thresholds) + if kind in normals: + grade = graded(value, kind, thresholds, *normals[kind], "%")[0] + else: + grade = status_class(value, kind, thresholds) if _SEVERITY.get(grade, 0) > rank: worst, rank = (label, value, grade), _SEVERITY[grade] return worst @@ -327,6 +334,7 @@ METRICS: dict[str, tuple[str, str]] = { "load_1m": ("Load (1 min)", "runnable processes, averaged over a minute"), "load_5m": ("Load (5 min)", "runnable processes, averaged over 5 minutes"), "load_15m": ("Load (15 min)", "runnable processes, averaged over 15 minutes"), + "load_pct": ("Load per core", "1-minute load as a % of the host's cores; 100% keeps every core busy"), "uptime_secs": ("Uptime", "time since the host booted"), "seconds_since_seen": ("Time since the agent reported", "over 3 minutes means the agent is offline"), "clock_off_secs": ("Clock off by", "how far the host's clock is from Steward's, either way"), diff --git a/steward/core/norms.py b/steward/core/norms.py index f6c4872..4a79928 100644 --- a/steward/core/norms.py +++ b/steward/core/norms.py @@ -35,7 +35,7 @@ from collections.abc import Callable from dataclasses import dataclass from datetime import datetime, timedelta, timezone -from sqlalchemy import ARRAY, DateTime, String, bindparam, delete, select, text, tuple_ +from sqlalchemy import ARRAY, Boolean, DateTime, String, bindparam, delete, select, text, tuple_ from steward.models.norms import MetricNorm @@ -51,31 +51,45 @@ USUAL = 0.9 # "usual" is this percentile across past days: one odd d class Learned: """A metric that learns a normal. `floor` is the smallest departure, in the metric's own unit, worth calling an anomaly (#5740), so that - 0.1% → 0.3% of a share or 2 ms → 4 ms of latency is never news. + 0.1% → 0.3% of a share or 2 ms → 4 ms of latency is never news. It may + be a function, read at each pass, for a floor that is a setting (#5742). `gate` is (metric, least): the resource is judged only while that metric of its own averages at least `least` over the window, as a router's error share means nothing over a handful of requests (#5741). `least` may be a function, read at each pass, for a minimum that is a setting. `since`: history before it is not learned from, for a metric whose meaning - changed then.""" + changed then. `parts=False` learns only whole resources, not their parts + (":core3"): a host's CPU has a normal, each of its cores doesn't.""" source_module: str metric_name: str - floor: float + floor: float | Callable[[], float] gate: tuple[str, float | Callable[[], float]] | None = None since: datetime | None = None + parts: bool = True + + @property + def least_departure(self) -> float: + return float(self.floor() if callable(self.floor) else self.floor) LEARNED: dict[tuple[str, str], Learned] = {} -def learn_normal(source_module: str, metric_name: str, floor: float, *, +def learn_normal(source_module: str, metric_name: str, floor: float | Callable[[], float], *, gate: tuple[str, float | Callable[[], float]] | None = None, - since: datetime | None = None) -> None: + since: datetime | None = None, parts: bool = True) -> None: """Have Steward learn this metric's normal for every resource reporting it. A plugin calls this from setup(), as it calls prune_by_age.""" - LEARNED[(source_module, metric_name)] = Learned(source_module, metric_name, float(floor), - gate, since) + LEARNED[(source_module, metric_name)] = Learned( + source_module, metric_name, floor if callable(floor) else float(floor), gate, since, parts) + + +def floor_setting(app, name: str) -> Callable[[], float]: + """A floor read from Settings → Thresholds (norms.floor.) at each + pass, so a saved change applies at once.""" + from steward.core.settings import floor_of + return lambda: floor_of(app.config.get("THRESHOLDS", {}), name) @dataclass(frozen=True) @@ -103,20 +117,23 @@ def slot_of(at: datetime, kind: str) -> int: # own slot and its neighbours'. _REBUILD = text(""" WITH learned AS ( - SELECT key, since FROM unnest(CAST(:keys AS text[]), CAST(:starts AS timestamptz[])) AS k(key, since) + SELECT key, since, parts + FROM unnest(CAST(:keys AS text[]), CAST(:starts AS timestamptz[]), CAST(:parts AS boolean[])) + AS k(key, since, parts) ), all_hours AS ( SELECT h.source_module AS s, h.resource_name AS r, h.metric_name AS m, h.bucket AS b, COALESCE(h.value_p95, h.value_avg) AS hold, h.value_max AS peak FROM plugin_metrics_hourly h JOIN learned k ON k.key = h.source_module || '/' || h.metric_name - WHERE h.bucket >= k.since + WHERE h.bucket >= k.since AND (k.parts OR strpos(h.resource_name, ':') = 0) UNION ALL SELECT p.source_module, p.resource_name, p.metric_name, date_trunc('hour', p.recorded_at), percentile_cont(0.95) WITHIN GROUP (ORDER BY p.value), max(p.value) FROM plugin_metrics p JOIN learned k ON k.key = p.source_module || '/' || p.metric_name WHERE p.recorded_at >= k.since AND p.recorded_at < :this_hour + AND (k.parts OR strpos(p.resource_name, ':') = 0) GROUP BY 1, 2, 3, 4 ), hours AS ( @@ -157,7 +174,8 @@ SELECT s, r, m, (slot + n.off + slots) % slots, kind, FROM slotted CROSS JOIN generate_series(-CAST(:neighbours AS int), CAST(:neighbours AS int)) AS n(off) GROUP BY s, r, m, (slot + n.off + slots) % slots, kind, days """).bindparams(bindparam("keys", type_=ARRAY(String)), - bindparam("starts", type_=ARRAY(DateTime(timezone=True)))) + bindparam("starts", type_=ARRAY(DateTime(timezone=True))), + bindparam("parts", type_=ARRAY(Boolean))) async def rebuild_norms(session, now: datetime | None = None) -> int: @@ -172,6 +190,7 @@ async def rebuild_norms(session, now: datetime | None = None) -> int: "keys": [f"{s}/{m}" for s, m in LEARNED], # Each key learns from WEEK_WINDOW back, or from its own `since`. "starts": [max(week_since, k.since) if k.since else week_since for k in LEARNED.values()], + "parts": [k.parts for k in LEARNED.values()], "day_since": now - DAY_WINDOW, "this_hour": now.replace(minute=0, second=0, microsecond=0), "week_ready": now - timedelta(days=WEEK_MIN_DAYS), diff --git a/steward/core/settings.py b/steward/core/settings.py index 51a5d8e..b010ee4 100644 --- a/steward/core/settings.py +++ b/steward/core/settings.py @@ -100,6 +100,19 @@ DEFAULTS: dict[str, Any] = { # A router's 4xx share over fewer requests than this in 15 minutes isn't # graded, on the page or for anomalies: one 404 of three is 33% (#5741). "traefik.threshold.min_requests": 30, + # The smallest departure from a metric's normal worth calling an anomaly + # (#5742), on top of being a quarter over its usual hold: a share's floor + # is in percentage points, a response time's in ms. 5 points is the + # baseline; bursty figures (CPU, load) get 10 so an ordinary build or + # backup minute isn't news. + "norms.floor.host_cpu_pct": 10, + "norms.floor.host_mem_pct": 5, + "norms.floor.host_load_pct": 10, + "norms.floor.container_cpu_pct": 10, + "norms.floor.container_mem_pct": 5, + "norms.floor.monitor_ms": 100, + "norms.floor.router_4xx_pct": 2, + "norms.floor.router_p95_ms": 50, # Degraded/critical cutoffs for metric coloring (warn=amber, crit=red). # cpu/mem/disk/load are percentages (load is load-per-core %); temp in °C. # uptime is "higher is better" so its warn/crit are floors. Ping latency @@ -540,9 +553,23 @@ def to_thresholds_cfg(settings: dict[str, Any]) -> dict: # Fewer requests than this in 15 minutes and a router's 4xx share isn't graded. "router_requests": {"warn": g("traefik.threshold.min_requests", 30), "crit": g("traefik.threshold.min_requests", 30), "dir": "low"}, + # Not a cutoff: each normal's floor (norms.floor.*), read by floor_of(). + "floors": {name: float(g(f"norms.floor.{name}", DEFAULTS[f"norms.floor.{name}"])) + for name in NORM_FLOORS}, } +NORM_FLOORS = ("host_cpu_pct", "host_mem_pct", "host_load_pct", "container_cpu_pct", + "container_mem_pct", "monitor_ms", "router_4xx_pct", "router_p95_ms") + + +def floor_of(thresholds: dict, name: str) -> float: + """A normal's floor (one of NORM_FLOORS) from to_thresholds_cfg's output: + the smallest departure from it worth calling an anomaly (#5742).""" + got = (thresholds.get("floors") or {}).get(name) + return float(got if got is not None else DEFAULTS[f"norms.floor.{name}"]) + + _STATUS_STYLE = {"warn": "color:var(--yellow);font-weight:500;", "crit": "color:var(--red);font-weight:500;"} @@ -557,6 +584,12 @@ def threshold_style_for(value: Any, kind: str, thresholds: dict) -> str: return _STATUS_STYLE.get(status_class(value, kind, thresholds), "") +def grade_style(grade: str | None) -> str: + """threshold_style_for's style for a grade already worked out ("warn" / + "crit"); empty for anything else.""" + return _STATUS_STYLE.get(grade or "", "") + + def to_plugins_cfg(settings: dict[str, Any]) -> dict: """Assemble {plugin_name: {...config}} from all plugin.* keys.""" result = {} diff --git a/steward/hosts/service.py b/steward/hosts/service.py index 53c2bbf..b5a4864 100644 --- a/steward/hosts/service.py +++ b/steward/hosts/service.py @@ -9,6 +9,8 @@ from datetime import datetime, timedelta, timezone from sqlalchemy import and_, case, func, select +from steward.core.norms import norms_at +from steward.core.settings import floor_of from steward.core.status import _heartbeat_title, _last_n_results, _uptime_by_monitor from steward.models.hosts import Host from steward.models.monitors import Monitor, MonitorResult @@ -249,8 +251,9 @@ async def agent_overview_by_host( """Per-host agent glance (cpu/mem/disk/load + freshness) from plugin_metric_latest. {host_name: {cpu_pct, mem_used_pct, disk_used_pct_worst, disk_root, load_1m, - cores, load_per_core, last_at, fresh, clock_off_secs}} for each host with any - agent samples. Cores are counted from the per-core CPU sub-resources + cores, load_per_core, last_at, fresh, clock_off_secs, norms}} for each host + with any agent samples. `norms` is {kind: Norm}, the host's CPU, memory and + load normals for this hour (host_normals grades by them). Cores are counted from the per-core CPU sub-resources (":core"), as on the host page; load_per_core is load_1m as a % of them, the figure Settings → Thresholds grades "load" by. @@ -294,10 +297,50 @@ async def agent_overview_by_host( d["cores"] = cores.get(name) load = d.get("load_1m") d["load_per_core"] = round(load / d["cores"] * 100) if (load is not None and d["cores"]) else None + d["norms"] = {} out[name] = d + learned = await norms_at(db, [("host_agent", h, metric) for h in out + for metric, _f in HOST_NORMED.values()], now) + for (_s, host, metric), norm in learned.items(): + out[host]["norms"][_KIND_OF[metric]] = norm return out +# The host figures graded against their own normal (#5742): the threshold +# kind, the metric that learns it and its floor's setting (norms.floor.*). +# Disk keeps its cutoff: a nearly full disk is a failure (System 182). +HOST_NORMED = {"cpu": ("cpu_pct", "host_cpu_pct"), "mem": ("mem_used_pct", "host_mem_pct"), + "load": ("load_pct", "host_load_pct")} +_KIND_OF = {metric: kind for kind, (metric, _f) in HOST_NORMED.items()} + + +def host_normals(norms: dict, thresholds: dict) -> dict: + """{kind: (Norm, floor)} for worst_reading and anomalies.graded, from an + agent glance's `norms`.""" + return {kind: (norm, floor_of(thresholds, HOST_NORMED[kind][1])) + for kind, norm in (norms or {}).items()} + + +async def host_norms(db, host_name: str, now: datetime | None = None) -> dict: + """{kind: Norm}: one host's CPU, memory and load normals for this hour.""" + learned = await norms_at(db, [("host_agent", host_name, metric) + for metric, _f in HOST_NORMED.values()], + now or datetime.now(timezone.utc)) + return {_KIND_OF[metric]: norm for (_s, _h, metric), norm in learned.items()} + + +def host_grades(values: dict, norms: dict, thresholds: dict) -> dict: + """{kind: (status class, why)} for a host's readings ({kind: value}): + against Settings → Thresholds, except CPU, memory and load within the + host's own normal, which are "ok" and say what is usual (#5742).""" + from steward.core.anomalies import graded + from steward.core.formatting import status_class + normals = host_normals(norms, thresholds) + return {kind: (graded(value, kind, thresholds, *normals[kind], "%") if kind in normals + else (status_class(value, kind, thresholds), None)) + for kind, value in values.items()} + + def clock_trouble(agent: dict[str, dict], limit: float) -> dict | None: """Reporting hosts whose clock is more than `limit` seconds from Steward's (Settings → Thresholds' clock drift warning, milestone 490), @@ -326,19 +369,24 @@ def host_glance(hosts, status: dict, agent: dict, thresholds: dict) -> list[dict Each is {host, state, worst, agent}. State is "down" when a linked monitor is down, "stale" when the agent has stopped reporting, else the grade of the reading furthest past its cutoff ("crit" / "warn": CPU, memory, disk / - or load per core, against Settings → Thresholds), else "up"; "unknown" for + or load per core, against Settings → Thresholds; CPU, memory and load + within the host's own normal are not past it), else "up"; "unknown" for a host with neither monitors nor agent. `worst` is (label, value, grade) - or None, from formatting.worst_reading.""" + or None, from formatting.worst_reading; `grades` is host_grades' {kind: + (grade, why)} for a fresh agent, else {}.""" from steward.core.formatting import worst_reading rows = [] for h in hosts: st, a = status.get(h.id), agent.get(h.name) - worst = None + worst, grades = None, {} if a and a.get("fresh"): + grades = host_grades({"cpu": a.get("cpu_pct"), "mem": a.get("mem_used_pct"), + "disk": a.get("disk_root"), "load": a.get("load_per_core")}, + a.get("norms"), thresholds) worst = worst_reading(( ("CPU", a.get("cpu_pct"), "cpu"), ("Memory", a.get("mem_used_pct"), "mem"), ("Disk /", a.get("disk_root"), "disk"), ("Load", a.get("load_per_core"), "load")), - thresholds) + thresholds, host_normals(a.get("norms"), thresholds)) if st and st["state"] == "down": state = "down" elif a and not a.get("fresh"): @@ -349,5 +397,5 @@ def host_glance(hosts, status: dict, agent: dict, thresholds: dict) -> list[dict state = "up" else: state = "unknown" - rows.append({"host": h, "state": state, "worst": worst, "agent": a or {}}) + rows.append({"host": h, "state": state, "worst": worst, "agent": a or {}, "grades": grades}) return sorted(rows, key=lambda r: (GLANCE_ORDER[r["state"]], r["host"].name.lower())) diff --git a/steward/mcp/tools/overview.py b/steward/mcp/tools/overview.py index fb4f62b..5ed4d7c 100644 --- a/steward/mcp/tools/overview.py +++ b/steward/mcp/tools/overview.py @@ -111,6 +111,9 @@ def agent_findings(hosts, agent: dict, thresholds: dict) -> list[dict]: "host agent has stopped reporting; its metrics are out of date", since=a.get("last_at"), ids=ids, next=f'get_host(host="{h.id}")')) continue # stale values would only add noise + # CPU and memory within the host's own normal for the hour aren't + # findings (#5742); the fullest disk keeps its cutoff. + normals = host_svc.host_normals(a.get("norms"), thresholds) for metric, kind, label in _AGENT_CHECKS: value = a.get(metric) th = thresholds.get(kind) or {} @@ -122,6 +125,8 @@ def agent_findings(hosts, agent: dict, thresholds: dict) -> list[dict]: sev = "warning" else: continue + if kind in normals and not anom.against(value, normals[kind][0], normals[kind][1], "%")[0]: + continue out.append(finding( sev, f"host_{kind}_high", h.name, f"{label} at {rnd(value, 1)}% (warn {th['warn']}%, crit {th.get('crit')}%)", diff --git a/steward/monitors/routes.py b/steward/monitors/routes.py index 2b083a5..94c54e3 100644 --- a/steward/monitors/routes.py +++ b/steward/monitors/routes.py @@ -6,12 +6,12 @@ Host (host_id) or watch a free-standing custom destination (host_id = NULL). """ from __future__ import annotations import uuid -from datetime import datetime, timezone +from datetime import datetime, timedelta, timezone from quart import Blueprint, current_app, render_template, request, redirect, url_for from steward.auth.middleware import require_role -from steward.core import renames +from steward.core import anomalies as anom, renames from steward.core.audit import audit_request from steward.models.users import UserRole from steward.models.monitors import Monitor, MonitorType, MONITOR_TYPE_VALUES @@ -125,6 +125,10 @@ async def detail(monitor_id: str): st = (await monitor_svc.monitor_states(db, [m]))[m.id] failures = await monitor_svc.recent_failures(db, m.id) history = await monitor_svc.response_history(db, m.id, datetime.now(timezone.utc)) + norm = (await monitor_svc.latency_norms(db, [m])).get(m.id) + anomalies = await anom.views(db, await anom.list_anomalies( + db, source=m.type, resource=m.name, + since=datetime.now(timezone.utc) - timedelta(days=30), limit=20), links=False) host = await db.get(Host, m.host_id) if m.host_id else None thresholds = current_app.config.get("THRESHOLDS", {}) latest = st["latest"] @@ -136,7 +140,8 @@ async def detail(monitor_id: str): tls_days = (latest.tls_expires_at - datetime.now(timezone.utc)).total_seconds() / 86400.0 return await render_template( "monitors/detail.html", monitor=m, host=host, latest=latest, failures=failures, - state=monitor_svc.row_state(m, latest, st["uptime"].get("24h"), thresholds), + state=monitor_svc.row_state(m, latest, st["uptime"].get("24h"), thresholds, norm), + latency=monitor_svc.latency_grade(latest, thresholds, norm), anomalies=anomalies, since=since, since_at_least=since_at_least, uptime=st["uptime"], tls_days=tls_days, heartbeat=[{"state": "up" if r.is_up else "down", "title": _heartbeat_title(m.type, r)} for r in st["recent"]], diff --git a/steward/monitors/service.py b/steward/monitors/service.py index ba91d3e..92b2dc9 100644 --- a/steward/monitors/service.py +++ b/steward/monitors/service.py @@ -13,7 +13,10 @@ from datetime import datetime, timedelta, timezone from sqlalchemy import func, select +from steward.core import anomalies as anom from steward.core.formatting import status_class +from steward.core.norms import Norm, floor_setting, learn_normal, norms_at +from steward.core.settings import floor_of from steward.core.status import _heartbeat_title, _last_n_results, _uptime_by_monitor from steward.models.monitors import MONITOR_TYPE_VALUES, Monitor, MonitorResult, MonitorType @@ -145,12 +148,41 @@ async def check_name_free(db, mtype: str, name: str, exclude_id: str | None = No STATE_ORDER = {"down": 0, "degraded": 1, "pending": 2, "up": 3, "paused": 4} -def row_state(monitor: Monitor, latest, uptime_pct, thresholds: dict) -> str: +def learn_response_times(app) -> None: + """Each monitor's response time learns its normal (#5742), by type (the + metric's source), with its floor from Settings → Thresholds. A monitor + down keeps its fixed meaning whatever its history (System 182).""" + for mtype in MONITOR_TYPE_VALUES: + learn_normal(mtype, "response_ms", floor_setting(app, "monitor_ms")) + + +async def latency_norms(db, monitors: list[Monitor], now: datetime | None = None) -> dict[str, Norm]: + """{monitor_id: its response time's normal for this hour}; a monitor with + too little history yet is absent, and graded on the fixed cutoffs.""" + got = await norms_at(db, [(m.type, m.name, "response_ms") for m in monitors], + now or datetime.now(timezone.utc)) + return {m.id: got[(m.type, m.name, "response_ms")] for m in monitors + if (m.type, m.name, "response_ms") in got} + + +def latency_grade(latest, thresholds: dict, norm: Norm | None = None) -> tuple[str, str | None]: + """(status class, why) for the latest response time: against the latency + cutoffs, except within its own normal for the hour, which is "ok" and says + what is usual.""" + if latest is None or latest.response_ms is None: + return "unknown", None + return anom.graded(latest.response_ms, "latency", thresholds, norm, + floor_of(thresholds, "monitor_ms"), " ms") + + +def row_state(monitor: Monitor, latest, uptime_pct, thresholds: dict, + norm: Norm | None = None) -> str: """paused, pending (never checked), down, degraded or up. Degraded is up but worth a look: uptime over the window at or below its warning cutoff, or the latest response time past its critical one - (Settings → Thresholds). A slow-but-tolerable reply is coloured, not sorted. + (Settings → Thresholds) and out of its normal (`norm`, #5742). A + slow-but-tolerable reply is coloured, not sorted. """ if not monitor.enabled: return "paused" @@ -159,7 +191,7 @@ def row_state(monitor: Monitor, latest, uptime_pct, thresholds: dict) -> str: if not latest.is_up: return "down" if (status_class(uptime_pct, "uptime", thresholds) in ("warn", "crit") - or status_class(latest.response_ms, "latency", thresholds) == "crit"): + or latency_grade(latest, thresholds, norm)[0] == "crit"): return "degraded" return "up" @@ -197,11 +229,13 @@ async def monitor_rows(db, thresholds: dict): uptime = await _uptime_by_monitor(db, ids) now = datetime.now(timezone.utc) + norms = await latency_norms(db, monitors, now) data = [] for m in monitors: rows = recent.get(m.id, []) latest = rows[-1] if rows else None windows = uptime.get(m.id, {}) + latency, usual = latency_grade(latest, thresholds, norms.get(m.id)) tls_days = None if latest and latest.tls_expires_at: tls_days = (latest.tls_expires_at - now).total_seconds() / 86400.0 @@ -209,7 +243,11 @@ async def monitor_rows(db, thresholds: dict): data.append({ "monitor": m, "latest": latest, - "state": row_state(m, latest, windows.get("24h"), thresholds), + "state": row_state(m, latest, windows.get("24h"), thresholds, norms.get(m.id)), + # The latest response time's grade, and what is usual when that + # quieted it (#5742). + "latency": latency, + "latency_usual": usual, "heartbeat": [{"state": "up" if r.is_up else "down", "title": _heartbeat_title(m.type, r)} for r in rows], "spark": sparkline_svg(times, width=56, height=16, stroke="currentColor") if len(times) > 1 else "", diff --git a/steward/settings/routes.py b/steward/settings/routes.py index 02ce91e..dcec521 100644 --- a/steward/settings/routes.py +++ b/steward/settings/routes.py @@ -15,7 +15,7 @@ from steward.core.settings import ( get_stored_plugin_names, find_orphaned_plugins, to_smtp_cfg, to_webhook_cfg, to_ansible_cfg, to_plugins_cfg, to_oidc_cfg, to_ldap_cfg, to_thresholds_cfg, to_access_cfg, DEFAULTS, seal_source_tokens, - is_plugin_secret, + is_plugin_secret, NORM_FLOORS, ) settings_bp = Blueprint("settings", __name__, url_prefix="/settings") @@ -145,6 +145,8 @@ _THRESHOLD_FIELDS = [ ("error_5xx_pct", "traefik.threshold.error_5xx_pct", True), ("error_4xx_pct", "traefik.threshold.error_4xx_pct", True), ("min_requests", "traefik.threshold.min_requests", False), + # Each normal's floor (#5742), named by its own setting key. + *[(f"norms.floor.{name}", f"norms.floor.{name}", True) for name in NORM_FLOORS], ] # Docker time-series retention windows (days). Read fresh by the cleanup task, @@ -165,7 +167,8 @@ async def thresholds(): settings = await get_all_settings(db) from steward.core.plugin_manager import _LOADED_PLUGINS return await render_template("settings/thresholds.html", settings=settings, - traefik_on="traefik" in _LOADED_PLUGINS) + traefik_on="traefik" in _LOADED_PLUGINS, + plugins_on=set(_LOADED_PLUGINS)) @settings_bp.post("/thresholds/") diff --git a/steward/templates/hosts/overview_widget.html b/steward/templates/hosts/overview_widget.html index 1428827..1fe2dcb 100644 --- a/steward/templates/hosts/overview_widget.html +++ b/steward/templates/hosts/overview_widget.html @@ -1,6 +1,8 @@ {# hosts/overview_widget.html — the dashboard Hosts widget (#5251): a verdict, then one row per host, worst first, with its resources under one header. - Load is shown against the host's cores and graded per core. Reachability + Load is shown against the host's cores and graded per core. CPU, memory + and load within the host's own normal for the hour read healthy (#5742), + with what is usual on hover. Reachability and uptime belong to the Monitors widget, so they aren't repeated here. #} {% from "_macros.html" import status_dot %} {% if not rows %} @@ -30,10 +32,12 @@ {%- elif not a %}no agent{% endif %} {% for v, kind in ((a.get('cpu_pct'), 'cpu'), (a.get('mem_used_pct'), 'mem'), (a.get('disk_root'), 'disk')) %} - {% if v is none %}—{% else %}{{ "%.0f"|format(v) }}%{% endif %} + {% set g = (r.grades or {}).get(kind) or (none, none) %} + {% if v is none %}—{% else %}{{ "%.0f"|format(v) }}%{% endif %} {% endfor %} - + {% set g = (r.grades or {}).get('load') or (none, none) %} + {%- if a.get('load_1m') is none %}—{% elif a.get('cores') %}{{ "%.1f"|format(a.load_1m) }} / {{ a.cores }} cores{% else %}{{ "%.1f"|format(a.load_1m) }}{% endif -%} diff --git a/steward/templates/monitors/detail.html b/steward/templates/monitors/detail.html index 3dbab84..2337d7e 100644 --- a/steward/templates/monitors/detail.html +++ b/steward/templates/monitors/detail.html @@ -1,7 +1,8 @@ {% extends "base.html" %} {% from "_macros.html" import crumbs, page_header, stat_tile %} {# One monitor: its state and for how long, latency and uptime against - Settings → Thresholds, its response time over the last day, its last + Settings → Thresholds (latency against its own normal once it has one, + with its anomalies), its response time over the last day, its last checks, the failures behind them and how it is set up. #} {% block title %}{{ monitor.name }} — Monitors — Steward{% endblock %} {% block breadcrumb %}{{ crumbs([("Monitors", "/monitors/"), (monitor.name, "")]) }}{% endblock %} @@ -23,7 +24,7 @@ {% set since_hint = ((("for at least " if since_at_least else "for ") ~ (since|ago|striptags|replace(" ago", ""))) if since and state in ("up", "down", "degraded") else none) %} {{ stat_tile("State", status_info(state)[1], state=state, hint=since_hint) }} {% if latest and latest.response_ms is not none %} - {{ stat_tile("Latency", "%.0f ms"|format(latest.response_ms), state=status_class(latest.response_ms, "latency")) }} + {{ stat_tile("Latency", "%.0f ms"|format(latest.response_ms), state=latency[0], hint=latency[1]) }} {% endif %} {% for window in ("24h", "7d", "30d") %} {% set pct = uptime.get(window) %} @@ -34,6 +35,8 @@ {% endif %} +{% with back="/monitors/" ~ monitor.id, base_resource=monitor.name %}{% include "_anomaly_card.html" %}{% endwith %} + {% set timed = history | selectattr("ms", "ne", none) | list %} {% if timed|length > 1 %}

diff --git a/steward/templates/monitors/rows.html b/steward/templates/monitors/rows.html index f6b301d..d8074c5 100644 --- a/steward/templates/monitors/rows.html +++ b/steward/templates/monitors/rows.html @@ -51,7 +51,7 @@ {% if d.state in ('down', 'paused', 'pending') %}{{ status_badge(d.state) }} {% elif latest.response_ms is not none %} {% if d.spark %}{{ d.spark|safe }}{% endif %} - {{ "%.0f"|format(latest.response_ms) }} ms + {{ "%.0f"|format(latest.response_ms) }} ms {% elif m.type == 'dns' and latest.resolved_ip %}{{ latest.resolved_ip }} {% else %}{{ status_badge('up') }}{% endif %} diff --git a/steward/templates/monitors/widget.html b/steward/templates/monitors/widget.html index c37a752..d078280 100644 --- a/steward/templates/monitors/widget.html +++ b/steward/templates/monitors/widget.html @@ -26,10 +26,10 @@ {{- status_dot(d.state) -}} {% if session.user_id %}{{ m.name }}{% else %}{{ m.name }}{% endif %} {%- if d.state == 'down' %}went down {{ d.down_since|ago if d.down_since else '' }}{% if d.down_reason %} · {{ d.down_reason }}{% endif %} - {%- elif d.state == 'degraded' %}{% if latest and status_class(latest.response_ms, 'latency') == 'crit' %}slow replies{% else %}checks failing now and then{% endif %} + {%- elif d.state == 'degraded' %}{% if d.latency == 'crit' %}slow replies{% else %}checks failing now and then{% endif %} {%- elif d.state == 'pending' %}not checked yet{% endif %} - + {%- if latest and latest.is_up and latest.response_ms is not none %}{{ "%.0f"|format(latest.response_ms) }} ms{% else %}—{% endif -%} {% for k in ("24h", "7d", "30d") %}{% set u = d.uptime.get(k) %} diff --git a/steward/templates/settings/thresholds.html b/steward/templates/settings/thresholds.html index caa6cf9..d80be1c 100644 --- a/steward/templates/settings/thresholds.html +++ b/steward/templates/settings/thresholds.html @@ -35,12 +35,12 @@ Higher is worse for everything except uptime, where the cutoffs are floors.

- {{ pair("CPU usage", "cpu_warn", "thresholds.cpu_warn", "cpu_crit", "thresholds.cpu_crit", "%", "Average CPU utilization per host.") }} - {{ pair("Memory usage", "mem_warn", "thresholds.mem_warn", "mem_crit", "thresholds.mem_crit", "%", "Memory in use per host.") }} + {{ pair("CPU usage", "cpu_warn", "thresholds.cpu_warn", "cpu_crit", "thresholds.cpu_crit", "%", "Average CPU utilization per host. A host or container with its own normal is judged against that instead (see Normals below).") }} + {{ pair("Memory usage", "mem_warn", "thresholds.mem_warn", "mem_crit", "thresholds.mem_crit", "%", "Memory in use per host, or a container's share of its limit. Judged against a normal once there is one.") }} {{ pair("Disk usage", "disk_warn", "thresholds.disk_warn", "disk_crit", "thresholds.disk_crit", "%", "Filesystem usage.") }} - {{ pair("Load", "load_warn", "thresholds.load_warn", "load_crit", "thresholds.load_crit", "% per core", "Load average normalized by core count (100% = one full core per core).") }} + {{ pair("Load", "load_warn", "thresholds.load_warn", "load_crit", "thresholds.load_crit", "% per core", "Load average normalized by core count (100% = one full core per core). Judged against a host's normal once it has one.") }} {{ pair("Temperature", "temp_warn", "thresholds.temp_warn", "temp_crit", "thresholds.temp_crit", "°C", "Hottest reported sensor per host.") }} - {{ pair("Response time", "latency_warn", "ping.threshold.good_ms", "latency_crit", "ping.threshold.warn_ms", "ms", "How long a monitor's check takes to answer. Also grades Traefik routers' p95 (until a router has its own normal) and the UniFi WAN round trip.") }} + {{ pair("Response time", "latency_warn", "ping.threshold.good_ms", "latency_crit", "ping.threshold.warn_ms", "ms", "How long a monitor's check takes to answer. Also grades Traefik routers' p95 and the UniFi WAN round trip. A monitor or router with its own normal is judged against that instead.") }} {{ pair("Uptime / SLA", "uptime_warn", "thresholds.uptime_warn", "uptime_crit", "thresholds.uptime_crit", "%", "Lower is worse — amber below warn, red below crit.", step="0.1") }} {{ pair("TLS certificate expiry", "tls_warn", "thresholds.tls_warn_days", "tls_crit", "thresholds.tls_crit_days", "days left", "HTTPS monitors whose certificate expires within these many days are flagged. Lower is worse.") }} {{ pair("Clock drift", "clock_warn", "thresholds.clock_warn_s", "clock_crit", "thresholds.clock_crit_s", "seconds", "How far a host's clock may be from Steward's, either way, before its page and the Hosts list call it out.") }} @@ -99,6 +99,42 @@ {% endmacro %} +{# Each normal's floor (#5742): the smallest departure from it worth calling an anomaly. #} +{% macro floor(label, key, unit, hint, step="1") %} +
+ + + {% if hint %}
{{ hint }}
{% endif %} +
+{% endmacro %} + +
+

Normals

+

+ Once one has 5 days of history, these learn what is normal for each host, container, monitor or router + at each hour, and are judged against that instead of the cutoffs above: a reading over a cutoff but usual + for that hour shows healthy, and one well above its usual level is a warning and an anomaly to review. + Disk space, a monitor that is down, an agent that stopped reporting and a router's 5xx share are failures + and always keep their cutoffs. A reading counts as out of its normal only when it is a quarter above its + usual level and at least this much above it: +

+ {% if "host_agent" in (plugins_on or ()) %} + {{ floor("Host CPU", "norms.floor.host_cpu_pct", "% points", "CPU is bursty: a build or a backup moves it this much without anything being wrong.") }} + {{ floor("Host memory", "norms.floor.host_mem_pct", "% points", "Memory moves slowly, so a smaller step already means something changed.") }} + {{ floor("Host load per core", "norms.floor.host_load_pct", "% points", "Bursty, like CPU.") }} + {% endif %} + {% if "docker" in (plugins_on or ()) %} + {{ floor("Container CPU", "norms.floor.container_cpu_pct", "% of one core", "10 is a tenth of a core; with many containers, smaller steps are everyday noise.") }} + {{ floor("Container memory", "norms.floor.container_mem_pct", "% points", "Of its limit, or of the host's memory when it has none, so a container without a limit only counts large growth.") }} + {% endif %} + {{ floor("Monitor response time", "norms.floor.monitor_ms", "ms", "One check per interval is noisy; a slower reply than this over its usual is worth a look.") }} + {% if traefik_on %} + {{ floor("Router 4xx share", "norms.floor.router_4xx_pct", "% points", "", step="0.1") }} + {{ floor("Router p95", "norms.floor.router_p95_ms", "ms", "") }} + {% endif %} +
+

Metric anomalies

diff --git a/tests/core/test_norm_grading.py b/tests/core/test_norm_grading.py new file mode 100644 index 0000000..646f932 --- /dev/null +++ b/tests/core/test_norm_grading.py @@ -0,0 +1,135 @@ +"""Hosts, containers and monitor response times are graded against their own +normal (#5742, System 182): a reading past its fixed cutoff but within its +normal for the hour reads healthy and says what is usual; one well out of it +keeps its grade. How far out counts is each normal's floor, a setting on +Settings → Thresholds. Disk keeps its cutoff whatever the history.""" +from types import SimpleNamespace as NS + +from quart import render_template, session + +from plugins.docker.service import usage +from plugins.host_agent.routes import _expand_sample_to_metrics +from steward.core import norms +from steward.core.anomalies import graded +from steward.core.formatting import worst_reading +from steward.core.norms import Norm, floor_setting +from steward.core.settings import DEFAULTS, NORM_FLOORS, floor_of, to_thresholds_cfg +from steward.hosts.service import host_glance, host_grades +from steward.monitors.service import latency_grade, row_state + +TH = to_thresholds_cfg({}) # cpu 80/90, mem 80/90, disk 80/90, load 80/100, latency 50/200 ms + + +def _norm(hold): + return Norm(kind="day", slot=14, usual_hold=hold, usual_peak=hold * 1.2, days_seen=7) + + +def test_each_floor_is_a_setting_with_the_chosen_default(): + assert {n: floor_of(TH, n) for n in NORM_FLOORS} == { + "host_cpu_pct": 10.0, "host_mem_pct": 5.0, "host_load_pct": 10.0, + "container_cpu_pct": 10.0, "container_mem_pct": 5.0, "monitor_ms": 100.0, + "router_4xx_pct": 2.0, "router_p95_ms": 50.0} + assert floor_of(to_thresholds_cfg({"norms.floor.host_cpu_pct": 25}), "host_cpu_pct") == 25.0 + assert floor_of({}, "monitor_ms") == DEFAULTS["norms.floor.monitor_ms"] + + +def test_a_floor_from_settings_is_read_at_each_pass(): + app = NS(config={"THRESHOLDS": to_thresholds_cfg({})}) + norms.learn_normal("t5742", "x_pct", floor_setting(app, "host_mem_pct")) + try: + learned = norms.LEARNED[("t5742", "x_pct")] + assert learned.least_departure == 5.0 + app.config["THRESHOLDS"] = to_thresholds_cfg({"norms.floor.host_mem_pct": 12}) + assert learned.least_departure == 12.0 + finally: + norms.LEARNED.pop(("t5742", "x_pct"), None) + + +def test_past_the_cutoff_but_within_its_normal_reads_healthy(): + assert graded(93.0, "cpu", TH, _norm(88.0), 10.0, "%") == ("ok", "usual 88.0% at this hour") + assert graded(93.0, "cpu", TH, None, 10.0, "%") == ("crit", None) + # Out of its normal: a quarter over the usual hold and past the floor. + assert graded(93.0, "cpu", TH, _norm(40.0), 10.0, "%") == ("crit", None) + # Under the cutoff there is nothing to quiet. + assert graded(30.0, "cpu", TH, _norm(5.0), 10.0, "%") == ("ok", None) + + +def test_a_departure_smaller_than_the_floor_is_still_usual(): + # 1.25 × 70 is 87.5 and 91 is over it, but only 21 over: under a 25-point floor. + assert graded(91.0, "cpu", TH, _norm(70.0), 25.0, "%")[0] == "ok" + assert graded(91.0, "cpu", TH, _norm(70.0), 10.0, "%")[0] == "crit" + + +def test_a_hosts_worst_reading_skips_what_is_usual_for_it(): + readings = (("CPU", 95.0, "cpu"), ("Memory", 85.0, "mem"), ("Disk /", 50.0, "disk")) + assert worst_reading(readings, TH) == ("CPU", 95.0, "crit") + usual = {"cpu": (_norm(92.0), 10.0), "mem": (_norm(84.0), 5.0)} + assert worst_reading(readings, TH, usual) is None + + +def test_disk_keeps_its_cutoff_whatever_the_history(): + grades = host_grades({"cpu": 95.0, "disk": 95.0}, {"cpu": _norm(93.0)}, TH) + assert grades["cpu"] == ("ok", "usual 93.0% at this hour") + assert grades["disk"] == ("crit", None) + + +def test_the_hosts_glance_grades_each_host_by_its_own_normal(): + hosts = [NS(id="b", name="builder"), NS(id="w", name="web")] + agent = {"builder": {"cpu_pct": 95.0, "load_per_core": 120, "fresh": True, + "norms": {"cpu": _norm(90.0), "load": _norm(110.0)}}, + "web": {"cpu_pct": 95.0, "fresh": True, "norms": {"cpu": _norm(20.0)}}} + rows = {r["host"].name: r for r in host_glance(hosts, {}, agent, TH)} + assert rows["builder"]["state"] == "up" and rows["builder"]["worst"] is None + assert rows["builder"]["grades"]["load"] == ("ok", "usual 110.0% at this hour") + assert rows["web"]["state"] == "crit" and rows["web"]["worst"][0] == "CPU" + + +def test_a_monitor_slow_as_usual_is_up_not_degraded(): + monitor = NS(enabled=True) + slow = NS(is_up=True, response_ms=450.0) + assert row_state(monitor, slow, 100.0, TH) == "degraded" + assert row_state(monitor, slow, 100.0, TH, _norm(400.0)) == "up" + assert latency_grade(slow, TH, _norm(400.0)) == ("ok", "usual 400 ms at this hour") + # 1.25 × 300 = 375, and 150 over is past the 100 ms floor. + assert row_state(monitor, slow, 100.0, TH, _norm(300.0)) == "degraded" + + +def test_a_monitor_down_stays_down_whatever_its_normal(): + down = NS(is_up=False, response_ms=None) + assert row_state(NS(enabled=True), down, 100.0, TH, _norm(400.0)) == "down" + + +def test_a_containers_cpu_is_learned_per_core_and_said_in_cores(): + # 3.8 cores of a 4-core host is 95% of it: crit, unless 3.6 cores is usual. + c = NS(cpu_pct=380.0, mem_pct=50.0, mem_usage_bytes=2**30, mem_limit_bytes=2 * 2**30) + u = usage(c, 4, 64 * 2**30, thresholds=TH, norms={"cpu_pct": _norm(360.0)}) + assert u["cpu_grade"] == "ok" and u["cpu_usual"] == "usual 3.60 cores at this hour" + assert usage(c, 4, 64 * 2**30, thresholds=TH)["cpu_grade"] == "crit" + assert usage(c, 4, 64 * 2**30)["cpu_grade"] is None # no thresholds: the page grades + + +def test_a_containers_memory_within_its_normal_reads_healthy(): + c = NS(cpu_pct=1.0, mem_pct=92.0, mem_usage_bytes=2**30, mem_limit_bytes=2 * 2**30) + u = usage(c, 4, 64 * 2**30, thresholds=TH, norms={"mem_pct": _norm(90.0)}) + assert (u["mem_grade"], u["mem_usual"]) == ("ok", "usual 90.0% at this hour") + assert usage(c, 4, 64 * 2**30, thresholds=TH)["mem_grade"] == "crit" + + +def test_ingest_records_load_per_core(): + sample = {"load": {"1m": 6.0, "5m": 5.0, "15m": 4.0}, "cpu_cores": [10.0] * 8} + rows = _expand_sample_to_metrics(sample, "h", None) + (load_pct,) = [r for r in rows if r.metric_name == "load_pct"] + assert load_pct.resource_name == "h" and load_pct.value == 75.0 + assert not [r for r in _expand_sample_to_metrics({"load": {"1m": 1.0}}, "h", None) + if r.metric_name == "load_pct"] # no cores reported: no figure + + +async def test_the_thresholds_page_offers_each_running_kinds_floor(app): + async with app.test_request_context("/settings/thresholds/"): + session["user_role"] = "admin" + page = await render_template("settings/thresholds.html", settings=dict(DEFAULTS), + traefik_on=False, plugins_on={"host_agent"}) + assert 'id="normals"' in page + assert 'name="norms.floor.host_cpu_pct"' in page and 'name="norms.floor.monitor_ms"' in page + assert 'name="norms.floor.container_cpu_pct"' not in page # docker isn't running + assert 'name="norms.floor.router_4xx_pct"' not in page diff --git a/tests/integration/test_anomalies.py b/tests/integration/test_anomalies.py index 90275bf..9691a71 100644 --- a/tests/integration/test_anomalies.py +++ b/tests/integration/test_anomalies.py @@ -333,3 +333,45 @@ def test_a_metric_learns_only_from_its_since(app, src): learned, held_back = asyncio.run(_both()) assert learned is not None and learned.usual_hold == 90.0 assert held_back is None + + +def test_a_metric_learned_without_its_parts_ignores_them(app, src): + """A host's CPU learns a normal and each of its cores doesn't (#5742): + with parts=False, ":core0" is neither learned nor judged, and the + floor may be a function read at each pass.""" + from sqlalchemy import select + from steward.core import norms + from steward.core.anomalies import detect, levels_now + from steward.core.norms import norms_at, rebuild_norms + from steward.models.metrics import PluginMetricHourly + from steward.models.norms import MetricAnomaly + host = _res("whole") + part = f"{host}:core0" + rows = [PluginMetricHourly(source_module=src, resource_name=r, metric_name=METRIC, + bucket=NOW.replace(hour=0) - timedelta(days=d) + timedelta(hours=h), + value_avg=10.0, value_max=10.0, value_p95=10.0, sample_count=60) + for r in (host, part) for d in range(1, 8) for h in range(24)] + at = NOW + timedelta(days=3) + + async def _go(): + norms.learn_normal(src, METRIC, lambda: 5.0, parts=False) + async with app.db_sessionmaker() as s: + async with s.begin(): + s.add_all(rows) + async with s.begin(): + await rebuild_norms(s, NOW) + learned = await norms_at(s, [(src, host, METRIC), (src, part, METRIC)], NOW.replace(hour=14)) + async with s.begin(): + await _report(s, src, host, at, 90.0) + await _report(s, src, part, at, 90.0) + levels = await levels_now(s, at) + async with s.begin(): + await detect(s, at) + opened = list((await s.execute(select(MetricAnomaly.resource_name).where( + MetricAnomaly.source_module == src))).scalars()) + return learned, levels, opened + + learned, levels, opened = asyncio.run(_go()) + assert set(learned) == {(src, host, METRIC)} + assert (src, host, METRIC) in levels and (src, part, METRIC) not in levels + assert opened == [host] # 90 against a usual 10: the host departs, its core isn't judged