Host CPU, memory and load per core, container CPU and memory, and every monitor's response time learn a normal per hour (milestone 513, System 182). A reading past its fixed cutoff but within its normal reads healthy and says what is usual at this hour; one well out of it keeps its grade and is a metric anomaly. Disk, a monitor down and an agent offline keep their cutoffs. - Each normal's floor, the smallest departure that counts, is a setting: Settings -> Thresholds -> Normals (norms.floor.*). Defaults: host CPU 10 points, host memory 5, load per core 10, container CPU 10 points of one core, container memory 5, monitor response 100 ms; Traefik's 2-point 4xx and 50 ms p95 floors move there from constants. learn_normal takes a floor function (floor_setting) read at each pass. - learn_normal(parts=False): a host learns, its cores don't. - Ingest records load_pct (load_1m / cores x 100), the figure pages grade. - Shared grading: anomalies.against / graded, worst_reading(normals=), hosts' host_grades, docker usage(thresholds=, norms=), monitors' latency_grade. Applied to host vitals, the Hosts widget, MCP overview, Docker usage, and the monitor list, widget and page. - The monitor and container pages list their anomalies. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
244 lines
11 KiB
Python
244 lines
11 KiB
Python
"""Unified monitors management surface (/monitors) — ping, DNS, HTTP.
|
|
|
|
Replaces the former /ping, /dns and /plugins/http pages. One list, one
|
|
type-aware add/edit form, one dashboard widget. A monitor may be linked to a
|
|
Host (host_id) or watch a free-standing custom destination (host_id = NULL).
|
|
"""
|
|
from __future__ import annotations
|
|
import uuid
|
|
from datetime import datetime, timedelta, timezone
|
|
|
|
from quart import Blueprint, current_app, render_template, request, redirect, url_for
|
|
|
|
from steward.auth.middleware import require_role
|
|
from steward.core import anomalies as anom, renames
|
|
from steward.core.audit import audit_request
|
|
from steward.models.users import UserRole
|
|
from steward.models.monitors import Monitor, MonitorType, MONITOR_TYPE_VALUES
|
|
from steward.core.status import _heartbeat_title, streak_start
|
|
from steward.models.hosts import Host
|
|
from steward.monitors import service as monitor_svc
|
|
|
|
monitors_bp = Blueprint("monitors", __name__, url_prefix="/monitors")
|
|
|
|
|
|
def _form_opts(form, mtype: str) -> dict:
|
|
"""The submitted fields of `mtype`'s config, for monitor_svc.build_config.
|
|
An unchecked checkbox is absent from the form, so it reads as False."""
|
|
if mtype == MonitorType.tcp.value:
|
|
return {"port": form.get("port")}
|
|
if mtype == MonitorType.dns.value:
|
|
return {"expected_ip": form.get("expected_ip"), "resolver": form.get("resolver")}
|
|
if mtype == MonitorType.http.value:
|
|
return {
|
|
"method": form.get("method"),
|
|
"expected_status": form.get("expected_status"),
|
|
"content_match": form.get("content_match"),
|
|
"headers": form.get("headers_json"),
|
|
"timeout_seconds": form.get("timeout_seconds"),
|
|
"follow_redirects": "follow_redirects" in form,
|
|
"verify_ssl": "verify_ssl" in form,
|
|
}
|
|
return {}
|
|
|
|
|
|
@monitors_bp.get("/")
|
|
@require_role(UserRole.viewer)
|
|
async def index():
|
|
poll_interval = current_app.config.get("MONITORS_POLL_INTERVAL", 60)
|
|
return await render_template("monitors/index.html", poll_interval=poll_interval)
|
|
|
|
|
|
@monitors_bp.get("/rows")
|
|
@require_role(UserRole.viewer)
|
|
async def rows():
|
|
async with current_app.db_sessionmaker() as db:
|
|
data, counts = await monitor_svc.monitor_rows(db, current_app.config.get("THRESHOLDS", {}))
|
|
return await render_template("monitors/rows.html", monitor_data=data, counts=counts)
|
|
|
|
|
|
@monitors_bp.post("/add")
|
|
@require_role(UserRole.operator)
|
|
async def add_monitor():
|
|
form = await request.form
|
|
mtype = form.get("type", "").strip()
|
|
try:
|
|
fields = monitor_svc.monitor_fields(
|
|
mtype, target=form.get("target", ""), name=form.get("name", ""),
|
|
check_interval_seconds=form.get("check_interval_seconds"),
|
|
opts=_form_opts(form, mtype))
|
|
except ValueError as exc:
|
|
return await _new_form(form, mtype, str(exc)), 400
|
|
|
|
host_id = form.get("host_id", "").strip() or None
|
|
monitor = Monitor(id=str(uuid.uuid4()), host_id=host_id, enabled=True,
|
|
created_at=datetime.now(timezone.utc), **fields)
|
|
async with current_app.db_sessionmaker() as db:
|
|
async with db.begin():
|
|
try:
|
|
await monitor_svc.check_name_free(db, monitor.type, monitor.name)
|
|
except ValueError as exc:
|
|
return await _new_form(form, mtype, str(exc)), 400
|
|
db.add(monitor)
|
|
|
|
# A host-scoped add returns to that host's hub; otherwise the new monitor's page.
|
|
if host_id:
|
|
return redirect(url_for("hosts.host_detail", host_id=host_id))
|
|
return redirect(url_for("monitors.detail", monitor_id=monitor.id))
|
|
|
|
|
|
async def _new_form(form, mtype: str, error: str | None = None) -> str:
|
|
"""The add form, holding what was typed when `error` sends it back. Opened
|
|
from a host's page (host_id), it files the monitor under that host and
|
|
starts from the host's address."""
|
|
values = {k: form.get(k, "") for k in ("name", "target", "host_id")}
|
|
values["check_interval_seconds"] = form.get("check_interval_seconds", 0)
|
|
host = None
|
|
if values["host_id"]:
|
|
async with current_app.db_sessionmaker() as db:
|
|
host = await db.get(Host, values["host_id"])
|
|
if host is None:
|
|
values["host_id"] = ""
|
|
elif not values["target"]:
|
|
values["target"] = host.address or host.name
|
|
return await render_template(
|
|
"monitors/form.html", monitor=None, values=values, host=host,
|
|
mtype=mtype if mtype in MONITOR_TYPE_VALUES else MonitorType.icmp.value,
|
|
config=_form_opts(form, mtype) if error and mtype in MONITOR_TYPE_VALUES else {},
|
|
types=list(MonitorType), error=error,
|
|
)
|
|
|
|
|
|
@monitors_bp.get("/new")
|
|
@require_role(UserRole.operator)
|
|
async def new_form():
|
|
return await _new_form({"host_id": request.args.get("host_id", "")}, MonitorType.icmp.value)
|
|
|
|
|
|
@monitors_bp.get("/<monitor_id>")
|
|
@require_role(UserRole.viewer)
|
|
async def detail(monitor_id: str):
|
|
async with current_app.db_sessionmaker() as db:
|
|
m = await db.get(Monitor, monitor_id)
|
|
if not m:
|
|
return redirect(url_for("monitors.index"))
|
|
st = (await monitor_svc.monitor_states(db, [m]))[m.id]
|
|
failures = await monitor_svc.recent_failures(db, m.id)
|
|
history = await monitor_svc.response_history(db, m.id, datetime.now(timezone.utc))
|
|
norm = (await monitor_svc.latency_norms(db, [m])).get(m.id)
|
|
anomalies = await anom.views(db, await anom.list_anomalies(
|
|
db, source=m.type, resource=m.name,
|
|
since=datetime.now(timezone.utc) - timedelta(days=30), limit=20), links=False)
|
|
host = await db.get(Host, m.host_id) if m.host_id else None
|
|
thresholds = current_app.config.get("THRESHOLDS", {})
|
|
latest = st["latest"]
|
|
since, since_at_least = streak_start(st["recent"])
|
|
if st["down_since"]:
|
|
since, since_at_least = st["down_since"], False
|
|
tls_days = None
|
|
if latest and latest.tls_expires_at:
|
|
tls_days = (latest.tls_expires_at - datetime.now(timezone.utc)).total_seconds() / 86400.0
|
|
return await render_template(
|
|
"monitors/detail.html", monitor=m, host=host, latest=latest, failures=failures,
|
|
state=monitor_svc.row_state(m, latest, st["uptime"].get("24h"), thresholds, norm),
|
|
latency=monitor_svc.latency_grade(latest, thresholds, norm), anomalies=anomalies,
|
|
since=since, since_at_least=since_at_least, uptime=st["uptime"], tls_days=tls_days,
|
|
heartbeat=[{"state": "up" if r.is_up else "down", "title": _heartbeat_title(m.type, r)}
|
|
for r in st["recent"]],
|
|
history=history, poll_interval=current_app.config.get("MONITORS_POLL_INTERVAL", 60),
|
|
)
|
|
|
|
|
|
@monitors_bp.get("/<monitor_id>/edit")
|
|
@require_role(UserRole.operator)
|
|
async def edit_form(monitor_id: str):
|
|
async with current_app.db_sessionmaker() as db:
|
|
m = await db.get(Monitor, monitor_id)
|
|
if not m:
|
|
return redirect(url_for("monitors.index"))
|
|
return await render_template(
|
|
"monitors/form.html", monitor=m, values=None, mtype=m.type, config=m.config,
|
|
types=list(MonitorType), error=None,
|
|
)
|
|
|
|
|
|
@monitors_bp.post("/<monitor_id>/edit")
|
|
@require_role(UserRole.operator)
|
|
async def edit_monitor(monitor_id: str):
|
|
form = await request.form
|
|
async with current_app.db_sessionmaker() as db:
|
|
async with db.begin():
|
|
m = await db.get(Monitor, monitor_id)
|
|
if not m:
|
|
return redirect(url_for("monitors.index"))
|
|
# Type is immutable on edit (changes the meaning of config/target);
|
|
# to switch type, delete and re-add.
|
|
try:
|
|
fields = monitor_svc.monitor_fields(
|
|
m.type, target=form.get("target", m.target), name=form.get("name", ""),
|
|
check_interval_seconds=form.get("check_interval_seconds"),
|
|
opts=_form_opts(form, m.type))
|
|
if fields["name"] != m.name:
|
|
await monitor_svc.check_name_free(db, m.type, fields["name"], exclude_id=m.id)
|
|
except ValueError as exc:
|
|
return str(exc), 400
|
|
old_name = m.name
|
|
for k, v in fields.items():
|
|
setattr(m, k, v)
|
|
# Its rules, windows and history go by (type, name): move them with it.
|
|
moved = await renames.carry_rename(db, ((m.type, None),), old_name, m.name)
|
|
if moved:
|
|
await audit_request("monitor.updated", "monitor", m.name,
|
|
{"monitor_id": m.id, "renamed_from": old_name, "carried": moved})
|
|
return redirect(url_for("monitors.detail", monitor_id=m.id))
|
|
|
|
|
|
@monitors_bp.post("/<monitor_id>/toggle")
|
|
@require_role(UserRole.operator)
|
|
async def toggle_monitor(monitor_id: str):
|
|
async with current_app.db_sessionmaker() as db:
|
|
async with db.begin():
|
|
m = await db.get(Monitor, monitor_id)
|
|
if m:
|
|
m.enabled = not m.enabled
|
|
return redirect(request.referrer or url_for("monitors.index"))
|
|
|
|
|
|
@monitors_bp.post("/<monitor_id>/delete")
|
|
@require_role(UserRole.operator)
|
|
async def delete_monitor(monitor_id: str):
|
|
async with current_app.db_sessionmaker() as db:
|
|
async with db.begin():
|
|
m = await db.get(Monitor, monitor_id)
|
|
if m:
|
|
await db.delete(m)
|
|
referrer = request.referrer or ""
|
|
if f"/monitors/{monitor_id}" in referrer:
|
|
return redirect(url_for("monitors.index"))
|
|
return redirect(referrer or url_for("monitors.index"))
|
|
|
|
|
|
@monitors_bp.get("/widget")
|
|
@require_role(UserRole.viewer)
|
|
async def widget():
|
|
type_filter = request.args.get("type_filter", "all")
|
|
show_down_only = request.args.get("show_down_only", "no") == "yes"
|
|
limit = max(1, min(50, int(request.args.get("limit", 10) or 10)))
|
|
widget_id = request.args.get("wid", "0")
|
|
|
|
async with current_app.db_sessionmaker() as db:
|
|
data, counts = await monitor_svc.monitor_rows(db, current_app.config.get("THRESHOLDS", {}))
|
|
if type_filter in MONITOR_TYPE_VALUES:
|
|
data = [d for d in data if d["monitor"].type == type_filter]
|
|
# Counted after the type filter, so the verdict speaks for what is listed.
|
|
counts = {k: sum(d["state"] == k for d in data) for k in ("up", "degraded", "down", "pending", "paused")}
|
|
problems = [d for d in data if d["state"] in ("down", "degraded", "pending")]
|
|
rest = [] if show_down_only else [d for d in data if d["state"] not in ("down", "degraded", "pending")]
|
|
shown = max(0, limit - len(problems))
|
|
|
|
return await render_template(
|
|
"monitors/widget.html",
|
|
problems=problems, rest=rest[:shown], more=len(rest) - min(len(rest), shown),
|
|
counts=counts, total=len(data), show_down_only=show_down_only, widget_id=widget_id,
|
|
)
|