#5065 "Renames carry their alerts". Alert rules, maintenance-window scopes and metric history key a thing by (source, name), so a rename used to leave the rules watching a name nothing reported under, and split the history. - core/renames.carry_rename moves AlertRule.resource_name, MaintenanceWindow.scope_resource, plugin_metrics and plugin_metrics_hourly to the new name in the caller's transaction. A host rename also carries its parts: host_agent "<host>:…" and docker "<host>/…" (HOST_SOURCES). Matching is exact (a prefix compare, not LIKE or a collation range). History left under the new name by something deleted is dropped first rather than merged in. - Web monitor edit, MCP update_monitor and web host edit call it; MCP answers with `carried` (it used to return a warning naming stranded rules). - A name another monitor of the same type, or another host, already has is refused on create and rename: two would share one history. - The rename is audited (monitor.updated / host.renamed) with what moved. Tests: an integration test renames a firing monitor through the web route and finds its rule still firing and cleared by a sample on the new name, its window and raw+hourly history moved and an orphan's history dropped; a host rename moves agent and container parts but not a longer-named host's or another source's; a guard that every host_agent resource is "<host>" or "<host>:…". Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
217 lines
9.1 KiB
Python
217 lines
9.1 KiB
Python
"""Monitor queries and validation shared by the monitors pages and the MCP tools.
|
|
|
|
Each function takes an open session (or plain values), with no Quart request
|
|
context, so the routes and the /mcp tools (which run outside Quart's
|
|
pipeline) call the same code and refuse the same input for the same reasons.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
from collections.abc import Mapping
|
|
from datetime import datetime, timezone
|
|
|
|
from sqlalchemy import func, select
|
|
|
|
from steward.core.status import _heartbeat_title, _last_n_results, _uptime_by_monitor
|
|
from steward.core.time_range import parse_range
|
|
from steward.models.monitors import MONITOR_TYPE_VALUES, Monitor, MonitorResult, MonitorType
|
|
|
|
HTTP_METHODS = ("GET", "HEAD", "POST")
|
|
# The config keys each type takes (models/monitors.py documents their meaning).
|
|
CONFIG_KEYS: dict[str, tuple[str, ...]] = {
|
|
MonitorType.icmp.value: (),
|
|
MonitorType.tcp.value: ("port",),
|
|
MonitorType.dns.value: ("expected_ip",),
|
|
MonitorType.http.value: ("method", "expected_status", "content_match", "headers",
|
|
"timeout_seconds", "follow_redirects", "verify_ssl"),
|
|
}
|
|
|
|
|
|
def _whole(name: str, value, default: int, lo: int, hi: int | None = None) -> int:
|
|
if value is None or value == "":
|
|
return default
|
|
if isinstance(value, bool):
|
|
raise ValueError(f"{name} must be a whole number, got {value!r}.")
|
|
try:
|
|
n = int(value)
|
|
except (TypeError, ValueError):
|
|
raise ValueError(f"{name} must be a whole number, got {value!r}.") from None
|
|
if n < lo or (hi is not None and n > hi):
|
|
bounds = f"between {lo} and {hi}" if hi is not None else f"at least {lo}"
|
|
raise ValueError(f"{name} must be {bounds}, got {n}.")
|
|
return n
|
|
|
|
|
|
def _headers(value) -> dict:
|
|
if value is None or value == "":
|
|
return {}
|
|
if isinstance(value, str):
|
|
try:
|
|
value = json.loads(value)
|
|
except ValueError:
|
|
raise ValueError("headers must be a JSON object of name: value strings.") from None
|
|
if not isinstance(value, dict) or not all(isinstance(k, str) and isinstance(v, str)
|
|
for k, v in value.items()):
|
|
raise ValueError("headers must be an object of name: value strings.")
|
|
return value
|
|
|
|
|
|
def normalise_target(mtype: str, target: str) -> str:
|
|
"""For http, default a bare host to https://; others pass through."""
|
|
target = (target or "").strip()
|
|
if mtype == MonitorType.http.value and target and not target.startswith(("http://", "https://")):
|
|
return "https://" + target
|
|
return target
|
|
|
|
|
|
def build_config(mtype: str, opts: Mapping) -> dict:
|
|
"""The type-specific config from already-typed or form-string values.
|
|
|
|
Missing keys take their defaults; a key the type does not take, or a
|
|
value out of range, raises ValueError saying which and why.
|
|
"""
|
|
unknown = sorted(set(opts) - set(CONFIG_KEYS[mtype]))
|
|
if unknown:
|
|
allowed = ", ".join(CONFIG_KEYS[mtype]) or "no config"
|
|
raise ValueError(f"a {mtype} monitor does not take {', '.join(unknown)}; it takes {allowed}.")
|
|
if mtype == MonitorType.tcp.value:
|
|
return {"port": _whole("port", opts.get("port"), 80, 1, 65535)}
|
|
if mtype == MonitorType.dns.value:
|
|
return {"expected_ip": (str(opts.get("expected_ip") or "").strip() or None)}
|
|
if mtype == MonitorType.http.value:
|
|
method = str(opts.get("method") or "GET").strip().upper()
|
|
if method not in HTTP_METHODS:
|
|
raise ValueError(f"method {method!r} is not one of {', '.join(HTTP_METHODS)}.")
|
|
return {
|
|
"method": method,
|
|
"expected_status": _whole("expected_status", opts.get("expected_status"), 200, 100, 599),
|
|
"content_match": str(opts.get("content_match") or "").strip(),
|
|
"headers": _headers(opts.get("headers")),
|
|
"timeout_seconds": _whole("timeout_seconds", opts.get("timeout_seconds"), 10, 1, 60),
|
|
"follow_redirects": bool(opts.get("follow_redirects", True)),
|
|
"verify_ssl": bool(opts.get("verify_ssl", True)),
|
|
}
|
|
return {} # icmp
|
|
|
|
|
|
def monitor_fields(mtype: str, *, target: str, name: str = "", check_interval_seconds=0,
|
|
opts: Mapping | None = None) -> dict:
|
|
"""Validated column values for a new or edited monitor; ValueError says why not."""
|
|
mtype = (mtype or "").strip().lower()
|
|
if mtype not in MONITOR_TYPE_VALUES:
|
|
raise ValueError(f"type {mtype!r} is not one of {', '.join(MONITOR_TYPE_VALUES)}.")
|
|
target = normalise_target(mtype, target)
|
|
if not target:
|
|
raise ValueError("a monitor needs a target: an address, or a URL for http.")
|
|
if len(target) > 2048:
|
|
raise ValueError("target is longer than 2048 characters.")
|
|
return {
|
|
"type": mtype,
|
|
"target": target,
|
|
"name": ((name or "").strip() or target)[:128],
|
|
"config_json": json.dumps(build_config(mtype, opts or {})),
|
|
"check_interval_seconds": _whole("check_interval_seconds", check_interval_seconds, 0, 0),
|
|
}
|
|
|
|
|
|
async def check_name_free(db, mtype: str, name: str, exclude_id: str | None = None) -> None:
|
|
"""ValueError when another monitor of this type already has this name.
|
|
|
|
A monitor's checks are recorded, and its alert rules watch it, by
|
|
(type, name), so two monitors sharing both would share one history.
|
|
"""
|
|
q = select(Monitor.id).where(Monitor.type == mtype, Monitor.name == name)
|
|
if exclude_id:
|
|
q = q.where(Monitor.id != exclude_id)
|
|
if (await db.execute(q.limit(1))).first():
|
|
raise ValueError(f"another {mtype} monitor is already named {name!r}; its checks and "
|
|
"alert rules go by that name, so pick a different one.")
|
|
|
|
|
|
async def monitor_rows(db, range_key: str):
|
|
"""Per-monitor state: latest result, heartbeat strip, uptime and TLS days.
|
|
|
|
Returns (rows, up, down, pending, range_key). `range_key` picks the uptime
|
|
window surfaced (24h/7d/30d; anything else falls back to 24h) and comes
|
|
back normalised.
|
|
"""
|
|
_since, range_key = parse_range(range_key)
|
|
monitors = list((await db.execute(select(Monitor).order_by(Monitor.name))).scalars())
|
|
ids = [m.id for m in monitors]
|
|
recent = await _last_n_results(db, ids)
|
|
uptime = await _uptime_by_monitor(db, ids)
|
|
|
|
# Map the selected range to the uptime bucket we surface in the list.
|
|
bucket = {"24h": "24h", "7d": "7d", "30d": "30d"}.get(range_key, "24h")
|
|
now = datetime.now(timezone.utc)
|
|
data = []
|
|
for m in monitors:
|
|
rows = recent.get(m.id, [])
|
|
latest = rows[-1] if rows else None
|
|
tls_days = None
|
|
if latest and latest.tls_expires_at:
|
|
tls_days = (latest.tls_expires_at - now).total_seconds() / 86400.0
|
|
data.append({
|
|
"monitor": m,
|
|
"latest": latest,
|
|
"heartbeat": [{"state": "up" if r.is_up else "down",
|
|
"title": _heartbeat_title(m.type, r)} for r in rows],
|
|
"uptime_pct": uptime.get(m.id, {}).get(bucket),
|
|
"tls_days": tls_days,
|
|
})
|
|
up = sum(1 for d in data if d["latest"] and d["latest"].is_up)
|
|
down = sum(1 for d in data if d["latest"] and not d["latest"].is_up)
|
|
pending = sum(1 for d in data if not d["latest"])
|
|
return data, up, down, pending, range_key
|
|
|
|
|
|
async def down_since(db, monitor_id: str):
|
|
"""When the monitor's current run of failures began, or None if its latest check is up.
|
|
|
|
The first failed check after the last success; if it has never succeeded,
|
|
its first check.
|
|
"""
|
|
last_up = (await db.execute(
|
|
select(func.max(MonitorResult.checked_at))
|
|
.where(MonitorResult.monitor_id == monitor_id, MonitorResult.is_up.is_(True))
|
|
)).scalar()
|
|
stmt = select(func.min(MonitorResult.checked_at)).where(MonitorResult.monitor_id == monitor_id)
|
|
if last_up is not None:
|
|
stmt = stmt.where(MonitorResult.checked_at > last_up)
|
|
return (await db.execute(stmt)).scalar()
|
|
|
|
|
|
async def monitor_states(db, monitors: list[Monitor]) -> dict[str, dict]:
|
|
"""{monitor_id: {latest, recent, uptime, down_since}} for the given monitors.
|
|
|
|
`recent` is the last HEARTBEAT_COUNT results oldest-first; `uptime` is
|
|
{"24h","7d","30d"} percentages; `down_since` is set only for a monitor
|
|
whose latest check failed.
|
|
"""
|
|
ids = [m.id for m in monitors]
|
|
recent = await _last_n_results(db, ids)
|
|
uptime = await _uptime_by_monitor(db, ids)
|
|
out: dict[str, dict] = {}
|
|
for m in monitors:
|
|
rows = recent.get(m.id, [])
|
|
latest = rows[-1] if rows else None
|
|
out[m.id] = {
|
|
"latest": latest,
|
|
"recent": rows,
|
|
"uptime": uptime.get(m.id, {}),
|
|
"down_since": None,
|
|
}
|
|
if latest is not None and not latest.is_up:
|
|
out[m.id]["down_since"] = await down_since(db, m.id)
|
|
return out
|
|
|
|
|
|
async def recent_failures(db, monitor_id: str, limit: int = 10) -> list[MonitorResult]:
|
|
"""The monitor's latest failed checks, newest first."""
|
|
return list((await db.execute(
|
|
select(MonitorResult)
|
|
.where(MonitorResult.monitor_id == monitor_id, MonitorResult.is_up.is_(False))
|
|
.order_by(MonitorResult.checked_at.desc())
|
|
.limit(limit)
|
|
)).scalars())
|