Files
FabledSteward/steward/monitors/service.py
T
bvandeusenandClaude Opus 5.5 c9f5c049db
CI / lint (push) Successful in 3s
CI / integration (push) Successful in 50s
CI / unit (push) Successful in 1m6s
CI / publish (push) Successful in 1m2s
feat(alerts): renames carry their alerts — monitor and host renames move rules, windows and history (#5065)
#5065 "Renames carry their alerts". Alert rules, maintenance-window scopes
and metric history key a thing by (source, name), so a rename used to leave
the rules watching a name nothing reported under, and split the history.

- core/renames.carry_rename moves AlertRule.resource_name,
  MaintenanceWindow.scope_resource, plugin_metrics and plugin_metrics_hourly
  to the new name in the caller's transaction. A host rename also carries
  its parts: host_agent "<host>:…" and docker "<host>/…" (HOST_SOURCES).
  Matching is exact (a prefix compare, not LIKE or a collation range).
  History left under the new name by something deleted is dropped first
  rather than merged in.
- Web monitor edit, MCP update_monitor and web host edit call it; MCP
  answers with `carried` (it used to return a warning naming stranded rules).
- A name another monitor of the same type, or another host, already has is
  refused on create and rename: two would share one history.
- The rename is audited (monitor.updated / host.renamed) with what moved.

Tests: an integration test renames a firing monitor through the web route
and finds its rule still firing and cleared by a sample on the new name,
its window and raw+hourly history moved and an orphan's history dropped; a
host rename moves agent and container parts but not a longer-named host's
or another source's; a guard that every host_agent resource is "<host>" or
"<host>:…".

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-06 14:02:43 -04:00

217 lines
9.1 KiB
Python

"""Monitor queries and validation shared by the monitors pages and the MCP tools.
Each function takes an open session (or plain values), with no Quart request
context, so the routes and the /mcp tools (which run outside Quart's
pipeline) call the same code and refuse the same input for the same reasons.
"""
from __future__ import annotations
import json
from collections.abc import Mapping
from datetime import datetime, timezone
from sqlalchemy import func, select
from steward.core.status import _heartbeat_title, _last_n_results, _uptime_by_monitor
from steward.core.time_range import parse_range
from steward.models.monitors import MONITOR_TYPE_VALUES, Monitor, MonitorResult, MonitorType
HTTP_METHODS = ("GET", "HEAD", "POST")
# The config keys each type takes (models/monitors.py documents their meaning).
CONFIG_KEYS: dict[str, tuple[str, ...]] = {
MonitorType.icmp.value: (),
MonitorType.tcp.value: ("port",),
MonitorType.dns.value: ("expected_ip",),
MonitorType.http.value: ("method", "expected_status", "content_match", "headers",
"timeout_seconds", "follow_redirects", "verify_ssl"),
}
def _whole(name: str, value, default: int, lo: int, hi: int | None = None) -> int:
if value is None or value == "":
return default
if isinstance(value, bool):
raise ValueError(f"{name} must be a whole number, got {value!r}.")
try:
n = int(value)
except (TypeError, ValueError):
raise ValueError(f"{name} must be a whole number, got {value!r}.") from None
if n < lo or (hi is not None and n > hi):
bounds = f"between {lo} and {hi}" if hi is not None else f"at least {lo}"
raise ValueError(f"{name} must be {bounds}, got {n}.")
return n
def _headers(value) -> dict:
if value is None or value == "":
return {}
if isinstance(value, str):
try:
value = json.loads(value)
except ValueError:
raise ValueError("headers must be a JSON object of name: value strings.") from None
if not isinstance(value, dict) or not all(isinstance(k, str) and isinstance(v, str)
for k, v in value.items()):
raise ValueError("headers must be an object of name: value strings.")
return value
def normalise_target(mtype: str, target: str) -> str:
"""For http, default a bare host to https://; others pass through."""
target = (target or "").strip()
if mtype == MonitorType.http.value and target and not target.startswith(("http://", "https://")):
return "https://" + target
return target
def build_config(mtype: str, opts: Mapping) -> dict:
"""The type-specific config from already-typed or form-string values.
Missing keys take their defaults; a key the type does not take, or a
value out of range, raises ValueError saying which and why.
"""
unknown = sorted(set(opts) - set(CONFIG_KEYS[mtype]))
if unknown:
allowed = ", ".join(CONFIG_KEYS[mtype]) or "no config"
raise ValueError(f"a {mtype} monitor does not take {', '.join(unknown)}; it takes {allowed}.")
if mtype == MonitorType.tcp.value:
return {"port": _whole("port", opts.get("port"), 80, 1, 65535)}
if mtype == MonitorType.dns.value:
return {"expected_ip": (str(opts.get("expected_ip") or "").strip() or None)}
if mtype == MonitorType.http.value:
method = str(opts.get("method") or "GET").strip().upper()
if method not in HTTP_METHODS:
raise ValueError(f"method {method!r} is not one of {', '.join(HTTP_METHODS)}.")
return {
"method": method,
"expected_status": _whole("expected_status", opts.get("expected_status"), 200, 100, 599),
"content_match": str(opts.get("content_match") or "").strip(),
"headers": _headers(opts.get("headers")),
"timeout_seconds": _whole("timeout_seconds", opts.get("timeout_seconds"), 10, 1, 60),
"follow_redirects": bool(opts.get("follow_redirects", True)),
"verify_ssl": bool(opts.get("verify_ssl", True)),
}
return {} # icmp
def monitor_fields(mtype: str, *, target: str, name: str = "", check_interval_seconds=0,
opts: Mapping | None = None) -> dict:
"""Validated column values for a new or edited monitor; ValueError says why not."""
mtype = (mtype or "").strip().lower()
if mtype not in MONITOR_TYPE_VALUES:
raise ValueError(f"type {mtype!r} is not one of {', '.join(MONITOR_TYPE_VALUES)}.")
target = normalise_target(mtype, target)
if not target:
raise ValueError("a monitor needs a target: an address, or a URL for http.")
if len(target) > 2048:
raise ValueError("target is longer than 2048 characters.")
return {
"type": mtype,
"target": target,
"name": ((name or "").strip() or target)[:128],
"config_json": json.dumps(build_config(mtype, opts or {})),
"check_interval_seconds": _whole("check_interval_seconds", check_interval_seconds, 0, 0),
}
async def check_name_free(db, mtype: str, name: str, exclude_id: str | None = None) -> None:
"""ValueError when another monitor of this type already has this name.
A monitor's checks are recorded, and its alert rules watch it, by
(type, name), so two monitors sharing both would share one history.
"""
q = select(Monitor.id).where(Monitor.type == mtype, Monitor.name == name)
if exclude_id:
q = q.where(Monitor.id != exclude_id)
if (await db.execute(q.limit(1))).first():
raise ValueError(f"another {mtype} monitor is already named {name!r}; its checks and "
"alert rules go by that name, so pick a different one.")
async def monitor_rows(db, range_key: str):
"""Per-monitor state: latest result, heartbeat strip, uptime and TLS days.
Returns (rows, up, down, pending, range_key). `range_key` picks the uptime
window surfaced (24h/7d/30d; anything else falls back to 24h) and comes
back normalised.
"""
_since, range_key = parse_range(range_key)
monitors = list((await db.execute(select(Monitor).order_by(Monitor.name))).scalars())
ids = [m.id for m in monitors]
recent = await _last_n_results(db, ids)
uptime = await _uptime_by_monitor(db, ids)
# Map the selected range to the uptime bucket we surface in the list.
bucket = {"24h": "24h", "7d": "7d", "30d": "30d"}.get(range_key, "24h")
now = datetime.now(timezone.utc)
data = []
for m in monitors:
rows = recent.get(m.id, [])
latest = rows[-1] if rows else None
tls_days = None
if latest and latest.tls_expires_at:
tls_days = (latest.tls_expires_at - now).total_seconds() / 86400.0
data.append({
"monitor": m,
"latest": latest,
"heartbeat": [{"state": "up" if r.is_up else "down",
"title": _heartbeat_title(m.type, r)} for r in rows],
"uptime_pct": uptime.get(m.id, {}).get(bucket),
"tls_days": tls_days,
})
up = sum(1 for d in data if d["latest"] and d["latest"].is_up)
down = sum(1 for d in data if d["latest"] and not d["latest"].is_up)
pending = sum(1 for d in data if not d["latest"])
return data, up, down, pending, range_key
async def down_since(db, monitor_id: str):
"""When the monitor's current run of failures began, or None if its latest check is up.
The first failed check after the last success; if it has never succeeded,
its first check.
"""
last_up = (await db.execute(
select(func.max(MonitorResult.checked_at))
.where(MonitorResult.monitor_id == monitor_id, MonitorResult.is_up.is_(True))
)).scalar()
stmt = select(func.min(MonitorResult.checked_at)).where(MonitorResult.monitor_id == monitor_id)
if last_up is not None:
stmt = stmt.where(MonitorResult.checked_at > last_up)
return (await db.execute(stmt)).scalar()
async def monitor_states(db, monitors: list[Monitor]) -> dict[str, dict]:
"""{monitor_id: {latest, recent, uptime, down_since}} for the given monitors.
`recent` is the last HEARTBEAT_COUNT results oldest-first; `uptime` is
{"24h","7d","30d"} percentages; `down_since` is set only for a monitor
whose latest check failed.
"""
ids = [m.id for m in monitors]
recent = await _last_n_results(db, ids)
uptime = await _uptime_by_monitor(db, ids)
out: dict[str, dict] = {}
for m in monitors:
rows = recent.get(m.id, [])
latest = rows[-1] if rows else None
out[m.id] = {
"latest": latest,
"recent": rows,
"uptime": uptime.get(m.id, {}),
"down_since": None,
}
if latest is not None and not latest.is_up:
out[m.id]["down_since"] = await down_since(db, m.id)
return out
async def recent_failures(db, monitor_id: str, limit: int = 10) -> list[MonitorResult]:
"""The monitor's latest failed checks, newest first."""
return list((await db.execute(
select(MonitorResult)
.where(MonitorResult.monitor_id == monitor_id, MonitorResult.is_up.is_(False))
.order_by(MonitorResult.checked_at.desc())
.limit(limit)
)).scalars())