feat(monitors): unify ping/dns/http into one Monitor entity + custom targets
CI / lint (push) Successful in 2s
CI / unit (push) Successful in 8s
CI / integration (push) Successful in 2m17s
CI / publish (push) Successful in 1m10s

Collapse the three former check types into a single core `Monitor` entity
with one management surface (/monitors), one result table (monitor_results),
and a single scheduled task. Every type can now watch a free-standing custom
destination (optional host_id) — not just a registered Host.

- models: Monitor + MonitorResult replace PingResult/DnsResult; Host loses its
  ping/dns facet columns (now Monitor rows linked by host_id).
- checks: monitors/{ping,dns,http}.py pure probes + runner.run_monitor
  dispatcher; one monitor_check scheduler with a per-monitor due-filter.
- status: single monitor_status_source replaces the three sources.
- UI: /monitors blueprint (type-aware add/edit/list/widget); host hub shows a
  host's linked monitors + "add monitor for this host"; nav + widget registry
  + alert metric catalog rewired. http plugin folded into core and removed.
- migration 0022 merges the http branch, data-migrates host facets +
  http_monitors + all three result histories, drops the old tables/columns.

Resolves the per-host ping/dns auto-attach issue (#275): monitors are now
explicit, never auto-added to every host.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_016Jg27rgypiW2efULXJDtMC
This commit is contained in:
2026-06-18 08:56:13 -04:00
parent 591706bd39
commit 35f658b573
53 changed files with 1628 additions and 1839 deletions
+19 -47
View File
@@ -1,62 +1,34 @@
"""DNS resolution check — pure probe returning a result dict."""
from __future__ import annotations
import asyncio
import logging
import socket
from sqlalchemy import select
from sqlalchemy.ext.asyncio import AsyncSession
from steward.core.alerts import record_metric
from steward.models.hosts import Host
from steward.models.monitors import DnsResult, DnsStatus
logger = logging.getLogger(__name__)
async def dns_check(host: Host, session: AsyncSession) -> None:
"""Resolve host and write dns_result + metrics."""
result = await _resolve(host.address)
async def dns_check(address: str, expected_ip: str | None = None) -> dict:
"""Resolve `address`. Returns {is_up, resolved_ip, error_msg}.
if result["resolved"]:
resolved_ip = result["ip"] # first A/AAAA record (stored in DB)
all_ips = result["all_ips"]
Up = resolution succeeded AND (no expected_ip, or at least one returned
record matches expected_ip). resolved_ip is the first A/AAAA record.
"""
result = await _resolve(address)
if not result["resolved"]:
return {"is_up": False, "resolved_ip": None, "error_msg": "Resolution failed"}
# Check against expected IP: pass only if at least one returned record matches
if host.dns_expected_ip and host.dns_expected_ip not in all_ips:
status = DnsStatus.failed
dns_resolved_metric = 0.0
else:
status = DnsStatus.resolved
dns_resolved_metric = 1.0
# Detect IP change vs most recent successful resolution
prev_result = await session.execute(
select(DnsResult)
.where(DnsResult.host_id == host.id, DnsResult.status == DnsStatus.resolved)
.order_by(DnsResult.resolved_at.desc())
.limit(1)
)
prev = prev_result.scalar_one_or_none()
ip_changed = 1.0 if (prev and prev.resolved_ip != resolved_ip) else 0.0
else:
resolved_ip = None
status = DnsStatus.failed
dns_resolved_metric = 0.0
ip_changed = 0.0
session.add(DnsResult(
host_id=host.id,
status=status,
resolved_ip=resolved_ip,
))
await session.flush()
await record_metric(session, "dns", host.name, "resolved", dns_resolved_metric)
await record_metric(session, "dns", host.name, "ip_changed", ip_changed)
resolved_ip = result["ip"]
if expected_ip and expected_ip not in result["all_ips"]:
return {
"is_up": False,
"resolved_ip": resolved_ip,
"error_msg": f"Expected {expected_ip} not in {', '.join(result['all_ips'])}",
}
return {"is_up": True, "resolved_ip": resolved_ip, "error_msg": None}
async def _resolve(address: str) -> dict:
"""Resolve hostname. Returns all A/AAAA records; stores first as resolved_ip."""
"""Resolve hostname. Returns all A/AAAA records; first is the resolved_ip."""
try:
loop = asyncio.get_running_loop()
infos = await loop.run_in_executor(
@@ -67,5 +39,5 @@ async def _resolve(address: str) -> dict:
return {"resolved": True, "ip": all_ips[0], "all_ips": all_ips}
return {"resolved": False, "ip": None, "all_ips": []}
except Exception as exc:
logger.debug(f"DNS resolution failed for {address}: {exc}")
logger.debug("DNS resolution failed for %s: %s", address, exc)
return {"resolved": False, "ip": None, "all_ips": []}
+100
View File
@@ -0,0 +1,100 @@
"""HTTP(S) check — one synthetic request, returns a normalised result dict.
Moved into core from the former http plugin (plugins/http/checker.py) when
ping/dns/http were unified under the Monitor entity.
"""
from __future__ import annotations
import asyncio
import logging
import ssl
import time
from datetime import datetime, timezone
from urllib.parse import urlparse
import httpx
logger = logging.getLogger(__name__)
async def _get_tls_expiry(hostname: str, port: int) -> datetime | None:
"""Bare TLS handshake to read the certificate's notAfter date."""
ctx = ssl.create_default_context()
try:
reader, writer = await asyncio.wait_for(
asyncio.open_connection(hostname, port, ssl=ctx),
timeout=5.0,
)
cert = writer.get_extra_info("ssl_object").getpeercert()
writer.close()
try:
await writer.wait_closed()
except Exception:
pass
exp_str = cert.get("notAfter", "")
if not exp_str:
return None
# Format: "Mar 22 12:00:00 2027 GMT"
return datetime.strptime(exp_str, "%b %d %H:%M:%S %Y %Z").replace(
tzinfo=timezone.utc
)
except Exception as exc:
logger.debug("TLS check failed for %s:%d%s", hostname, port, exc)
return None
async def http_check(
url: str,
method: str = "GET",
expected_status: int = 200,
content_match: str = "",
headers: dict | None = None,
timeout_seconds: int = 10,
follow_redirects: bool = True,
verify_ssl: bool = True,
) -> dict:
"""Perform one HTTP check. Returns the MonitorResult field dict."""
result: dict = {
"is_up": False,
"status_code": None,
"response_ms": None,
"content_matched": None,
"error_msg": None,
"tls_expires_at": None,
}
parsed = urlparse(url)
is_https = parsed.scheme.lower() == "https"
t0 = time.monotonic()
try:
async with httpx.AsyncClient(
follow_redirects=follow_redirects,
verify=verify_ssl,
timeout=timeout_seconds,
) as client:
response = await client.request(method, url, headers=headers or {})
result["response_ms"] = round((time.monotonic() - t0) * 1000, 1)
result["status_code"] = response.status_code
status_ok = response.status_code == expected_status
if content_match:
matched = content_match in response.text
result["content_matched"] = matched
result["is_up"] = status_ok and matched
else:
result["is_up"] = status_ok
except httpx.TimeoutException:
result["response_ms"] = round((time.monotonic() - t0) * 1000, 1)
result["error_msg"] = "Timeout"
except httpx.ConnectError as exc:
result["error_msg"] = f"Connection error: {exc}"
except Exception as exc:
result["error_msg"] = str(exc)[:512]
# TLS expiry — only for HTTPS once we have any response.
if is_https and result["status_code"] is not None and parsed.hostname:
port = parsed.port or 443
result["tls_expires_at"] = await _get_tls_expiry(parsed.hostname, port)
return result
+15 -37
View File
@@ -1,43 +1,17 @@
"""ICMP / TCP reachability checks — pure probes returning a result dict."""
from __future__ import annotations
import asyncio
import logging
import time
from sqlalchemy.ext.asyncio import AsyncSession
from steward.core.alerts import record_metric
from steward.models.hosts import Host, ProbeType
from steward.models.monitors import PingResult, PingStatus
logger = logging.getLogger(__name__)
TCP_TIMEOUT = 5.0
DEFAULT_TCP_PORT = 80
async def ping_check(host: Host, session: AsyncSession) -> None:
"""Probe a single host and write ping_result + metrics."""
if host.probe_type == ProbeType.icmp:
result = await _icmp_ping(host.address)
else:
result = await _tcp_ping(host.address, host.probe_port or DEFAULT_TCP_PORT)
status = PingStatus.up if result["up"] else PingStatus.down
session.add(PingResult(
host_id=host.id,
status=status,
response_time_ms=result.get("response_time_ms"),
))
await session.flush()
await record_metric(session, "ping", host.name, "up", 1.0 if result["up"] else 0.0)
await record_metric(
session, "ping", host.name, "response_time_ms",
result.get("response_time_ms") or 0.0,
)
async def _tcp_ping(address: str, port: int) -> dict:
async def tcp_check(address: str, port: int) -> dict:
"""TCP connect probe. Returns {is_up, response_ms}."""
start = time.monotonic()
try:
reader, writer = await asyncio.wait_for(
@@ -46,13 +20,17 @@ async def _tcp_ping(address: str, port: int) -> dict:
)
writer.close()
await writer.wait_closed()
return {"up": True, "response_time_ms": round((time.monotonic() - start) * 1000, 2)}
return {"is_up": True, "response_ms": round((time.monotonic() - start) * 1000, 2)}
except Exception:
return {"up": False, "response_time_ms": None}
return {"is_up": False, "response_ms": None}
async def _icmp_ping(address: str) -> dict:
"""Use system ping command (setuid binary; no raw socket privilege needed in Python)."""
async def icmp_check(address: str) -> dict:
"""ICMP echo via the system ping binary (setuid; no raw-socket privilege).
Falls back to a TCP connect on the default port if the ping binary is
unavailable or errors, so a host with ICMP filtered still gets a signal.
"""
start = time.monotonic()
try:
proc = await asyncio.create_subprocess_exec(
@@ -62,8 +40,8 @@ async def _icmp_ping(address: str) -> dict:
)
await asyncio.wait_for(proc.wait(), timeout=5.0)
if proc.returncode == 0:
return {"up": True, "response_time_ms": round((time.monotonic() - start) * 1000, 2)}
return {"up": False, "response_time_ms": None}
return {"is_up": True, "response_ms": round((time.monotonic() - start) * 1000, 2)}
return {"is_up": False, "response_ms": None}
except Exception as exc:
logger.warning(f"ICMP ping failed for {address}: {exc}, falling back to TCP")
return await _tcp_ping(address, DEFAULT_TCP_PORT)
logger.warning("ICMP ping failed for %s: %s, falling back to TCP", address, exc)
return await tcp_check(address, DEFAULT_TCP_PORT)
+215
View File
@@ -0,0 +1,215 @@
"""Unified monitors management surface (/monitors) — ping, DNS, HTTP.
Replaces the former /ping, /dns and /plugins/http pages. One list, one
type-aware add/edit form, one dashboard widget. A monitor may be linked to a
Host (host_id) or watch a free-standing custom destination (host_id = NULL).
"""
from __future__ import annotations
import json
import uuid
from datetime import datetime, timezone
from quart import Blueprint, current_app, render_template, request, redirect, url_for
from sqlalchemy import select
from steward.auth.middleware import require_role
from steward.models.users import UserRole
from steward.models.monitors import Monitor, MonitorType, MONITOR_TYPE_VALUES
from steward.core.status import _last_n_results, _uptime_by_monitor, _heartbeat_title
from steward.core.time_range import parse_range, DEFAULT_RANGE
monitors_bp = Blueprint("monitors", __name__, url_prefix="/monitors")
def _build_config(form, mtype: str) -> dict:
"""Assemble the type-specific config dict from submitted form fields."""
if mtype == MonitorType.tcp.value:
return {"port": int(form.get("port") or 80)}
if mtype == MonitorType.dns.value:
return {"expected_ip": (form.get("expected_ip", "").strip() or None)}
if mtype == MonitorType.http.value:
try:
headers = json.loads(form.get("headers_json") or "{}")
except (ValueError, TypeError):
headers = {}
return {
"method": form.get("method", "GET").upper(),
"expected_status": int(form.get("expected_status", 200) or 200),
"content_match": form.get("content_match", "").strip(),
"headers": headers,
"timeout_seconds": int(form.get("timeout_seconds", 10) or 10),
"follow_redirects": "follow_redirects" in form,
"verify_ssl": "verify_ssl" in form,
}
return {} # icmp
def _normalise_target(mtype: str, target: str) -> str:
"""For http, default a bare host to https://; others pass through."""
target = target.strip()
if mtype == MonitorType.http.value and target and not target.startswith(("http://", "https://")):
return "https://" + target
return target
async def _rows_data(range_key: str):
"""Per-monitor display data for the list / widget."""
since, range_key = parse_range(range_key)
async with current_app.db_sessionmaker() as db:
monitors = list((await db.execute(
select(Monitor).order_by(Monitor.name)
)).scalars())
ids = [m.id for m in monitors]
recent = await _last_n_results(db, ids)
uptime = await _uptime_by_monitor(db, ids)
# Map the selected range to the uptime bucket we surface in the list.
bucket = {"24h": "24h", "7d": "7d", "30d": "30d"}.get(range_key, "24h")
now = datetime.now(timezone.utc)
data = []
for m in monitors:
rows = recent.get(m.id, [])
latest = rows[-1] if rows else None
tls_days = None
if latest and latest.tls_expires_at:
tls_days = (latest.tls_expires_at - now).total_seconds() / 86400.0
data.append({
"monitor": m,
"latest": latest,
"heartbeat": [{"state": "up" if r.is_up else "down",
"title": _heartbeat_title(m.type, r)} for r in rows],
"uptime_pct": uptime.get(m.id, {}).get(bucket),
"tls_days": tls_days,
})
up = sum(1 for d in data if d["latest"] and d["latest"].is_up)
down = sum(1 for d in data if d["latest"] and not d["latest"].is_up)
pending = sum(1 for d in data if not d["latest"])
return data, up, down, pending, range_key
@monitors_bp.get("/")
@require_role(UserRole.viewer)
async def index():
poll_interval = current_app.config.get("MONITORS_POLL_INTERVAL", 60)
return await render_template(
"monitors/index.html",
poll_interval=poll_interval,
current_range=request.args.get("range", DEFAULT_RANGE),
types=list(MonitorType),
)
@monitors_bp.get("/rows")
@require_role(UserRole.viewer)
async def rows():
data, up, down, pending, range_key = await _rows_data(
request.args.get("range", DEFAULT_RANGE)
)
return await render_template(
"monitors/rows.html",
monitor_data=data, up=up, down=down, pending=pending, range_key=range_key,
)
@monitors_bp.post("/add")
@require_role(UserRole.operator)
async def add_monitor():
form = await request.form
mtype = form.get("type", "").strip()
target = _normalise_target(mtype, form.get("target", ""))
if mtype not in MONITOR_TYPE_VALUES or not target:
return redirect(url_for("monitors.index"))
host_id = form.get("host_id", "").strip() or None
monitor = Monitor(
id=str(uuid.uuid4()),
name=form.get("name", "").strip() or target,
type=mtype,
target=target,
host_id=host_id,
config_json=json.dumps(_build_config(form, mtype)),
enabled=True,
check_interval_seconds=int(form.get("check_interval_seconds", 0) or 0),
created_at=datetime.now(timezone.utc),
)
async with current_app.db_sessionmaker() as db:
async with db.begin():
db.add(monitor)
# A host-scoped add returns to that host's hub; otherwise the monitors page.
if host_id:
return redirect(url_for("hosts.host_detail", host_id=host_id))
return redirect(url_for("monitors.index"))
@monitors_bp.get("/<monitor_id>/edit")
@require_role(UserRole.operator)
async def edit_form(monitor_id: str):
async with current_app.db_sessionmaker() as db:
m = await db.get(Monitor, monitor_id)
if not m:
return redirect(url_for("monitors.index"))
return await render_template(
"monitors/edit.html", monitor=m, config=m.config, types=list(MonitorType),
)
@monitors_bp.post("/<monitor_id>/edit")
@require_role(UserRole.operator)
async def edit_monitor(monitor_id: str):
form = await request.form
async with current_app.db_sessionmaker() as db:
async with db.begin():
m = await db.get(Monitor, monitor_id)
if not m:
return redirect(url_for("monitors.index"))
# Type is immutable on edit (changes the meaning of config/target);
# to switch type, delete and re-add.
m.name = form.get("name", "").strip() or m.target
m.target = _normalise_target(m.type, form.get("target", m.target))
m.config_json = json.dumps(_build_config(form, m.type))
m.check_interval_seconds = int(form.get("check_interval_seconds", 0) or 0)
return redirect(url_for("monitors.index"))
@monitors_bp.post("/<monitor_id>/toggle")
@require_role(UserRole.operator)
async def toggle_monitor(monitor_id: str):
async with current_app.db_sessionmaker() as db:
async with db.begin():
m = await db.get(Monitor, monitor_id)
if m:
m.enabled = not m.enabled
return redirect(request.referrer or url_for("monitors.index"))
@monitors_bp.post("/<monitor_id>/delete")
@require_role(UserRole.operator)
async def delete_monitor(monitor_id: str):
async with current_app.db_sessionmaker() as db:
async with db.begin():
m = await db.get(Monitor, monitor_id)
if m:
await db.delete(m)
return redirect(request.referrer or url_for("monitors.index"))
@monitors_bp.get("/widget")
@require_role(UserRole.viewer)
async def widget():
type_filter = request.args.get("type_filter", "all")
show_down_only = request.args.get("show_down_only", "no") == "yes"
limit = max(1, min(50, int(request.args.get("limit", 10) or 10)))
widget_id = request.args.get("wid", "0")
data, up, down, pending, _ = await _rows_data(DEFAULT_RANGE)
if type_filter in MONITOR_TYPE_VALUES:
data = [d for d in data if d["monitor"].type == type_filter]
if show_down_only:
data = [d for d in data if not (d["latest"] and d["latest"].is_up)]
return await render_template(
"monitors/widget.html",
monitor_data=data[:limit], up=up, down=down, pending=pending,
show_down_only=show_down_only, widget_id=widget_id,
)
+56
View File
@@ -0,0 +1,56 @@
"""Unified check dispatcher: Monitor -> normalised MonitorResult fields.
One entry point, `run_monitor`, dispatches on Monitor.type to the per-type
probes (ping/dns/http) and returns a dict with every MonitorResult field so the
scheduler can persist a row uniformly regardless of type.
"""
from __future__ import annotations
import logging
from steward.models.monitors import Monitor, MonitorType
from .ping import tcp_check, icmp_check, DEFAULT_TCP_PORT
from .dns import dns_check
from .http import http_check
logger = logging.getLogger(__name__)
# Every result carries these keys; type-specific probes fill the relevant ones.
_BLANK = {
"is_up": False,
"response_ms": None,
"status_code": None,
"resolved_ip": None,
"content_matched": None,
"tls_expires_at": None,
"error_msg": None,
}
async def run_monitor(monitor: Monitor) -> dict:
"""Run one monitor's check and return a full MonitorResult field dict."""
cfg = monitor.config
result = dict(_BLANK)
try:
if monitor.type == MonitorType.icmp.value:
result.update(await icmp_check(monitor.target))
elif monitor.type == MonitorType.tcp.value:
result.update(await tcp_check(monitor.target, int(cfg.get("port") or DEFAULT_TCP_PORT)))
elif monitor.type == MonitorType.dns.value:
result.update(await dns_check(monitor.target, cfg.get("expected_ip") or None))
elif monitor.type == MonitorType.http.value:
result.update(await http_check(
url=monitor.target,
method=cfg.get("method", "GET"),
expected_status=int(cfg.get("expected_status", 200) or 200),
content_match=cfg.get("content_match", "") or "",
headers=cfg.get("headers") or {},
timeout_seconds=int(cfg.get("timeout_seconds", 10) or 10),
follow_redirects=bool(cfg.get("follow_redirects", True)),
verify_ssl=bool(cfg.get("verify_ssl", True)),
))
else:
result["error_msg"] = f"Unknown monitor type {monitor.type!r}"
except Exception as exc: # a probe should never take the scheduler down
logger.exception("Monitor %r (%s) check raised", monitor.name, monitor.type)
result["error_msg"] = str(exc)[:512]
return result
+76
View File
@@ -0,0 +1,76 @@
"""Single scheduled task that runs every due Monitor of any type."""
from __future__ import annotations
import asyncio
import logging
from datetime import datetime, timezone
from sqlalchemy import select
from steward.core.alerts import record_metric
from steward.models.monitors import Monitor, MonitorResult
from .runner import run_monitor
logger = logging.getLogger(__name__)
async def run_due_monitors(app) -> None:
"""Check every enabled Monitor whose interval has elapsed, persist results.
Per-monitor `check_interval_seconds` overrides the global
MONITORS_POLL_INTERVAL (0 = use global), mirroring how the scheduler fires
on the global cadence but each monitor decides if it is actually due.
"""
global_interval = int(app.config.get("MONITORS_POLL_INTERVAL", 60))
now = datetime.now(timezone.utc)
async with app.db_sessionmaker() as session:
monitors = list((await session.execute(
select(Monitor).where(Monitor.enabled.is_(True))
)).scalars())
due: list[Monitor] = []
for m in monitors:
interval = m.check_interval_seconds or global_interval
if m.last_checked_at is None or (now - m.last_checked_at).total_seconds() >= interval:
due.append(m)
if not due:
return
pairs = await asyncio.gather(
*[run_monitor(m) for m in due], return_exceptions=True
)
check_time = datetime.now(timezone.utc)
async with app.db_sessionmaker() as session:
async with session.begin():
for monitor, res in zip(due, pairs):
if isinstance(res, Exception):
logger.error("Monitor %r check errored: %s", monitor.name, res)
continue
session.add(MonitorResult(
monitor_id=monitor.id,
checked_at=check_time,
is_up=res["is_up"],
response_ms=res["response_ms"],
status_code=res["status_code"],
resolved_ip=res["resolved_ip"],
content_matched=res["content_matched"],
tls_expires_at=res["tls_expires_at"],
error_msg=res["error_msg"],
))
db_monitor = await session.get(Monitor, monitor.id)
if db_monitor:
db_monitor.last_checked_at = check_time
# Alert pipeline — keyed by monitor type (source) + name.
await record_metric(
session=session, source_module=monitor.type,
resource_name=monitor.name, metric_name="is_up",
value=1.0 if res["is_up"] else 0.0,
)
if res["response_ms"] is not None:
await record_metric(
session=session, source_module=monitor.type,
resource_name=monitor.name, metric_name="response_ms",
value=res["response_ms"],
)