feat(monitors): unify ping/dns/http into one Monitor entity + custom targets
Collapse the three former check types into a single core `Monitor` entity
with one management surface (/monitors), one result table (monitor_results),
and a single scheduled task. Every type can now watch a free-standing custom
destination (optional host_id) — not just a registered Host.
- models: Monitor + MonitorResult replace PingResult/DnsResult; Host loses its
ping/dns facet columns (now Monitor rows linked by host_id).
- checks: monitors/{ping,dns,http}.py pure probes + runner.run_monitor
dispatcher; one monitor_check scheduler with a per-monitor due-filter.
- status: single monitor_status_source replaces the three sources.
- UI: /monitors blueprint (type-aware add/edit/list/widget); host hub shows a
host's linked monitors + "add monitor for this host"; nav + widget registry
+ alert metric catalog rewired. http plugin folded into core and removed.
- migration 0022 merges the http branch, data-migrates host facets +
http_monitors + all three result histories, drops the old tables/columns.
Resolves the per-host ping/dns auto-attach issue (#275): monitors are now
explicit, never auto-added to every host.
Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_016Jg27rgypiW2efULXJDtMC
This commit is contained in:
+19
-47
@@ -1,62 +1,34 @@
|
||||
"""DNS resolution check — pure probe returning a result dict."""
|
||||
from __future__ import annotations
|
||||
import asyncio
|
||||
import logging
|
||||
import socket
|
||||
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from steward.core.alerts import record_metric
|
||||
from steward.models.hosts import Host
|
||||
from steward.models.monitors import DnsResult, DnsStatus
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
async def dns_check(host: Host, session: AsyncSession) -> None:
|
||||
"""Resolve host and write dns_result + metrics."""
|
||||
result = await _resolve(host.address)
|
||||
async def dns_check(address: str, expected_ip: str | None = None) -> dict:
|
||||
"""Resolve `address`. Returns {is_up, resolved_ip, error_msg}.
|
||||
|
||||
if result["resolved"]:
|
||||
resolved_ip = result["ip"] # first A/AAAA record (stored in DB)
|
||||
all_ips = result["all_ips"]
|
||||
Up = resolution succeeded AND (no expected_ip, or at least one returned
|
||||
record matches expected_ip). resolved_ip is the first A/AAAA record.
|
||||
"""
|
||||
result = await _resolve(address)
|
||||
if not result["resolved"]:
|
||||
return {"is_up": False, "resolved_ip": None, "error_msg": "Resolution failed"}
|
||||
|
||||
# Check against expected IP: pass only if at least one returned record matches
|
||||
if host.dns_expected_ip and host.dns_expected_ip not in all_ips:
|
||||
status = DnsStatus.failed
|
||||
dns_resolved_metric = 0.0
|
||||
else:
|
||||
status = DnsStatus.resolved
|
||||
dns_resolved_metric = 1.0
|
||||
|
||||
# Detect IP change vs most recent successful resolution
|
||||
prev_result = await session.execute(
|
||||
select(DnsResult)
|
||||
.where(DnsResult.host_id == host.id, DnsResult.status == DnsStatus.resolved)
|
||||
.order_by(DnsResult.resolved_at.desc())
|
||||
.limit(1)
|
||||
)
|
||||
prev = prev_result.scalar_one_or_none()
|
||||
ip_changed = 1.0 if (prev and prev.resolved_ip != resolved_ip) else 0.0
|
||||
else:
|
||||
resolved_ip = None
|
||||
status = DnsStatus.failed
|
||||
dns_resolved_metric = 0.0
|
||||
ip_changed = 0.0
|
||||
|
||||
session.add(DnsResult(
|
||||
host_id=host.id,
|
||||
status=status,
|
||||
resolved_ip=resolved_ip,
|
||||
))
|
||||
await session.flush()
|
||||
|
||||
await record_metric(session, "dns", host.name, "resolved", dns_resolved_metric)
|
||||
await record_metric(session, "dns", host.name, "ip_changed", ip_changed)
|
||||
resolved_ip = result["ip"]
|
||||
if expected_ip and expected_ip not in result["all_ips"]:
|
||||
return {
|
||||
"is_up": False,
|
||||
"resolved_ip": resolved_ip,
|
||||
"error_msg": f"Expected {expected_ip} not in {', '.join(result['all_ips'])}",
|
||||
}
|
||||
return {"is_up": True, "resolved_ip": resolved_ip, "error_msg": None}
|
||||
|
||||
|
||||
async def _resolve(address: str) -> dict:
|
||||
"""Resolve hostname. Returns all A/AAAA records; stores first as resolved_ip."""
|
||||
"""Resolve hostname. Returns all A/AAAA records; first is the resolved_ip."""
|
||||
try:
|
||||
loop = asyncio.get_running_loop()
|
||||
infos = await loop.run_in_executor(
|
||||
@@ -67,5 +39,5 @@ async def _resolve(address: str) -> dict:
|
||||
return {"resolved": True, "ip": all_ips[0], "all_ips": all_ips}
|
||||
return {"resolved": False, "ip": None, "all_ips": []}
|
||||
except Exception as exc:
|
||||
logger.debug(f"DNS resolution failed for {address}: {exc}")
|
||||
logger.debug("DNS resolution failed for %s: %s", address, exc)
|
||||
return {"resolved": False, "ip": None, "all_ips": []}
|
||||
|
||||
@@ -0,0 +1,100 @@
|
||||
"""HTTP(S) check — one synthetic request, returns a normalised result dict.
|
||||
|
||||
Moved into core from the former http plugin (plugins/http/checker.py) when
|
||||
ping/dns/http were unified under the Monitor entity.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
import asyncio
|
||||
import logging
|
||||
import ssl
|
||||
import time
|
||||
from datetime import datetime, timezone
|
||||
from urllib.parse import urlparse
|
||||
|
||||
import httpx
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
async def _get_tls_expiry(hostname: str, port: int) -> datetime | None:
|
||||
"""Bare TLS handshake to read the certificate's notAfter date."""
|
||||
ctx = ssl.create_default_context()
|
||||
try:
|
||||
reader, writer = await asyncio.wait_for(
|
||||
asyncio.open_connection(hostname, port, ssl=ctx),
|
||||
timeout=5.0,
|
||||
)
|
||||
cert = writer.get_extra_info("ssl_object").getpeercert()
|
||||
writer.close()
|
||||
try:
|
||||
await writer.wait_closed()
|
||||
except Exception:
|
||||
pass
|
||||
exp_str = cert.get("notAfter", "")
|
||||
if not exp_str:
|
||||
return None
|
||||
# Format: "Mar 22 12:00:00 2027 GMT"
|
||||
return datetime.strptime(exp_str, "%b %d %H:%M:%S %Y %Z").replace(
|
||||
tzinfo=timezone.utc
|
||||
)
|
||||
except Exception as exc:
|
||||
logger.debug("TLS check failed for %s:%d — %s", hostname, port, exc)
|
||||
return None
|
||||
|
||||
|
||||
async def http_check(
|
||||
url: str,
|
||||
method: str = "GET",
|
||||
expected_status: int = 200,
|
||||
content_match: str = "",
|
||||
headers: dict | None = None,
|
||||
timeout_seconds: int = 10,
|
||||
follow_redirects: bool = True,
|
||||
verify_ssl: bool = True,
|
||||
) -> dict:
|
||||
"""Perform one HTTP check. Returns the MonitorResult field dict."""
|
||||
result: dict = {
|
||||
"is_up": False,
|
||||
"status_code": None,
|
||||
"response_ms": None,
|
||||
"content_matched": None,
|
||||
"error_msg": None,
|
||||
"tls_expires_at": None,
|
||||
}
|
||||
|
||||
parsed = urlparse(url)
|
||||
is_https = parsed.scheme.lower() == "https"
|
||||
|
||||
t0 = time.monotonic()
|
||||
try:
|
||||
async with httpx.AsyncClient(
|
||||
follow_redirects=follow_redirects,
|
||||
verify=verify_ssl,
|
||||
timeout=timeout_seconds,
|
||||
) as client:
|
||||
response = await client.request(method, url, headers=headers or {})
|
||||
result["response_ms"] = round((time.monotonic() - t0) * 1000, 1)
|
||||
result["status_code"] = response.status_code
|
||||
|
||||
status_ok = response.status_code == expected_status
|
||||
if content_match:
|
||||
matched = content_match in response.text
|
||||
result["content_matched"] = matched
|
||||
result["is_up"] = status_ok and matched
|
||||
else:
|
||||
result["is_up"] = status_ok
|
||||
|
||||
except httpx.TimeoutException:
|
||||
result["response_ms"] = round((time.monotonic() - t0) * 1000, 1)
|
||||
result["error_msg"] = "Timeout"
|
||||
except httpx.ConnectError as exc:
|
||||
result["error_msg"] = f"Connection error: {exc}"
|
||||
except Exception as exc:
|
||||
result["error_msg"] = str(exc)[:512]
|
||||
|
||||
# TLS expiry — only for HTTPS once we have any response.
|
||||
if is_https and result["status_code"] is not None and parsed.hostname:
|
||||
port = parsed.port or 443
|
||||
result["tls_expires_at"] = await _get_tls_expiry(parsed.hostname, port)
|
||||
|
||||
return result
|
||||
+15
-37
@@ -1,43 +1,17 @@
|
||||
"""ICMP / TCP reachability checks — pure probes returning a result dict."""
|
||||
from __future__ import annotations
|
||||
import asyncio
|
||||
import logging
|
||||
import time
|
||||
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from steward.core.alerts import record_metric
|
||||
from steward.models.hosts import Host, ProbeType
|
||||
from steward.models.monitors import PingResult, PingStatus
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
TCP_TIMEOUT = 5.0
|
||||
DEFAULT_TCP_PORT = 80
|
||||
|
||||
|
||||
async def ping_check(host: Host, session: AsyncSession) -> None:
|
||||
"""Probe a single host and write ping_result + metrics."""
|
||||
if host.probe_type == ProbeType.icmp:
|
||||
result = await _icmp_ping(host.address)
|
||||
else:
|
||||
result = await _tcp_ping(host.address, host.probe_port or DEFAULT_TCP_PORT)
|
||||
|
||||
status = PingStatus.up if result["up"] else PingStatus.down
|
||||
session.add(PingResult(
|
||||
host_id=host.id,
|
||||
status=status,
|
||||
response_time_ms=result.get("response_time_ms"),
|
||||
))
|
||||
await session.flush()
|
||||
|
||||
await record_metric(session, "ping", host.name, "up", 1.0 if result["up"] else 0.0)
|
||||
await record_metric(
|
||||
session, "ping", host.name, "response_time_ms",
|
||||
result.get("response_time_ms") or 0.0,
|
||||
)
|
||||
|
||||
|
||||
async def _tcp_ping(address: str, port: int) -> dict:
|
||||
async def tcp_check(address: str, port: int) -> dict:
|
||||
"""TCP connect probe. Returns {is_up, response_ms}."""
|
||||
start = time.monotonic()
|
||||
try:
|
||||
reader, writer = await asyncio.wait_for(
|
||||
@@ -46,13 +20,17 @@ async def _tcp_ping(address: str, port: int) -> dict:
|
||||
)
|
||||
writer.close()
|
||||
await writer.wait_closed()
|
||||
return {"up": True, "response_time_ms": round((time.monotonic() - start) * 1000, 2)}
|
||||
return {"is_up": True, "response_ms": round((time.monotonic() - start) * 1000, 2)}
|
||||
except Exception:
|
||||
return {"up": False, "response_time_ms": None}
|
||||
return {"is_up": False, "response_ms": None}
|
||||
|
||||
|
||||
async def _icmp_ping(address: str) -> dict:
|
||||
"""Use system ping command (setuid binary; no raw socket privilege needed in Python)."""
|
||||
async def icmp_check(address: str) -> dict:
|
||||
"""ICMP echo via the system ping binary (setuid; no raw-socket privilege).
|
||||
|
||||
Falls back to a TCP connect on the default port if the ping binary is
|
||||
unavailable or errors, so a host with ICMP filtered still gets a signal.
|
||||
"""
|
||||
start = time.monotonic()
|
||||
try:
|
||||
proc = await asyncio.create_subprocess_exec(
|
||||
@@ -62,8 +40,8 @@ async def _icmp_ping(address: str) -> dict:
|
||||
)
|
||||
await asyncio.wait_for(proc.wait(), timeout=5.0)
|
||||
if proc.returncode == 0:
|
||||
return {"up": True, "response_time_ms": round((time.monotonic() - start) * 1000, 2)}
|
||||
return {"up": False, "response_time_ms": None}
|
||||
return {"is_up": True, "response_ms": round((time.monotonic() - start) * 1000, 2)}
|
||||
return {"is_up": False, "response_ms": None}
|
||||
except Exception as exc:
|
||||
logger.warning(f"ICMP ping failed for {address}: {exc}, falling back to TCP")
|
||||
return await _tcp_ping(address, DEFAULT_TCP_PORT)
|
||||
logger.warning("ICMP ping failed for %s: %s, falling back to TCP", address, exc)
|
||||
return await tcp_check(address, DEFAULT_TCP_PORT)
|
||||
|
||||
@@ -0,0 +1,215 @@
|
||||
"""Unified monitors management surface (/monitors) — ping, DNS, HTTP.
|
||||
|
||||
Replaces the former /ping, /dns and /plugins/http pages. One list, one
|
||||
type-aware add/edit form, one dashboard widget. A monitor may be linked to a
|
||||
Host (host_id) or watch a free-standing custom destination (host_id = NULL).
|
||||
"""
|
||||
from __future__ import annotations
|
||||
import json
|
||||
import uuid
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from quart import Blueprint, current_app, render_template, request, redirect, url_for
|
||||
from sqlalchemy import select
|
||||
|
||||
from steward.auth.middleware import require_role
|
||||
from steward.models.users import UserRole
|
||||
from steward.models.monitors import Monitor, MonitorType, MONITOR_TYPE_VALUES
|
||||
from steward.core.status import _last_n_results, _uptime_by_monitor, _heartbeat_title
|
||||
from steward.core.time_range import parse_range, DEFAULT_RANGE
|
||||
|
||||
monitors_bp = Blueprint("monitors", __name__, url_prefix="/monitors")
|
||||
|
||||
|
||||
def _build_config(form, mtype: str) -> dict:
|
||||
"""Assemble the type-specific config dict from submitted form fields."""
|
||||
if mtype == MonitorType.tcp.value:
|
||||
return {"port": int(form.get("port") or 80)}
|
||||
if mtype == MonitorType.dns.value:
|
||||
return {"expected_ip": (form.get("expected_ip", "").strip() or None)}
|
||||
if mtype == MonitorType.http.value:
|
||||
try:
|
||||
headers = json.loads(form.get("headers_json") or "{}")
|
||||
except (ValueError, TypeError):
|
||||
headers = {}
|
||||
return {
|
||||
"method": form.get("method", "GET").upper(),
|
||||
"expected_status": int(form.get("expected_status", 200) or 200),
|
||||
"content_match": form.get("content_match", "").strip(),
|
||||
"headers": headers,
|
||||
"timeout_seconds": int(form.get("timeout_seconds", 10) or 10),
|
||||
"follow_redirects": "follow_redirects" in form,
|
||||
"verify_ssl": "verify_ssl" in form,
|
||||
}
|
||||
return {} # icmp
|
||||
|
||||
|
||||
def _normalise_target(mtype: str, target: str) -> str:
|
||||
"""For http, default a bare host to https://; others pass through."""
|
||||
target = target.strip()
|
||||
if mtype == MonitorType.http.value and target and not target.startswith(("http://", "https://")):
|
||||
return "https://" + target
|
||||
return target
|
||||
|
||||
|
||||
async def _rows_data(range_key: str):
|
||||
"""Per-monitor display data for the list / widget."""
|
||||
since, range_key = parse_range(range_key)
|
||||
async with current_app.db_sessionmaker() as db:
|
||||
monitors = list((await db.execute(
|
||||
select(Monitor).order_by(Monitor.name)
|
||||
)).scalars())
|
||||
ids = [m.id for m in monitors]
|
||||
recent = await _last_n_results(db, ids)
|
||||
uptime = await _uptime_by_monitor(db, ids)
|
||||
|
||||
# Map the selected range to the uptime bucket we surface in the list.
|
||||
bucket = {"24h": "24h", "7d": "7d", "30d": "30d"}.get(range_key, "24h")
|
||||
now = datetime.now(timezone.utc)
|
||||
data = []
|
||||
for m in monitors:
|
||||
rows = recent.get(m.id, [])
|
||||
latest = rows[-1] if rows else None
|
||||
tls_days = None
|
||||
if latest and latest.tls_expires_at:
|
||||
tls_days = (latest.tls_expires_at - now).total_seconds() / 86400.0
|
||||
data.append({
|
||||
"monitor": m,
|
||||
"latest": latest,
|
||||
"heartbeat": [{"state": "up" if r.is_up else "down",
|
||||
"title": _heartbeat_title(m.type, r)} for r in rows],
|
||||
"uptime_pct": uptime.get(m.id, {}).get(bucket),
|
||||
"tls_days": tls_days,
|
||||
})
|
||||
up = sum(1 for d in data if d["latest"] and d["latest"].is_up)
|
||||
down = sum(1 for d in data if d["latest"] and not d["latest"].is_up)
|
||||
pending = sum(1 for d in data if not d["latest"])
|
||||
return data, up, down, pending, range_key
|
||||
|
||||
|
||||
@monitors_bp.get("/")
|
||||
@require_role(UserRole.viewer)
|
||||
async def index():
|
||||
poll_interval = current_app.config.get("MONITORS_POLL_INTERVAL", 60)
|
||||
return await render_template(
|
||||
"monitors/index.html",
|
||||
poll_interval=poll_interval,
|
||||
current_range=request.args.get("range", DEFAULT_RANGE),
|
||||
types=list(MonitorType),
|
||||
)
|
||||
|
||||
|
||||
@monitors_bp.get("/rows")
|
||||
@require_role(UserRole.viewer)
|
||||
async def rows():
|
||||
data, up, down, pending, range_key = await _rows_data(
|
||||
request.args.get("range", DEFAULT_RANGE)
|
||||
)
|
||||
return await render_template(
|
||||
"monitors/rows.html",
|
||||
monitor_data=data, up=up, down=down, pending=pending, range_key=range_key,
|
||||
)
|
||||
|
||||
|
||||
@monitors_bp.post("/add")
|
||||
@require_role(UserRole.operator)
|
||||
async def add_monitor():
|
||||
form = await request.form
|
||||
mtype = form.get("type", "").strip()
|
||||
target = _normalise_target(mtype, form.get("target", ""))
|
||||
if mtype not in MONITOR_TYPE_VALUES or not target:
|
||||
return redirect(url_for("monitors.index"))
|
||||
|
||||
host_id = form.get("host_id", "").strip() or None
|
||||
monitor = Monitor(
|
||||
id=str(uuid.uuid4()),
|
||||
name=form.get("name", "").strip() or target,
|
||||
type=mtype,
|
||||
target=target,
|
||||
host_id=host_id,
|
||||
config_json=json.dumps(_build_config(form, mtype)),
|
||||
enabled=True,
|
||||
check_interval_seconds=int(form.get("check_interval_seconds", 0) or 0),
|
||||
created_at=datetime.now(timezone.utc),
|
||||
)
|
||||
async with current_app.db_sessionmaker() as db:
|
||||
async with db.begin():
|
||||
db.add(monitor)
|
||||
|
||||
# A host-scoped add returns to that host's hub; otherwise the monitors page.
|
||||
if host_id:
|
||||
return redirect(url_for("hosts.host_detail", host_id=host_id))
|
||||
return redirect(url_for("monitors.index"))
|
||||
|
||||
|
||||
@monitors_bp.get("/<monitor_id>/edit")
|
||||
@require_role(UserRole.operator)
|
||||
async def edit_form(monitor_id: str):
|
||||
async with current_app.db_sessionmaker() as db:
|
||||
m = await db.get(Monitor, monitor_id)
|
||||
if not m:
|
||||
return redirect(url_for("monitors.index"))
|
||||
return await render_template(
|
||||
"monitors/edit.html", monitor=m, config=m.config, types=list(MonitorType),
|
||||
)
|
||||
|
||||
|
||||
@monitors_bp.post("/<monitor_id>/edit")
|
||||
@require_role(UserRole.operator)
|
||||
async def edit_monitor(monitor_id: str):
|
||||
form = await request.form
|
||||
async with current_app.db_sessionmaker() as db:
|
||||
async with db.begin():
|
||||
m = await db.get(Monitor, monitor_id)
|
||||
if not m:
|
||||
return redirect(url_for("monitors.index"))
|
||||
# Type is immutable on edit (changes the meaning of config/target);
|
||||
# to switch type, delete and re-add.
|
||||
m.name = form.get("name", "").strip() or m.target
|
||||
m.target = _normalise_target(m.type, form.get("target", m.target))
|
||||
m.config_json = json.dumps(_build_config(form, m.type))
|
||||
m.check_interval_seconds = int(form.get("check_interval_seconds", 0) or 0)
|
||||
return redirect(url_for("monitors.index"))
|
||||
|
||||
|
||||
@monitors_bp.post("/<monitor_id>/toggle")
|
||||
@require_role(UserRole.operator)
|
||||
async def toggle_monitor(monitor_id: str):
|
||||
async with current_app.db_sessionmaker() as db:
|
||||
async with db.begin():
|
||||
m = await db.get(Monitor, monitor_id)
|
||||
if m:
|
||||
m.enabled = not m.enabled
|
||||
return redirect(request.referrer or url_for("monitors.index"))
|
||||
|
||||
|
||||
@monitors_bp.post("/<monitor_id>/delete")
|
||||
@require_role(UserRole.operator)
|
||||
async def delete_monitor(monitor_id: str):
|
||||
async with current_app.db_sessionmaker() as db:
|
||||
async with db.begin():
|
||||
m = await db.get(Monitor, monitor_id)
|
||||
if m:
|
||||
await db.delete(m)
|
||||
return redirect(request.referrer or url_for("monitors.index"))
|
||||
|
||||
|
||||
@monitors_bp.get("/widget")
|
||||
@require_role(UserRole.viewer)
|
||||
async def widget():
|
||||
type_filter = request.args.get("type_filter", "all")
|
||||
show_down_only = request.args.get("show_down_only", "no") == "yes"
|
||||
limit = max(1, min(50, int(request.args.get("limit", 10) or 10)))
|
||||
widget_id = request.args.get("wid", "0")
|
||||
|
||||
data, up, down, pending, _ = await _rows_data(DEFAULT_RANGE)
|
||||
if type_filter in MONITOR_TYPE_VALUES:
|
||||
data = [d for d in data if d["monitor"].type == type_filter]
|
||||
if show_down_only:
|
||||
data = [d for d in data if not (d["latest"] and d["latest"].is_up)]
|
||||
|
||||
return await render_template(
|
||||
"monitors/widget.html",
|
||||
monitor_data=data[:limit], up=up, down=down, pending=pending,
|
||||
show_down_only=show_down_only, widget_id=widget_id,
|
||||
)
|
||||
@@ -0,0 +1,56 @@
|
||||
"""Unified check dispatcher: Monitor -> normalised MonitorResult fields.
|
||||
|
||||
One entry point, `run_monitor`, dispatches on Monitor.type to the per-type
|
||||
probes (ping/dns/http) and returns a dict with every MonitorResult field so the
|
||||
scheduler can persist a row uniformly regardless of type.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
import logging
|
||||
|
||||
from steward.models.monitors import Monitor, MonitorType
|
||||
from .ping import tcp_check, icmp_check, DEFAULT_TCP_PORT
|
||||
from .dns import dns_check
|
||||
from .http import http_check
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Every result carries these keys; type-specific probes fill the relevant ones.
|
||||
_BLANK = {
|
||||
"is_up": False,
|
||||
"response_ms": None,
|
||||
"status_code": None,
|
||||
"resolved_ip": None,
|
||||
"content_matched": None,
|
||||
"tls_expires_at": None,
|
||||
"error_msg": None,
|
||||
}
|
||||
|
||||
|
||||
async def run_monitor(monitor: Monitor) -> dict:
|
||||
"""Run one monitor's check and return a full MonitorResult field dict."""
|
||||
cfg = monitor.config
|
||||
result = dict(_BLANK)
|
||||
try:
|
||||
if monitor.type == MonitorType.icmp.value:
|
||||
result.update(await icmp_check(monitor.target))
|
||||
elif monitor.type == MonitorType.tcp.value:
|
||||
result.update(await tcp_check(monitor.target, int(cfg.get("port") or DEFAULT_TCP_PORT)))
|
||||
elif monitor.type == MonitorType.dns.value:
|
||||
result.update(await dns_check(monitor.target, cfg.get("expected_ip") or None))
|
||||
elif monitor.type == MonitorType.http.value:
|
||||
result.update(await http_check(
|
||||
url=monitor.target,
|
||||
method=cfg.get("method", "GET"),
|
||||
expected_status=int(cfg.get("expected_status", 200) or 200),
|
||||
content_match=cfg.get("content_match", "") or "",
|
||||
headers=cfg.get("headers") or {},
|
||||
timeout_seconds=int(cfg.get("timeout_seconds", 10) or 10),
|
||||
follow_redirects=bool(cfg.get("follow_redirects", True)),
|
||||
verify_ssl=bool(cfg.get("verify_ssl", True)),
|
||||
))
|
||||
else:
|
||||
result["error_msg"] = f"Unknown monitor type {monitor.type!r}"
|
||||
except Exception as exc: # a probe should never take the scheduler down
|
||||
logger.exception("Monitor %r (%s) check raised", monitor.name, monitor.type)
|
||||
result["error_msg"] = str(exc)[:512]
|
||||
return result
|
||||
@@ -0,0 +1,76 @@
|
||||
"""Single scheduled task that runs every due Monitor of any type."""
|
||||
from __future__ import annotations
|
||||
import asyncio
|
||||
import logging
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from sqlalchemy import select
|
||||
|
||||
from steward.core.alerts import record_metric
|
||||
from steward.models.monitors import Monitor, MonitorResult
|
||||
from .runner import run_monitor
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
async def run_due_monitors(app) -> None:
|
||||
"""Check every enabled Monitor whose interval has elapsed, persist results.
|
||||
|
||||
Per-monitor `check_interval_seconds` overrides the global
|
||||
MONITORS_POLL_INTERVAL (0 = use global), mirroring how the scheduler fires
|
||||
on the global cadence but each monitor decides if it is actually due.
|
||||
"""
|
||||
global_interval = int(app.config.get("MONITORS_POLL_INTERVAL", 60))
|
||||
now = datetime.now(timezone.utc)
|
||||
|
||||
async with app.db_sessionmaker() as session:
|
||||
monitors = list((await session.execute(
|
||||
select(Monitor).where(Monitor.enabled.is_(True))
|
||||
)).scalars())
|
||||
|
||||
due: list[Monitor] = []
|
||||
for m in monitors:
|
||||
interval = m.check_interval_seconds or global_interval
|
||||
if m.last_checked_at is None or (now - m.last_checked_at).total_seconds() >= interval:
|
||||
due.append(m)
|
||||
if not due:
|
||||
return
|
||||
|
||||
pairs = await asyncio.gather(
|
||||
*[run_monitor(m) for m in due], return_exceptions=True
|
||||
)
|
||||
check_time = datetime.now(timezone.utc)
|
||||
|
||||
async with app.db_sessionmaker() as session:
|
||||
async with session.begin():
|
||||
for monitor, res in zip(due, pairs):
|
||||
if isinstance(res, Exception):
|
||||
logger.error("Monitor %r check errored: %s", monitor.name, res)
|
||||
continue
|
||||
session.add(MonitorResult(
|
||||
monitor_id=monitor.id,
|
||||
checked_at=check_time,
|
||||
is_up=res["is_up"],
|
||||
response_ms=res["response_ms"],
|
||||
status_code=res["status_code"],
|
||||
resolved_ip=res["resolved_ip"],
|
||||
content_matched=res["content_matched"],
|
||||
tls_expires_at=res["tls_expires_at"],
|
||||
error_msg=res["error_msg"],
|
||||
))
|
||||
db_monitor = await session.get(Monitor, monitor.id)
|
||||
if db_monitor:
|
||||
db_monitor.last_checked_at = check_time
|
||||
|
||||
# Alert pipeline — keyed by monitor type (source) + name.
|
||||
await record_metric(
|
||||
session=session, source_module=monitor.type,
|
||||
resource_name=monitor.name, metric_name="is_up",
|
||||
value=1.0 if res["is_up"] else 0.0,
|
||||
)
|
||||
if res["response_ms"] is not None:
|
||||
await record_metric(
|
||||
session=session, source_module=monitor.type,
|
||||
resource_name=monitor.name, metric_name="response_ms",
|
||||
value=res["response_ms"],
|
||||
)
|
||||
Reference in New Issue
Block a user