bvandeusenandClaude Opus 5.5
58bfc7c361
#5060 "One evaluation path: record_metrics batch API; host-agent ingest evaluates rules". - record_metrics(session, rows) writes metric rows and evaluates the rules that watch them, reading rules, states and maintenance windows once per batch. Each rule sees its key's samples oldest first, so an agent catching up after an outage counts consecutive checks as they happened. - Host-agent ingest now goes through it. Before, it wrote plugin_metrics directly and no host_agent rule could ever fire. The catalog gains the per-mount disk_used_pct ("<host>:<mount>"). - MetricBatch gathers a transaction's samples; the docker ingest, traefik, unifi, snmp, monitor and self-monitor writers record one batch each instead of a rule query per sample. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
79 lines
3.0 KiB
Python
79 lines
3.0 KiB
Python
"""Single scheduled task that runs every due Monitor of any type."""
|
|
from __future__ import annotations
|
|
import asyncio
|
|
import logging
|
|
from datetime import datetime, timezone
|
|
|
|
from sqlalchemy import select
|
|
|
|
from steward.core.alerts import MetricBatch
|
|
from steward.models.monitors import Monitor, MonitorResult
|
|
from .runner import run_monitor
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
async def run_due_monitors(app) -> None:
|
|
"""Check every enabled Monitor whose interval has elapsed, persist results.
|
|
|
|
Per-monitor `check_interval_seconds` overrides the global
|
|
MONITORS_POLL_INTERVAL (0 = use global), mirroring how the scheduler fires
|
|
on the global cadence but each monitor decides if it is actually due.
|
|
"""
|
|
global_interval = int(app.config.get("MONITORS_POLL_INTERVAL", 60))
|
|
now = datetime.now(timezone.utc)
|
|
|
|
async with app.db_sessionmaker() as session:
|
|
monitors = list((await session.execute(
|
|
select(Monitor).where(Monitor.enabled.is_(True))
|
|
)).scalars())
|
|
|
|
due: list[Monitor] = []
|
|
for m in monitors:
|
|
interval = m.check_interval_seconds or global_interval
|
|
if m.last_checked_at is None or (now - m.last_checked_at).total_seconds() >= interval:
|
|
due.append(m)
|
|
if not due:
|
|
return
|
|
|
|
pairs = await asyncio.gather(
|
|
*[run_monitor(m) for m in due], return_exceptions=True
|
|
)
|
|
check_time = datetime.now(timezone.utc)
|
|
|
|
async with app.db_sessionmaker() as session:
|
|
async with session.begin():
|
|
for monitor, res in zip(due, pairs):
|
|
if isinstance(res, Exception):
|
|
logger.error("Monitor %r check errored: %s", monitor.name, res)
|
|
continue
|
|
session.add(MonitorResult(
|
|
monitor_id=monitor.id,
|
|
checked_at=check_time,
|
|
is_up=res["is_up"],
|
|
response_ms=res["response_ms"],
|
|
status_code=res["status_code"],
|
|
resolved_ip=res["resolved_ip"],
|
|
content_matched=res["content_matched"],
|
|
tls_expires_at=res["tls_expires_at"],
|
|
error_msg=res["error_msg"],
|
|
))
|
|
db_monitor = await session.get(Monitor, monitor.id)
|
|
if db_monitor:
|
|
db_monitor.last_checked_at = check_time
|
|
|
|
# Alert pipeline — keyed by monitor type (source) + name.
|
|
batch = MetricBatch()
|
|
batch.add(
|
|
source_module=monitor.type,
|
|
resource_name=monitor.name, metric_name="is_up",
|
|
value=1.0 if res["is_up"] else 0.0,
|
|
)
|
|
if res["response_ms"] is not None:
|
|
batch.add(
|
|
source_module=monitor.type,
|
|
resource_name=monitor.name, metric_name="response_ms",
|
|
value=res["response_ms"],
|
|
)
|
|
await batch.record(session)
|