# steward/core/status.py """Unified monitor-status aggregation. A single readable "is everything up?" surface (the Status page + dashboard widget) is assembled from heterogeneous monitors. The core unified Monitor entity (ping/dns/http) contributes via `monitor_status_source`; plugins may register their own sources from setup(). Each source is an async callable(db) returning normalised StatusEntry objects so the Status page never imports plugin tables directly. """ from __future__ import annotations import logging from dataclasses import dataclass, field from datetime import datetime, timedelta, timezone from typing import Awaitable, Callable from sqlalchemy import and_, case, func, select logger = logging.getLogger(__name__) HEARTBEAT_COUNT = 30 # number of recent checks shown in the heartbeat bar @dataclass class StatusEntry: """One monitored thing, normalised across monitor types for display.""" kind: str # "icmp" | "tcp" | "dns" | "http" | ... key: str # unique within kind (monitor id) name: str target: str = "" # address / URL shown as subtitle status: str = "pending" # "up" | "down" | "pending" last_checked: datetime | None = None uptime: dict[str, float | None] = field(default_factory=dict) # 24h/7d/30d heartbeat: list[dict] = field(default_factory=list) # oldest-first {state,title} latency_ms: float | None = None spark: list[float] = field(default_factory=list) # response series tls_days: float | None = None # days until TLS expiry (http only) detail_url: str | None = None # When the current up/down state began, as far as the recent checks show; # since_at_least means every check we have agrees, so it began earlier. since: datetime | None = None since_at_least: bool = False StatusSource = Callable[[object], Awaitable[list[StatusEntry]]] _SOURCES: list[StatusSource] = [] def register_status_source(fn: StatusSource) -> None: """Register a status source. Idempotent on the same callable.""" if fn not in _SOURCES: _SOURCES.append(fn) def clear_status_sources() -> None: """Reset the registry (used by tests).""" _SOURCES.clear() async def collect_status(db) -> list[StatusEntry]: """Gather entries from every registered source, down-first then by name. A failing source is logged and skipped so one broken plugin can't blank the whole Status page. """ entries: list[StatusEntry] = [] for src in list(_SOURCES): try: entries.extend(await src(db)) except Exception: logger.exception("status source %r failed", getattr(src, "__name__", src)) # Down first (most urgent), then pending, then up; alphabetical within. order = {"down": 0, "pending": 1, "up": 2} entries.sort(key=lambda e: (order.get(e.status, 3), e.kind, e.name.lower())) return entries def sparkline_svg(values: list[float], width: int = 80, height: int = 20, stroke: str = "#6060c0", fill_width: bool = False, floor: float | None = None) -> str: """Tiny inline SVG polyline for a response-time series. With fill_width the line stretches across its container (CSS sets the width) and keeps a 1.5px stroke. With `floor` the line is drawn from zero up to its own peak, but never against less than `floor`, so a value idling near zero draws flat rather than its noise filling the height (UI & Design ruling 8).""" vals = [v for v in values if v is not None] size = (f'width="100%" height="{height}" preserveAspectRatio="none"' if fill_width else f'width="{width}" height="{height}"') if len(vals) < 2: return f'' if floor is not None: mn, mx = 0.0, max(max(vals), floor) else: mn, mx = min(vals), max(vals) if mx == mn: mx = mn + 1.0 step = width / (len(vals) - 1) pts = [] for i, v in enumerate(vals): x = i * step y = height - (v - mn) / (mx - mn) * (height - 2) - 1 pts.append(f"{x:.1f},{y:.1f}") poly = " ".join(pts) style = "display:block;" if fill_width else "vertical-align:middle;" keep = ' vector-effect="non-scaling-stroke"' if fill_width else "" return ( f'' f'' f'' ) # ── Unified monitor status source ───────────────────────────────────────────── async def _last_n_results(db, monitor_ids: list[str], n: int = HEARTBEAT_COUNT) -> dict[str, list]: """Return {monitor_id: [MonitorResult]} of the last n results, oldest-first. One LATERAL probe per monitor walks ix_monitor_results_monitor_checked backwards and stops after n rows. A row_number() over each monitor's results read its whole history first, which held Status and the monitor widgets at ~16 s (#5208). """ from sqlalchemy import true from sqlalchemy.orm import aliased from steward.models.monitors import Monitor, MonitorResult if not monitor_ids: return {} ids = select(Monitor.id.label("mid")).where(Monitor.id.in_(monitor_ids)).subquery("wanted") recent = ( select(MonitorResult) .where(MonitorResult.monitor_id == ids.c.mid) .order_by(MonitorResult.checked_at.desc()) .limit(n) .lateral("recent") ) row = aliased(MonitorResult, recent) res = await db.execute(select(row).select_from(ids).join(recent, true())) out: dict[str, list] = {mid: [] for mid in monitor_ids} for r in res.scalars(): out[r.monitor_id].append(r) for rows in out.values(): rows.sort(key=lambda r: r.checked_at) return out async def _uptime_by_monitor(db, monitor_ids: list[str]) -> dict[str, dict]: """Per-monitor uptime % over 24h/7d/30d in one grouped query.""" from steward.models.monitors import MonitorResult if not monitor_ids: return {} now = datetime.now(timezone.utc) c30, c7, c24 = now - timedelta(days=30), now - timedelta(days=7), now - timedelta(hours=24) res = await db.execute( select( MonitorResult.monitor_id, func.count().label("total_30d"), func.sum(case((MonitorResult.is_up.is_(True), 1), else_=0)).label("up_30d"), func.sum(case((MonitorResult.checked_at >= c7, 1), else_=0)).label("total_7d"), func.sum(case((and_(MonitorResult.checked_at >= c7, MonitorResult.is_up.is_(True)), 1), else_=0)).label("up_7d"), func.sum(case((MonitorResult.checked_at >= c24, 1), else_=0)).label("total_24h"), func.sum(case((and_(MonitorResult.checked_at >= c24, MonitorResult.is_up.is_(True)), 1), else_=0)).label("up_24h"), ) .where(MonitorResult.monitor_id.in_(monitor_ids)) .where(MonitorResult.checked_at >= c30) .group_by(MonitorResult.monitor_id) ) def _pct(up, total): return round(float(up) / float(total) * 100, 2) if total else None out: dict[str, dict] = {} for r in res: out[r.monitor_id] = { "24h": _pct(r.up_24h, r.total_24h), "7d": _pct(r.up_7d, r.total_7d), "30d": _pct(r.up_30d, r.total_30d), } return out def streak_start(results) -> tuple[datetime | None, bool]: """(checked_at of the first check in the latest run of the same result, whether that run fills every check given). Results are oldest-first.""" if not results: return None, False i = len(results) - 1 while i > 0 and results[i - 1].is_up == results[-1].is_up: i -= 1 return results[i].checked_at, i == 0 def _heartbeat_title(mtype: str, r) -> str: """Type-aware tooltip for one heartbeat cell.""" ts = f"{r.checked_at:%H:%M:%S} UTC" if not r.is_up: return f"{(r.error_msg or 'Down')} — {ts}" if mtype == "http": code = r.status_code or "—" ms = f"{r.response_ms:.0f} ms" if r.response_ms is not None else "" return f"{code} · {ms} — {ts}".replace(" · — ", " — ") if mtype == "dns": return f"{r.resolved_ip or 'resolved'} — {ts}" if r.response_ms is not None: return f"{r.response_ms:.0f} ms — {ts}" return f"Up — {ts}" async def monitor_status_source(db) -> list[StatusEntry]: """Contribute every enabled Monitor (all types) to the Status page.""" from steward.models.monitors import Monitor monitors = list((await db.execute( select(Monitor).where(Monitor.enabled.is_(True)).order_by(Monitor.name) )).scalars()) ids = [m.id for m in monitors] recent = await _last_n_results(db, ids) uptime = await _uptime_by_monitor(db, ids) now = datetime.now(timezone.utc) entries: list[StatusEntry] = [] for m in monitors: rows = recent.get(m.id, []) latest = rows[-1] if rows else None status = "pending" if latest is None else ("up" if latest.is_up else "down") since, since_at_least = streak_start(rows) tls_days = None if latest and latest.tls_expires_at: tls_days = (latest.tls_expires_at - now).total_seconds() / 86400.0 entries.append(StatusEntry( kind=m.type, key=m.id, name=m.name, target=m.target, status=status, last_checked=latest.checked_at if latest else None, uptime=uptime.get(m.id, {}), heartbeat=[{"state": "up" if r.is_up else "down", "title": _heartbeat_title(m.type, r)} for r in rows], latency_ms=latest.response_ms if latest else None, spark=[r.response_ms for r in rows if r.response_ms is not None], tls_days=tls_days, detail_url=f"/monitors/{m.id}", since=since, since_at_least=since_at_least, )) return entries