"""get_overview: what is wrong right now, ranked, across everything Steward watches.""" from __future__ import annotations import logging from datetime import timedelta from sqlalchemy import select from steward.alerts import service as alert_svc from steward.hosts import service as host_svc from steward.mcp._context import mcp_app from steward.models.ansible import AnsibleRun, AnsibleRunStatus from steward.models.hosts import Host from steward.models.monitors import Monitor from steward.models.users import UserRole from steward.monitors import service as monitor_svc from ._shape import age_s, db_session, iso, now_utc, rnd logger = logging.getLogger(__name__) FINDINGS_CAP = 50 _SEVERITY = {"critical": 0, "warning": 1} def finding(severity: str, kind: str, subject: str, summary: str, *, since=None, ids: dict | None = None, next: str | None = None) -> dict: """One overview finding. Plugins' `mcp_overview(db)` return a list of these.""" return { "severity": severity, "kind": kind, "subject": subject, "summary": summary, "since": iso(since), "age_s": age_s(since), "ids": ids or {}, "next": next, } def rank(findings: list[dict]) -> list[dict]: """Critical first; within a severity, the longest-standing first.""" return sorted(findings, key=lambda f: (_SEVERITY.get(f.get("severity"), 2), -(f.get("age_s") or 0), f.get("subject", ""))) def monitor_findings(monitors, states: dict, host_names: dict, now, tls: dict | None = None) -> list[dict]: """Failing monitors, and HTTPS certificates inside the TLS window (`tls` is THRESHOLDS["tls_days"]: warn/crit days left).""" tls = tls or {"warn": 14, "crit": 3} out = [] for m in monitors: if not m.enabled: continue st = states.get(m.id) or {} latest = st.get("latest") if latest is None: continue where = f" on {host_names[m.host_id]}" if m.host_id in host_names else "" ids = {"monitor_id": m.id, "host_id": m.host_id} if not latest.is_up: out.append(finding( "critical", "monitor_down", m.name, f"{m.type} check of {m.target}{where} is failing: {latest.error_msg or 'down'}", since=st.get("down_since") or latest.checked_at, ids=ids, next=f'get_monitor(monitor="{m.id}")')) elif latest.tls_expires_at is not None: days = (latest.tls_expires_at - now).total_seconds() / 86400.0 if days <= tls["warn"]: out.append(finding( "critical" if days <= tls["crit"] else "warning", "tls_expiring", m.name, f"TLS certificate for {m.target} expires in {days:.1f} days", ids=ids, next=f'get_monitor(monitor="{m.id}")')) return out def alert_findings(active) -> list[dict]: out = [] for state, rule in active: s = state.state.value if s == "firing": sev, what = "critical", "is firing" elif s == "pending": sev, what = "warning", "is pending (breaching, not yet for long enough to fire)" else: sev, what = "warning", "is acknowledged but still breaching" out.append(finding( sev, f"alert_{s}", rule.name, f"{rule.metric_name} {rule.operator.value} {rule.threshold:g} on " f"{rule.source_module}/{rule.resource_name} {what}", since=state.last_transition_at, ids={"rule_id": rule.id}, next=f'get_alert_history(rule="{rule.id}")')) return out _AGENT_CHECKS = (("cpu_pct", "cpu", "CPU"), ("mem_used_pct", "mem", "memory"), ("disk_used_pct_worst", "disk", "fullest disk")) def agent_findings(hosts, agent: dict, thresholds: dict) -> list[dict]: out = [] for h in hosts: a = agent.get(h.name) if not a: continue ids = {"host_id": h.id} if not a.get("fresh"): out.append(finding( "warning", "agent_stale", h.name, "host agent has stopped reporting; its metrics are out of date", since=a.get("last_at"), ids=ids, next=f'get_host(host="{h.id}")')) continue # stale values would only add noise for metric, kind, label in _AGENT_CHECKS: value = a.get(metric) th = thresholds.get(kind) or {} if value is None or th.get("warn") is None: continue if th.get("crit") is not None and value >= th["crit"]: sev = "critical" elif value >= th["warn"]: sev = "warning" else: continue out.append(finding( sev, f"host_{kind}_high", h.name, f"{label} at {rnd(value, 1)}% (warn {th['warn']}%, crit {th.get('crit')}%)", ids=ids, next=f'get_host_metrics(host="{h.id}", metric="{metric}")')) return out def run_findings(runs) -> list[dict]: out = [] for r in runs: failures = ((r.results or {}).get("failures") or [])[:3] detail = f": {'; '.join(str(f)[:200] for f in failures)}" if failures else "" out.append(finding( "warning", f"ansible_run_{r.status.value}", r.playbook_path, f"Ansible run of {r.playbook_path} ({r.source_name}) {r.status.value}{detail}", since=r.finished_at or r.started_at, ids={"run_id": r.id})) return out async def _plugin_findings(server) -> tuple[list[dict], list[str]]: found, unavailable = [], [] for plugin, source in sorted((getattr(server, "overview_sources", None) or {}).items()): if not server.plugin_enabled(plugin): continue try: async with db_session() as db: found.extend(await source(db)) except Exception: logger.exception("MCP overview source of plugin %r failed", plugin) unavailable.append(plugin) return found, unavailable async def get_overview() -> dict: """What is wrong right now, across everything Steward watches. Start here. Returns `findings` ranked critical first, then longest-standing first: failing monitors (with the error and since when), firing, pending and still-breaching acknowledged alerts, TLS certificates near expiry, host agents that stopped reporting, hosts over their CPU, memory or disk limits (all cutoffs are set under Settings → Thresholds), Ansible runs that failed in the last 24h, and anything enabled plugins add (unhealthy containers, for one). Each finding carries the ids to follow and `next`, the call that shows its evidence. `counts` sizes the fleet, and `active_maintenance` names windows currently silencing alerts. `all_clear` is true when nothing is wrong. """ app = mcp_app() now = now_utc() async with db_session() as db: hosts = list((await db.execute(select(Host).order_by(Host.name))).scalars()) host_names = {h.id: h.name for h in hosts} monitors = list((await db.execute(select(Monitor).order_by(Monitor.name))).scalars()) states = await monitor_svc.monitor_states(db, [m for m in monitors if m.enabled]) active = await alert_svc.active_alerts(db) agent = await host_svc.agent_overview_by_host( db, [h.name for h in hosts], host_svc.agent_stale_after_seconds(app.config)) runs = list((await db.execute( select(AnsibleRun) .where(AnsibleRun.status.in_((AnsibleRunStatus.failed, AnsibleRunStatus.interrupted)), AnsibleRun.started_at >= now - timedelta(hours=24)) .order_by(AnsibleRun.started_at.desc()) .limit(20) )).scalars()) windows = await alert_svc.list_maintenance_windows(db) thresholds = app.config.get("THRESHOLDS") or {} findings = (monitor_findings(monitors, states, host_names, now, thresholds.get("tls_days")) + alert_findings(active) + agent_findings(hosts, agent, thresholds) + run_findings(runs)) plugin_found, unavailable = await _plugin_findings(getattr(app, "mcp_server", None)) findings = rank(findings + plugin_found) down = sum(1 for m in monitors if m.enabled and (st := states.get(m.id)) and st["latest"] is not None and not st["latest"].is_up) out = { "generated_at": iso(now), "all_clear": not findings, "counts": { "hosts": len(hosts), "monitors": len(monitors), "monitors_down": down, "alerts_active": len(active), "critical": sum(1 for f in findings if f["severity"] == "critical"), "warning": sum(1 for f in findings if f["severity"] == "warning"), }, "active_maintenance": [ {"window_id": w.id, "name": w.name, "end_at": iso(w.end_at), "scope": {"source_module": w.scope_source, "resource": w.scope_resource}} for w in windows if w.start_at <= now <= w.end_at ], "findings": findings[:FINDINGS_CAP], "truncated": len(findings) > FINDINGS_CAP, } if out["truncated"]: out["more"] = (f"{len(findings) - FINDINGS_CAP} lower-ranked findings left out; " "list_monitors(state=\"down\"), list_alerts and list_hosts show everything.") if unavailable: out["unavailable"] = [f"{p} plugin's findings could not be read (see Steward's log)" for p in unavailable] return out def register(register_tool) -> None: register_tool(get_overview, role=UserRole.viewer, read_only=True)