"""get_overview: what is wrong right now, ranked, across everything Steward watches.""" from __future__ import annotations import logging from datetime import timedelta from sqlalchemy import select from steward.alerts import service as alert_svc from steward.core import alerts from steward.core import anomalies as anom from steward.hosts import service as host_svc from steward.mcp._context import mcp_app from steward.models.ansible import AnsibleRun, AnsibleRunStatus from steward.models.hosts import Host from steward.models.monitors import Monitor from steward.models.users import UserRole from steward.monitors import service as monitor_svc from ._shape import age_s, db_session, iso, now_utc, rnd logger = logging.getLogger(__name__) FINDINGS_CAP = 50 _SEVERITY = {"critical": 0, "warning": 1} def finding(severity: str, kind: str, subject: str, summary: str, *, since=None, ids: dict | None = None, next: str | None = None) -> dict: """One overview finding. Plugins' `mcp_overview(db)` return a list of these.""" return { "severity": severity, "kind": kind, "subject": subject, "summary": summary, "since": iso(since), "age_s": age_s(since), "ids": ids or {}, "next": next, } def rank(findings: list[dict]) -> list[dict]: """Critical first; within a severity, the longest-standing first.""" return sorted(findings, key=lambda f: (_SEVERITY.get(f.get("severity"), 2), -(f.get("age_s") or 0), f.get("subject", ""))) def monitor_findings(monitors, states: dict, host_names: dict, now, tls: dict | None = None) -> list[dict]: """Failing monitors, and HTTPS certificates inside the TLS window (`tls` is THRESHOLDS["tls_days"]: warn/crit days left).""" tls = tls or {"warn": 14, "crit": 3} out = [] for m in monitors: if not m.enabled: continue st = states.get(m.id) or {} latest = st.get("latest") if latest is None: continue where = f" on {host_names[m.host_id]}" if m.host_id in host_names else "" ids = {"monitor_id": m.id, "host_id": m.host_id} if not latest.is_up: out.append(finding( "critical", "monitor_down", m.name, f"{m.type} check of {m.target}{where} is failing: {latest.error_msg or 'down'}", since=st.get("down_since") or latest.checked_at, ids=ids, next=f'get_monitor(monitor="{m.id}")')) elif latest.tls_expires_at is not None: days = (latest.tls_expires_at - now).total_seconds() / 86400.0 if days <= tls["warn"]: out.append(finding( "critical" if days <= tls["crit"] else "warning", "tls_expiring", m.name, f"TLS certificate for {m.target} expires in {days:.1f} days", ids=ids, next=f'get_monitor(monitor="{m.id}")')) return out def fold_agent_silence(active, agent: dict) -> tuple[list, dict[str, list[str]], set[str]]: """One silent agent is one finding (#5721). Returns (the active alerts left to list, {host: names of its rules silent with its agent}, the hosts whose "agent offline" alert is active and so speaks for the agent). A rule the agent feeds that has only gone quiet, on a host whose agent is stale, is folded under that host; one firing or pending stays listed, since what it measured before going quiet is news of its own.""" offline = {name for name, a in agent.items() if a and not a.get("fresh")} kept, silent, speaking = [], {}, set() for state, rule in active: if alerts.is_heartbeat(rule): speaking.add(rule.resource_name) kept.append((state, rule)) continue host = alerts.host_behind(rule) if host in offline and state.no_data_since is not None \ and state.state.value not in ("firing", "pending", "acknowledged"): silent.setdefault(host, []).append(rule.name) continue kept.append((state, rule)) return kept, silent, speaking def _also_silent(names: list[str] | None) -> str: return f"; also silent with it: {', '.join(sorted(names))}" if names else "" def alert_findings(active, silent: dict[str, list[str]] | None = None) -> list[dict]: out = [] silent = silent or {} for state, rule in active: s = state.state.value if s == "firing": sev, what = "critical", "is firing" elif s == "pending": sev, what = "warning", "is pending (breaching, not yet for long enough to fire)" elif s == "acknowledged": sev, what = "warning", "is acknowledged but still breaching" else: s, sev, what = "no_data", "warning", "has stopped reporting" if state.no_data_since is not None and s != "no_data": what += f" and has stopped reporting (no data since {iso(state.no_data_since)})" if alerts.is_heartbeat(rule): what += _also_silent(silent.get(rule.resource_name)) out.append(finding( sev, f"alert_{s}", rule.name, f"{rule.metric_name} {rule.operator.value} {rule.threshold:g} on " f"{rule.source_module}/{rule.resource_name} {what}", since=state.no_data_since if s == "no_data" else state.last_transition_at, ids={"rule_id": rule.id}, next=f'get_alert_history(rule="{rule.id}")')) return out _AGENT_CHECKS = (("cpu_pct", "cpu", "CPU"), ("mem_used_pct", "mem", "memory"), ("disk_used_pct_worst", "disk", "fullest disk (space or inodes)")) def agent_findings(hosts, agent: dict, thresholds: dict, silent: dict[str, list[str]] | None = None, speaking: set[str] | frozenset = frozenset()) -> list[dict]: """Hosts past a CPU, memory or disk cutoff, and agents that stopped reporting. A stale agent whose "agent offline" alert is active (`speaking`) is that alert's finding, not a second one (#5721).""" out = [] silent = silent or {} for h in hosts: a = agent.get(h.name) if not a: continue ids = {"host_id": h.id} if not a.get("fresh"): if h.name not in speaking: out.append(finding( "warning", "agent_stale", h.name, "host agent has stopped reporting; its metrics are out of date" + _also_silent(silent.get(h.name)), since=a.get("last_at"), ids=ids, next=f'get_host(host="{h.id}")')) continue # stale values would only add noise # CPU and memory within the host's own normal for the hour aren't # findings (#5742); the fullest disk keeps its cutoff. normals = host_svc.host_normals(a.get("norms"), thresholds) for metric, kind, label in _AGENT_CHECKS: value = a.get(metric) th = thresholds.get(kind) or {} if value is None or th.get("warn") is None: continue if th.get("crit") is not None and value >= th["crit"]: sev = "critical" elif value >= th["warn"]: sev = "warning" else: continue if kind in normals and not anom.against(value, normals[kind][0], normals[kind][1], "%")[0]: continue out.append(finding( sev, f"host_{kind}_high", h.name, f"{label} at {rnd(value, 1)}% (warn {th['warn']}%, crit {th.get('crit')}%)", ids=ids, next=f'get_host_metrics(host="{h.id}", metric="{metric}")')) return out def run_findings(runs) -> list[dict]: out = [] for r in runs: failures = ((r.results or {}).get("failures") or [])[:3] detail = f": {'; '.join(str(f)[:200] for f in failures)}" if failures else "" out.append(finding( "warning", f"ansible_run_{r.status.value}", r.playbook_path, f"Ansible run of {r.playbook_path} ({r.source_name}) {r.status.value}{detail}", since=r.finished_at or r.started_at, ids={"run_id": r.id})) return out def anomaly_findings(open_now) -> list[dict]: """Metric anomalies going on now (#5740): a warning each.""" from steward.core.formatting import metric_label, metric_value out = [] for a in open_now: out.append(finding( "warning", "metric_anomaly", f"{a.resource_name} {metric_label(a.metric_name)}", f"Metric anomaly: {metric_label(a.metric_name)} on {a.resource_name} held " f"{metric_value(a.metric_name, a.peak_level)} where " f"{metric_value(a.metric_name, a.usual_hold)} is usual for that hour" + (f" ({a.occurrences} times since {iso(a.started_at)})" if a.occurrences > 1 else ""), since=a.last_started_at, ids={"anomaly_id": a.id}, next=f'list_anomalies(review="{a.review}")')) return out async def _plugin_findings(server) -> tuple[list[dict], list[str]]: found, unavailable = [], [] for plugin, source in sorted((getattr(server, "overview_sources", None) or {}).items()): if not server.plugin_enabled(plugin): continue try: async with db_session() as db: found.extend(await source(db)) except Exception: logger.exception("MCP overview source of plugin %r failed", plugin) unavailable.append(plugin) return found, unavailable async def get_overview() -> dict: """What is wrong right now, across everything Steward watches. Start here. Returns `findings` ranked critical first, then longest-standing first: failing monitors (with the error and since when), firing, pending and still-breaching acknowledged alerts, TLS certificates near expiry, host agents that stopped reporting, hosts over their CPU, memory or disk limits (all cutoffs are set under Settings → Thresholds), Ansible runs that failed in the last 24h, metrics going on now well above what is normal for them at this hour (metric anomalies), and anything enabled plugins add (unhealthy containers, for one). Each finding carries the ids to follow and `next`, the call that shows its evidence. `counts` sizes the fleet; `counts.anomalies_to_review` is how many metric anomalies wait for someone to mark them normal or real (list_anomalies). `active_maintenance` names windows currently silencing alerts. `all_clear` is true when nothing is wrong. """ app = mcp_app() now = now_utc() async with db_session() as db: hosts = list((await db.execute(select(Host).order_by(Host.name))).scalars()) host_names = {h.id: h.name for h in hosts} monitors = list((await db.execute(select(Monitor).order_by(Monitor.name))).scalars()) states = await monitor_svc.monitor_states(db, [m for m in monitors if m.enabled]) active = await alert_svc.active_alerts(db) agent = await host_svc.agent_overview_by_host( db, [h.name for h in hosts], host_svc.agent_stale_after_seconds(app.config)) runs = list((await db.execute( select(AnsibleRun) .where(AnsibleRun.status.in_((AnsibleRunStatus.failed, AnsibleRunStatus.interrupted)), AnsibleRun.started_at >= now - timedelta(hours=24)) .order_by(AnsibleRun.started_at.desc()) .limit(20) )).scalars()) windows = await alert_svc.list_maintenance_windows(db) open_anomalies = await anom.open_anomalies(db) to_review = await anom.to_review_count(db) thresholds = app.config.get("THRESHOLDS") or {} listed, silent, speaking = fold_agent_silence(active, agent) findings = (monitor_findings(monitors, states, host_names, now, thresholds.get("tls_days")) + alert_findings(listed, silent) + agent_findings(hosts, agent, thresholds, silent, speaking) + run_findings(runs) + anomaly_findings(open_anomalies)) plugin_found, unavailable = await _plugin_findings(getattr(app, "mcp_server", None)) findings = rank(findings + plugin_found) down = sum(1 for m in monitors if m.enabled and (st := states.get(m.id)) and st["latest"] is not None and not st["latest"].is_up) out = { "generated_at": iso(now), "all_clear": not findings, "counts": { "hosts": len(hosts), "monitors": len(monitors), "monitors_down": down, "alerts_active": len(active), "critical": sum(1 for f in findings if f["severity"] == "critical"), "warning": sum(1 for f in findings if f["severity"] == "warning"), "anomalies_to_review": to_review, }, "active_maintenance": [ {"window_id": w.id, "name": w.name, "end_at": iso(w.end_at), "scope": {"source_module": w.scope_source, "resource": w.scope_resource}} for w in windows if w.start_at <= now <= w.end_at ], "findings": findings[:FINDINGS_CAP], "truncated": len(findings) > FINDINGS_CAP, } if out["truncated"]: out["more"] = (f"{len(findings) - FINDINGS_CAP} lower-ranked findings left out; " "list_monitors(state=\"down\"), list_alerts and list_hosts show everything.") if unavailable: out["unavailable"] = [f"{p} plugin's findings could not be read (see Steward's log)" for p in unavailable] return out def register(register_tool) -> None: register_tool(get_overview, role=UserRole.viewer, read_only=True)