bvandeusenandClaude Opus 5.5
6fa62c9520
still_failing was handed the query's one-pass result. The first failed run's any() consumed it, so a second failure in the window found no successes left and was reported though a later run had put it right (ci-runner-volumes came back once keepalived-survey also had a failure). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
370 lines
17 KiB
Python
370 lines
17 KiB
Python
"""get_overview: what is wrong right now, ranked, across everything Steward watches."""
|
|
from __future__ import annotations
|
|
|
|
import logging
|
|
from datetime import timedelta
|
|
|
|
from sqlalchemy import select
|
|
|
|
from steward.alerts import service as alert_svc
|
|
from steward.backup import job as backup_job
|
|
from steward.core import alerts
|
|
from steward.core import anomalies as anom
|
|
from steward.core.settings import get_all_settings
|
|
from steward.hosts import service as host_svc
|
|
from steward.mcp._context import mcp_app
|
|
from steward.models.ansible import AnsibleRun, AnsibleRunStatus
|
|
from steward.models.hosts import Host
|
|
from steward.models.monitors import Monitor
|
|
from steward.models.users import UserRole
|
|
from steward.monitors import service as monitor_svc
|
|
|
|
from ._shape import age_s, db_session, iso, now_utc, rnd
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
FINDINGS_CAP = 50
|
|
# A scheduled backup this long past its slot with none made is overdue: one
|
|
# 15-minute tick and the hour a failed try waits, with room to spare.
|
|
BACKUP_OVERDUE_AFTER = timedelta(hours=2)
|
|
_SEVERITY = {"critical": 0, "warning": 1}
|
|
|
|
|
|
def finding(severity: str, kind: str, subject: str, summary: str, *, since=None,
|
|
ids: dict | None = None, next: str | None = None) -> dict:
|
|
"""One overview finding. Plugins' `mcp_overview(db)` return a list of these."""
|
|
return {
|
|
"severity": severity, "kind": kind, "subject": subject, "summary": summary,
|
|
"since": iso(since), "age_s": age_s(since), "ids": ids or {}, "next": next,
|
|
}
|
|
|
|
|
|
def rank(findings: list[dict]) -> list[dict]:
|
|
"""Critical first; within a severity, the longest-standing first."""
|
|
return sorted(findings, key=lambda f: (_SEVERITY.get(f.get("severity"), 2),
|
|
-(f.get("age_s") or 0), f.get("subject", "")))
|
|
|
|
|
|
def monitor_findings(monitors, states: dict, host_names: dict, now,
|
|
tls: dict | None = None) -> list[dict]:
|
|
"""Failing monitors, and HTTPS certificates inside the TLS window
|
|
(`tls` is THRESHOLDS["tls_days"]: warn/crit days left)."""
|
|
tls = tls or {"warn": 14, "crit": 3}
|
|
out = []
|
|
for m in monitors:
|
|
if not m.enabled:
|
|
continue
|
|
st = states.get(m.id) or {}
|
|
latest = st.get("latest")
|
|
if latest is None:
|
|
continue
|
|
where = f" on {host_names[m.host_id]}" if m.host_id in host_names else ""
|
|
ids = {"monitor_id": m.id, "host_id": m.host_id}
|
|
if not latest.is_up:
|
|
out.append(finding(
|
|
"critical", "monitor_down", m.name,
|
|
f"{m.type} check of {m.target}{where} is failing: {latest.error_msg or 'down'}",
|
|
since=st.get("down_since") or latest.checked_at, ids=ids,
|
|
next=f'get_monitor(monitor="{m.id}")'))
|
|
elif latest.tls_expires_at is not None:
|
|
days = (latest.tls_expires_at - now).total_seconds() / 86400.0
|
|
if days <= tls["warn"]:
|
|
out.append(finding(
|
|
"critical" if days <= tls["crit"] else "warning", "tls_expiring", m.name,
|
|
f"TLS certificate for {m.target} expires in {days:.1f} days",
|
|
ids=ids, next=f'get_monitor(monitor="{m.id}")'))
|
|
return out
|
|
|
|
|
|
def fold_agent_silence(active, agent: dict) -> tuple[list, dict[str, list[str]], set[str]]:
|
|
"""One silent agent is one finding (#5721). Returns (the active alerts left
|
|
to list, {host: names of its rules silent with its agent}, the hosts
|
|
whose "agent offline" alert is active and so speaks for the agent).
|
|
|
|
A rule the agent feeds that has only gone quiet, on a host whose agent is
|
|
stale, is folded under that host; one firing or pending stays listed,
|
|
since what it measured before going quiet is news of its own."""
|
|
offline = {name for name, a in agent.items() if a and not a.get("fresh")}
|
|
kept, silent, speaking = [], {}, set()
|
|
for state, rule in active:
|
|
if alerts.is_heartbeat(rule):
|
|
speaking.add(rule.resource_name)
|
|
kept.append((state, rule))
|
|
continue
|
|
host = alerts.host_behind(rule)
|
|
if host in offline and state.no_data_since is not None \
|
|
and state.state.value not in ("firing", "pending", "acknowledged"):
|
|
silent.setdefault(host, []).append(rule.name)
|
|
continue
|
|
kept.append((state, rule))
|
|
return kept, silent, speaking
|
|
|
|
|
|
def _also_silent(names: list[str] | None) -> str:
|
|
return f"; also silent with it: {', '.join(sorted(names))}" if names else ""
|
|
|
|
|
|
def alert_findings(active, silent: dict[str, list[str]] | None = None) -> list[dict]:
|
|
out = []
|
|
silent = silent or {}
|
|
for state, rule in active:
|
|
s = state.state.value
|
|
if s == "firing":
|
|
sev, what = "critical", "is firing"
|
|
elif s == "pending":
|
|
sev, what = "warning", "is pending (breaching, not yet for long enough to fire)"
|
|
elif s == "acknowledged":
|
|
sev, what = "warning", "is acknowledged but still breaching"
|
|
else:
|
|
s, sev, what = "no_data", "warning", "has stopped reporting"
|
|
if state.no_data_since is not None and s != "no_data":
|
|
what += f" and has stopped reporting (no data since {iso(state.no_data_since)})"
|
|
if alerts.is_heartbeat(rule):
|
|
what += _also_silent(silent.get(rule.resource_name))
|
|
out.append(finding(
|
|
sev, f"alert_{s}", rule.name,
|
|
f"{rule.metric_name} {rule.operator.value} {rule.threshold:g} on "
|
|
f"{rule.source_module}/{rule.resource_name} {what}",
|
|
since=state.no_data_since if s == "no_data" else state.last_transition_at,
|
|
ids={"rule_id": rule.id},
|
|
next=f'get_alert_history(rule="{rule.id}")'))
|
|
return out
|
|
|
|
|
|
_AGENT_CHECKS = (("cpu_pct", "cpu", "CPU"), ("mem_used_pct", "mem", "memory"),
|
|
("disk_used_pct_worst", "disk", "fullest disk (space or inodes)"))
|
|
|
|
|
|
def agent_findings(hosts, agent: dict, thresholds: dict,
|
|
silent: dict[str, list[str]] | None = None,
|
|
speaking: set[str] | frozenset = frozenset()) -> list[dict]:
|
|
"""Hosts past a CPU, memory or disk cutoff, and agents that stopped
|
|
reporting. A stale agent whose "agent offline" alert is active (`speaking`)
|
|
is that alert's finding, not a second one (#5721)."""
|
|
out = []
|
|
silent = silent or {}
|
|
for h in hosts:
|
|
a = agent.get(h.name)
|
|
if not a:
|
|
continue
|
|
ids = {"host_id": h.id}
|
|
if not a.get("fresh"):
|
|
if h.name not in speaking:
|
|
out.append(finding(
|
|
"warning", "agent_stale", h.name,
|
|
"host agent has stopped reporting; its metrics are out of date"
|
|
+ _also_silent(silent.get(h.name)),
|
|
since=a.get("last_at"), ids=ids, next=f'get_host(host="{h.id}")'))
|
|
continue # stale values would only add noise
|
|
# CPU and memory within the host's own normal for the hour aren't
|
|
# findings (#5742); the fullest disk keeps its cutoff.
|
|
normals = host_svc.host_normals(a.get("norms"), thresholds)
|
|
for metric, kind, label in _AGENT_CHECKS:
|
|
value = a.get(metric)
|
|
th = thresholds.get(kind) or {}
|
|
if value is None or th.get("warn") is None:
|
|
continue
|
|
if th.get("crit") is not None and value >= th["crit"]:
|
|
sev = "critical"
|
|
elif value >= th["warn"]:
|
|
sev = "warning"
|
|
else:
|
|
continue
|
|
if kind in normals and not anom.against(value, normals[kind][0], normals[kind][1], "%")[0]:
|
|
continue
|
|
out.append(finding(
|
|
sev, f"host_{kind}_high", h.name,
|
|
f"{label} at {rnd(value, 1)}% (warn {th['warn']}%, crit {th.get('crit')}%)",
|
|
ids=ids, next=f'get_host_metrics(host="{h.id}", metric="{metric}")'))
|
|
return out
|
|
|
|
|
|
def _run_key(r) -> tuple:
|
|
return (r.source_name, r.playbook_path, r.inventory_scope)
|
|
|
|
|
|
def _dry(r) -> bool:
|
|
return bool((r.params or {}).get("check"))
|
|
|
|
|
|
def still_failing(failed, succeeded) -> list:
|
|
"""The failed runs nothing has since put right: a later success of the
|
|
same playbook from the same source on the same hosts clears one, as long
|
|
as it was at least as real (a dry run's success clears only a dry run's
|
|
failure). `succeeded` may be a one-pass result, so it is read once."""
|
|
succeeded = list(succeeded)
|
|
return [f for f in failed if not any(
|
|
_run_key(s) == _run_key(f) and s.started_at > f.started_at and (_dry(f) or not _dry(s))
|
|
for s in succeeded)]
|
|
|
|
|
|
def run_findings(runs) -> list[dict]:
|
|
out = []
|
|
for r in runs:
|
|
failures = ((r.results or {}).get("failures") or [])[:3]
|
|
detail = f": {'; '.join(str(f)[:200] for f in failures)}" if failures else ""
|
|
out.append(finding(
|
|
"warning", f"ansible_run_{r.status.value}", r.playbook_path,
|
|
f"Ansible run of {r.playbook_path} ({r.source_name}) {r.status.value}{detail}",
|
|
since=r.finished_at or r.started_at, ids={"run_id": r.id}))
|
|
return out
|
|
|
|
|
|
def anomaly_findings(open_now) -> list[dict]:
|
|
"""Metric anomalies going on now (#5740): a warning each."""
|
|
from steward.core.formatting import metric_label, metric_value
|
|
out = []
|
|
for a in open_now:
|
|
out.append(finding(
|
|
"warning", "metric_anomaly", f"{a.resource_name} {metric_label(a.metric_name)}",
|
|
f"Metric anomaly: {metric_label(a.metric_name)} on {a.resource_name} held "
|
|
f"{metric_value(a.metric_name, a.peak_level)} where "
|
|
f"{metric_value(a.metric_name, a.usual_hold)} is usual for that hour"
|
|
+ (f" ({a.occurrences} times since {iso(a.started_at)})" if a.occurrences > 1 else ""),
|
|
since=a.last_started_at, ids={"anomaly_id": a.id},
|
|
next=f'list_anomalies(review="{a.review}")'))
|
|
return out
|
|
|
|
|
|
def backup_findings(settings: dict, running: bool, now) -> list[dict]:
|
|
"""Steward's own backups (#5836), while scheduled ones are on: the bucket
|
|
isn't set up, the last one failed (since the first of the failures in a
|
|
row), or none has been made since the day's slot."""
|
|
from steward.core.formatting import zone
|
|
if running or not settings.get("backup.enabled"):
|
|
return []
|
|
subject, nxt = "Steward backups", "list_backups()"
|
|
if backup_job.target_from(settings) is None:
|
|
return [finding("warning", "backup_not_set_up", subject,
|
|
"Scheduled backups are on, but the bucket isn't set up, so none are made",
|
|
next=nxt)]
|
|
history = settings.get("backup.history") or []
|
|
last_ok = backup_job._moment(settings.get("backup.last_success_at"))
|
|
good = (f"the newest good one was sent {iso(last_ok)}" if last_ok
|
|
else "none has been sent yet")
|
|
failing = []
|
|
for entry in history:
|
|
if entry.get("ok"):
|
|
break
|
|
failing.append(entry)
|
|
if failing:
|
|
tries = f" ({len(failing)} tries in a row)" if len(failing) > 1 else ""
|
|
return [finding("warning", "backup_failed", subject,
|
|
f"The last backup failed{tries}: {failing[0].get('error') or 'no reason recorded'}; {good}",
|
|
since=backup_job._moment(failing[-1].get("at")), next=nxt)]
|
|
slot = backup_job.last_slot(now, int(settings.get("backup.schedule_hour", 3)),
|
|
zone(settings.get("backup.schedule_tz")))
|
|
if history and (last_ok is None or last_ok < slot) and now - slot >= BACKUP_OVERDUE_AFTER:
|
|
return [finding("warning", "backup_overdue", subject,
|
|
f"No backup has been made since the scheduled one at {iso(slot)}; {good}",
|
|
since=slot, next=nxt)]
|
|
return []
|
|
|
|
|
|
async def _plugin_findings(server) -> tuple[list[dict], list[str]]:
|
|
found, unavailable = [], []
|
|
for plugin, source in sorted((getattr(server, "overview_sources", None) or {}).items()):
|
|
if not server.plugin_enabled(plugin):
|
|
continue
|
|
try:
|
|
async with db_session() as db:
|
|
found.extend(await source(db))
|
|
except Exception:
|
|
logger.exception("MCP overview source of plugin %r failed", plugin)
|
|
unavailable.append(plugin)
|
|
return found, unavailable
|
|
|
|
|
|
async def get_overview() -> dict:
|
|
"""What is wrong right now, across everything Steward watches. Start here.
|
|
|
|
Returns `findings` ranked critical first, then longest-standing first:
|
|
failing monitors (with the error and since when), firing, pending and
|
|
still-breaching acknowledged alerts, TLS certificates near expiry, host
|
|
agents that stopped reporting, hosts over their CPU, memory or disk
|
|
limits (all cutoffs are set under Settings → Thresholds), Ansible runs that failed
|
|
in the last 24h, metrics going on now well above what is normal for them
|
|
at this hour (metric anomalies), Steward's own scheduled backup failing or
|
|
overdue, and anything enabled plugins add
|
|
(unhealthy containers, for one). Each finding carries the ids to follow
|
|
and `next`, the call that shows its evidence.
|
|
|
|
`counts` sizes the fleet; `counts.anomalies_to_review` is how many metric
|
|
anomalies wait for someone to mark them normal or real (list_anomalies).
|
|
`active_maintenance` names windows currently silencing alerts.
|
|
`all_clear` is true when nothing is wrong.
|
|
"""
|
|
app = mcp_app()
|
|
now = now_utc()
|
|
async with db_session() as db:
|
|
hosts = list((await db.execute(select(Host).order_by(Host.name))).scalars())
|
|
host_names = {h.id: h.name for h in hosts}
|
|
monitors = list((await db.execute(select(Monitor).order_by(Monitor.name))).scalars())
|
|
states = await monitor_svc.monitor_states(db, [m for m in monitors if m.enabled])
|
|
active = await alert_svc.active_alerts(db)
|
|
agent = await host_svc.agent_overview_by_host(
|
|
db, [h.name for h in hosts], host_svc.agent_stale_after_seconds(app.config))
|
|
runs = list((await db.execute(
|
|
select(AnsibleRun)
|
|
.where(AnsibleRun.status.in_((AnsibleRunStatus.failed, AnsibleRunStatus.interrupted)),
|
|
AnsibleRun.started_at >= now - timedelta(hours=24))
|
|
.order_by(AnsibleRun.started_at.desc())
|
|
.limit(20)
|
|
)).scalars())
|
|
if runs:
|
|
runs = still_failing(runs, (await db.execute(
|
|
select(AnsibleRun)
|
|
.where(AnsibleRun.status == AnsibleRunStatus.success,
|
|
AnsibleRun.started_at > min(r.started_at for r in runs),
|
|
AnsibleRun.playbook_path.in_({r.playbook_path for r in runs}))
|
|
)).scalars())
|
|
windows = await alert_svc.list_maintenance_windows(db)
|
|
open_anomalies = await anom.open_anomalies(db)
|
|
to_review = await anom.to_review_count(db)
|
|
settings = await get_all_settings(db)
|
|
|
|
thresholds = app.config.get("THRESHOLDS") or {}
|
|
listed, silent, speaking = fold_agent_silence(active, agent)
|
|
findings = (monitor_findings(monitors, states, host_names, now, thresholds.get("tls_days"))
|
|
+ alert_findings(listed, silent)
|
|
+ agent_findings(hosts, agent, thresholds, silent, speaking)
|
|
+ run_findings(runs)
|
|
+ anomaly_findings(open_anomalies)
|
|
+ backup_findings(settings, backup_job.is_running(), now))
|
|
plugin_found, unavailable = await _plugin_findings(getattr(app, "mcp_server", None))
|
|
findings = rank(findings + plugin_found)
|
|
|
|
down = sum(1 for m in monitors if m.enabled and (st := states.get(m.id))
|
|
and st["latest"] is not None and not st["latest"].is_up)
|
|
out = {
|
|
"generated_at": iso(now),
|
|
"all_clear": not findings,
|
|
"counts": {
|
|
"hosts": len(hosts),
|
|
"monitors": len(monitors),
|
|
"monitors_down": down,
|
|
"alerts_active": len(active),
|
|
"critical": sum(1 for f in findings if f["severity"] == "critical"),
|
|
"warning": sum(1 for f in findings if f["severity"] == "warning"),
|
|
"anomalies_to_review": to_review,
|
|
},
|
|
"active_maintenance": [
|
|
{"window_id": w.id, "name": w.name, "end_at": iso(w.end_at),
|
|
"scope": {"source_module": w.scope_source, "resource": w.scope_resource}}
|
|
for w in windows if w.start_at <= now <= w.end_at
|
|
],
|
|
"findings": findings[:FINDINGS_CAP],
|
|
"truncated": len(findings) > FINDINGS_CAP,
|
|
}
|
|
if out["truncated"]:
|
|
out["more"] = (f"{len(findings) - FINDINGS_CAP} lower-ranked findings left out; "
|
|
"list_monitors(state=\"down\"), list_alerts and list_hosts show everything.")
|
|
if unavailable:
|
|
out["unavailable"] = [f"{p} plugin's findings could not be read (see Steward's log)"
|
|
for p in unavailable]
|
|
return out
|
|
|
|
|
|
def register(register_tool) -> None:
|
|
register_tool(get_overview, role=UserRole.viewer, read_only=True)
|