bvandeusenandClaude Opus 5.5
4ded8ecee4
A redeploy's new process ticks at once, so its once-a-minute samples never broke for 90 s and get_anomaly showed none of the day's restarts. The first self-monitor tick in a process now records restart_gap_secs (time since the last process's last sample), and steward_gaps shows each one as a restart. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
260 lines
12 KiB
Python
260 lines
12 KiB
Python
"""What else was going on: events on a host, or one resource, in a window
|
|
(#5837), so an anomaly comes with its circumstances.
|
|
|
|
Core gathers alert transitions, Ansible runs, other metric anomalies and
|
|
Steward's own gaps in recording. A plugin adds its own through
|
|
`mcp_events(db, host_id, host_name, since, until)` in its __init__.py (the
|
|
host agent its maintenance jobs and downtime, Docker its container events),
|
|
and for a resource with no host through
|
|
`mcp_resource_events(db, source, resource, since, until)` (Traefik: the
|
|
containers of the swarm service behind a router, #5862); each returns a list
|
|
of `event(...)`.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import logging
|
|
from dataclasses import dataclass
|
|
from datetime import datetime, timedelta
|
|
|
|
from sqlalchemy import and_, or_, select
|
|
|
|
from steward.alerts.service import watches_host
|
|
from steward.core.alerts import host_of_resource
|
|
from steward.core.formatting import human_duration
|
|
from steward.mcp._context import mcp_app
|
|
from steward.models.alerts import AlertEvent, AlertRule
|
|
from steward.models.ansible import AnsibleRun
|
|
from steward.models.ansible_inventory import AnsibleTarget
|
|
from steward.models.hosts import Host
|
|
from steward.models.metrics import PluginMetric
|
|
from steward.models.monitors import Monitor
|
|
from steward.models.norms import MetricAnomaly, MetricAnomalySpan
|
|
|
|
from ._shape import db_session, iso
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
PER_SOURCE = 40 # events any one source contributes to one window: the
|
|
# nearest the occurrence's start, the rest counted (#5861)
|
|
# Steward samples its own health once a minute (core/self_monitor.py); a
|
|
# longer break between two samples is a stretch it wasn't recording.
|
|
STEWARD_GAP_S = 90.0
|
|
|
|
|
|
def event(at: datetime | None, kind: str, what: str, *, until: datetime | None = None,
|
|
subject: str | None = None, ids: dict | None = None) -> dict:
|
|
"""One thing that happened. `until` for something that lasted (a run, a
|
|
reboot); `subject` names what it happened to when that isn't the host."""
|
|
return {"at": iso(at), "until": iso(until), "kind": kind, "what": what,
|
|
"subject": subject, "ids": ids or {}}
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class Place:
|
|
"""Where a metric's resource lives: its host, when it has one."""
|
|
source: str
|
|
resource: str
|
|
host_id: str | None = None
|
|
host_name: str | None = None
|
|
|
|
|
|
async def place_of(db, source: str, resource: str) -> Place:
|
|
"""The host behind a resource: an agent-fed series names it ("nas:/",
|
|
"nas/web"), a monitor may be pinned to one; a Traefik router has none."""
|
|
name = host_of_resource(source, resource)
|
|
host = None
|
|
if name:
|
|
host = (await db.execute(select(Host).where(Host.name == name).limit(1))).scalar_one_or_none()
|
|
else:
|
|
monitor = (await db.execute(select(Monitor).where(
|
|
Monitor.type == source, Monitor.name == resource).limit(1))).scalar_one_or_none()
|
|
if monitor is not None and monitor.host_id:
|
|
host = await db.get(Host, monitor.host_id)
|
|
return Place(source, resource, host.id if host else None, host.name if host else None)
|
|
|
|
|
|
def _overlaps(start, end):
|
|
"""A row lasting from `start` to `end` (None: still going) overlaps the window."""
|
|
def clause(since, until):
|
|
return and_(start <= until, or_(end.is_(None), end >= since))
|
|
return clause
|
|
|
|
|
|
async def alert_events(db, place: Place, since: datetime, until: datetime) -> list[dict]:
|
|
"""Alert transitions on the resource itself, or anything its host
|
|
carries: its agent series, its containers, its monitors."""
|
|
monitor_keys = set()
|
|
if place.host_id:
|
|
monitor_keys = {(t, n) for t, n in (await db.execute(
|
|
select(Monitor.type, Monitor.name).where(Monitor.host_id == place.host_id))).all()}
|
|
rows = (await db.execute(
|
|
select(AlertEvent, AlertRule).join(AlertRule, AlertRule.id == AlertEvent.rule_id)
|
|
.where(AlertEvent.transitioned_at >= since, AlertEvent.transitioned_at <= until)
|
|
.order_by(AlertEvent.transitioned_at).limit(500))).all()
|
|
out = []
|
|
for ev, rule in rows:
|
|
mine = (rule.source_module, rule.resource_name) == (place.source, place.resource)
|
|
if not mine and not (place.host_name and watches_host(rule, place.host_name, monitor_keys)):
|
|
continue
|
|
note = f" ({ev.note})" if ev.note else ""
|
|
out.append(event(ev.transitioned_at, "alert", f"{rule.name}: {ev.from_state} → {ev.to_state}{note}",
|
|
subject=rule.resource_name, ids={"rule_id": rule.id}))
|
|
return out
|
|
|
|
|
|
async def ansible_runs(db, place: Place, since: datetime, until: datetime) -> list[dict]:
|
|
"""Ansible runs that touched the host while the window was open (by the
|
|
hosts in their recap: a run still going hasn't said yet)."""
|
|
if not place.host_id:
|
|
return []
|
|
names = {place.host_name} | set((await db.execute(
|
|
select(AnsibleTarget.name).where(AnsibleTarget.host_id == place.host_id))).scalars())
|
|
runs = (await db.execute(
|
|
select(AnsibleRun).where(_overlaps(AnsibleRun.started_at, AnsibleRun.finished_at)(since, until))
|
|
.order_by(AnsibleRun.started_at).limit(200))).scalars()
|
|
out = []
|
|
for r in runs:
|
|
if not names & set(((r.results or {}).get("hosts") or {}).keys()):
|
|
continue
|
|
check = " in check mode" if (r.params or {}).get("check") else ""
|
|
out.append(event(r.started_at, "ansible_run",
|
|
f"Ansible {r.playbook_path}{check}: {r.status.value}",
|
|
until=r.finished_at, ids={"run_id": r.id}))
|
|
return out
|
|
|
|
|
|
def _same_host(source: str, resource: str, place: Place) -> bool:
|
|
if (source, resource) == (place.source, place.resource):
|
|
return True
|
|
return bool(place.host_name) and host_of_resource(source, resource) == place.host_name
|
|
|
|
|
|
async def other_anomalies(db, place: Place, anomaly_id: str, since: datetime,
|
|
until: datetime) -> tuple[list[dict], int]:
|
|
"""Other anomalies going on in the window: those on the same host as
|
|
events, and how many there were anywhere else (many at once points at
|
|
something shared: the network, storage, Steward itself)."""
|
|
rows = (await db.execute(
|
|
select(MetricAnomalySpan, MetricAnomaly)
|
|
.join(MetricAnomaly, MetricAnomaly.id == MetricAnomalySpan.anomaly_id)
|
|
.where(MetricAnomaly.id != anomaly_id,
|
|
_overlaps(MetricAnomalySpan.started_at, MetricAnomalySpan.ended_at)(since, until))
|
|
.order_by(MetricAnomalySpan.started_at).limit(500))).all()
|
|
here, elsewhere = [], 0
|
|
for span, a in rows:
|
|
if _same_host(a.source_module, a.resource_name, place):
|
|
here.append(event(span.started_at, "anomaly",
|
|
f"{a.resource_name} {a.metric_name} anomaly (peak {round(span.peak_level, 3)}, "
|
|
f"usual {round(a.usual_hold, 3)}; {a.review})",
|
|
until=span.ended_at, subject=a.resource_name, ids={"anomaly_id": a.id}))
|
|
else:
|
|
elsewhere += 1
|
|
return here, elsewhere
|
|
|
|
|
|
def gaps_in(times: list[datetime], since: datetime, until: datetime,
|
|
restarts: list[tuple[datetime, float]] = ()) -> list[dict]:
|
|
"""Each restart (its first sample's time and how long since the last
|
|
one) and each other break longer than STEWARD_GAP_S between consecutive
|
|
samples that overlaps the window, as a "steward" event lasting the break.
|
|
A restart is shown however short: a quick redeploy never breaks the
|
|
minute, and still restarts every count Steward keeps."""
|
|
out = []
|
|
for at, secs in restarts:
|
|
if at >= since and at - timedelta(seconds=secs) <= until:
|
|
out.append(event(at - timedelta(seconds=secs), "steward",
|
|
f"Steward restarted, not recording for {human_duration(secs)}: "
|
|
"readings either side of it can be wrong", until=at))
|
|
restarted = {at for at, _ in restarts}
|
|
for before, after in zip(times, times[1:]):
|
|
secs = (after - before).total_seconds()
|
|
if secs > STEWARD_GAP_S and after >= since and before <= until and after not in restarted:
|
|
out.append(event(before, "steward",
|
|
f"Steward wasn't recording for {human_duration(secs)} (a restart, a deploy or a "
|
|
"stall): readings either side of a gap can be wrong", until=after))
|
|
return out
|
|
|
|
|
|
async def steward_gaps(db, since: datetime, until: datetime) -> list[dict]:
|
|
"""When Steward itself wasn't recording, read from its own once-a-minute
|
|
health sample. Shown beside every anomaly, placed or not: a restart
|
|
distorts whatever Steward measures, a Traefik counter as much as a host's
|
|
silence (#5862). Raw samples only, so a window older than their
|
|
retention shows none."""
|
|
rows = (await db.execute(
|
|
select(PluginMetric.metric_name, PluginMetric.recorded_at, PluginMetric.value).where(
|
|
PluginMetric.source_module == "steward", PluginMetric.resource_name == "process",
|
|
PluginMetric.metric_name.in_(("db_roundtrip_ms", "restart_gap_secs")),
|
|
PluginMetric.recorded_at >= since - timedelta(hours=1), PluginMetric.recorded_at <= until)
|
|
.order_by(PluginMetric.recorded_at))).all()
|
|
return gaps_in([at for name, at, _ in rows if name == "db_roundtrip_ms"], since, until,
|
|
[(at, value) for name, at, value in rows if name == "restart_gap_secs"])
|
|
|
|
|
|
async def plugin_events(place: Place, since: datetime, until: datetime) -> list[list[dict]]:
|
|
"""What enabled plugins know happened on the host (`mcp_events`), one
|
|
list per plugin, each read in its own session so one that fails can't
|
|
spoil the others' reads."""
|
|
server = getattr(mcp_app(), "mcp_server", None)
|
|
if server is None or not place.host_id:
|
|
return []
|
|
out = []
|
|
for plugin, source in sorted((getattr(server, "event_sources", None) or {}).items()):
|
|
if not server.plugin_enabled(plugin):
|
|
continue
|
|
try:
|
|
async with db_session() as db:
|
|
out.append(await source(db, place.host_id, place.host_name, since, until))
|
|
except Exception:
|
|
logger.exception("MCP events source of plugin %r failed", plugin)
|
|
return out
|
|
|
|
|
|
async def resource_events(place: Place, since: datetime, until: datetime) -> list[list[dict]]:
|
|
"""What enabled plugins know happened to the resource itself
|
|
(`mcp_resource_events`), for one with no host to read events by: one
|
|
list per plugin."""
|
|
server = getattr(mcp_app(), "mcp_server", None)
|
|
if server is None:
|
|
return []
|
|
out = []
|
|
for plugin, source in sorted((getattr(server, "resource_event_sources", None) or {}).items()):
|
|
if not server.plugin_enabled(plugin):
|
|
continue
|
|
try:
|
|
async with db_session() as db:
|
|
out.append(await source(db, place.source, place.resource, since, until))
|
|
except Exception:
|
|
logger.exception("MCP resource events source of plugin %r failed", plugin)
|
|
return out
|
|
|
|
|
|
def _at(e: dict) -> datetime | None:
|
|
return datetime.fromisoformat(e["at"].replace("Z", "+00:00")) if e.get("at") else None
|
|
|
|
|
|
def nearest(events: list[dict], focus: datetime, keep: int = PER_SOURCE) -> tuple[list[dict], int]:
|
|
"""The `keep` events nearest `focus` (an occurrence's start), and how
|
|
many were left out: a busy host's window is cut where it matters least."""
|
|
if len(events) <= keep:
|
|
return events, 0
|
|
by_distance = sorted(events, key=lambda e: abs((_at(e) - focus).total_seconds()) if _at(e) else float("inf"))
|
|
return by_distance[:keep], len(events) - keep
|
|
|
|
|
|
async def around(db, place: Place, anomaly_id: str, since: datetime, until: datetime,
|
|
focus: datetime) -> tuple[list[dict], int, int]:
|
|
"""Everything above for one window, oldest first; the count of anomalies
|
|
elsewhere; and how many events were left out, each source keeping its
|
|
PER_SOURCE nearest `focus`."""
|
|
anomalies, elsewhere = await other_anomalies(db, place, anomaly_id, since, until)
|
|
groups = [await alert_events(db, place, since, until), await ansible_runs(db, place, since, until),
|
|
anomalies, *await plugin_events(place, since, until),
|
|
*await resource_events(place, since, until), await steward_gaps(db, since, until)]
|
|
found, left_out = [], 0
|
|
for group in groups:
|
|
kept, cut = nearest(group, focus)
|
|
found += kept
|
|
left_out += cut
|
|
return sorted(found, key=lambda e: e["at"] or ""), elsewhere, left_out
|