bvandeusenandClaude Opus 5.5
c368e93e2c
Agent offline is an ordinary rule now. The host_agent scheduler records host_agent/<host>/seconds_since_seen every minute (from last_seen_at, or registration time for an agent that never reported), replacing the task that only logged stale agents. The catalog offers the metric, and picking it on a new rule fills "> 180". No data: each rule's state tracks its last sample and its usual gap, smoothed so that an outage counts as at most three gaps. A core task, alert_silence_check, runs every 60s and marks a rule no_data_since once its series has had no sample for three usual gaps (at least 5 minutes; an hour while the cadence is unknown). A never-sampled rule counts from when it started watching. The mark sits beside the state, so a firing alert that goes silent stays firing. Each silence records one event and sends one notification (rule option notify_no_data, default on; muted in maintenance windows). The next sample clears it with a "data resumed" event and notification. Creating, changing, disabling or re-enabling a rule restarts its watch. Migration 0029 adds the columns and seeds each existing rule's last sample and gap from its last day of data, so an upgrade does not flag every rule. Shown on the Alerts page (a NO DATA badge with its start time), in MCP list_alerts, list_alert_rules and get_host (state "no_data", no_data_since), in get_overview findings (alert_no_data, or a firing alert marked silent) and in the weekly report. The rule form has the notify option, and its metric reference now matches what traefik records and lists host_agent. Emails say "no data" and carry the event's note. The metric-catalog guard also reads positional MetricBatch.add arguments. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
126 lines
5.8 KiB
Python
126 lines
5.8 KiB
Python
"""Integration (#5062): against real Postgres, a rule whose series stops is
|
|
marked no_data once, notified once, and cleared by the next sample; a firing
|
|
alert that goes silent stays firing."""
|
|
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import os
|
|
import uuid
|
|
from datetime import datetime, timedelta, timezone
|
|
from unittest.mock import patch
|
|
|
|
import pytest
|
|
|
|
pytestmark = pytest.mark.integration
|
|
|
|
|
|
@pytest.fixture
|
|
def app():
|
|
if not os.environ.get("STEWARD_DATABASE_URL"):
|
|
pytest.skip("needs Postgres")
|
|
from steward.app import create_app
|
|
return create_app(testing=False)
|
|
|
|
|
|
def test_silence_is_marked_once_and_cleared_by_data(app):
|
|
from sqlalchemy import select, update
|
|
from steward.alerts import service as alert_svc
|
|
from steward.core import alerts
|
|
from steward.models.alerts import AlertEvent, AlertRule, AlertState, AlertStateEnum
|
|
from steward.models.metrics import PluginMetric
|
|
from steward.models.users import User, UserRole
|
|
|
|
sent: list[tuple[str, str, str | None]] = []
|
|
|
|
async def fake_dispatch(app_, notif_type, rule, value, event_id, note=None):
|
|
sent.append((rule.name, notif_type, note))
|
|
|
|
async def _go():
|
|
tag = uuid.uuid4().hex[:8]
|
|
res = f"quiet-{tag}"
|
|
now = datetime.now(timezone.utc)
|
|
async with app.db_sessionmaker() as s:
|
|
async with s.begin():
|
|
user = User(username=f"nd-{tag}", email=f"nd-{tag}@test", password_hash="x",
|
|
role=UserRole.admin)
|
|
s.add(user)
|
|
s.add(PluginMetric(source_module="host_agent", resource_name=res,
|
|
metric_name="cpu_pct", value=1.0,
|
|
recorded_at=now - timedelta(hours=1)))
|
|
await s.flush()
|
|
ids = {}
|
|
for key, notify in (("loud", True), ("hushed", False)):
|
|
fields = await alert_svc.rule_fields(
|
|
s, name=f"{key} {tag}", source_module="host_agent", resource_name=res,
|
|
metric_name="cpu_pct", operator=">", threshold=90,
|
|
notify_no_data=notify)
|
|
ids[key] = (await alert_svc.add_rule(s, fields, created_by=user.id)).id
|
|
# Every minute, from 20 to 18 minutes ago; then nothing.
|
|
batch = alerts.MetricBatch()
|
|
for ago in (20, 19, 18):
|
|
batch.add(source_module="host_agent", resource_name=res, metric_name="cpu_pct",
|
|
value=10.0, recorded_at=now - timedelta(minutes=ago))
|
|
await batch.record(s)
|
|
|
|
async def state(key):
|
|
async with app.db_sessionmaker() as s:
|
|
return (await s.execute(select(AlertState).where(
|
|
AlertState.rule_id == ids[key]))).scalar_one()
|
|
|
|
async def events(key):
|
|
async with app.db_sessionmaker() as s:
|
|
rows = (await s.execute(select(AlertEvent).where(AlertEvent.rule_id == ids[key])
|
|
.order_by(AlertEvent.transitioned_at))).scalars().all()
|
|
return [(e.from_state, e.to_state, e.note) for e in rows]
|
|
|
|
st = await state("loud")
|
|
assert st.sample_interval_s == pytest.approx(60)
|
|
assert st.last_sample_at == now - timedelta(minutes=18)
|
|
|
|
await alerts.check_silence(app)
|
|
await alerts.check_silence(app) # already marked: nothing more
|
|
await asyncio.sleep(0)
|
|
st = await state("loud")
|
|
assert st.state == AlertStateEnum.inactive
|
|
assert st.no_data_since == now - timedelta(minutes=18) + timedelta(seconds=300)
|
|
[(frm, to, note)] = await events("loud")
|
|
assert (frm, to) == ("inactive", "no_data")
|
|
assert note == "no sample for 18m (usually every 60s)"
|
|
assert len(await events("hushed")) == 1
|
|
assert sent == [(f"loud {tag}", "no_data", note)] # the hushed rule notifies nobody
|
|
|
|
async with app.db_sessionmaker() as s:
|
|
active = {r.id: alert_svc.alert_label(st_) for st_, r in await alert_svc.active_alerts(s)}
|
|
assert active[ids["loud"]] == active[ids["hushed"]] == "no_data"
|
|
|
|
# Data resumes, breaching: the mark clears and the rule fires.
|
|
async with app.db_sessionmaker() as s:
|
|
async with s.begin():
|
|
await alerts.record_metric(s, "host_agent", res, "cpu_pct", 95.0)
|
|
await asyncio.sleep(0)
|
|
st = await state("loud")
|
|
assert st.no_data_since is None and st.state == AlertStateEnum.firing
|
|
first, *same_instant = await events("loud")
|
|
assert first[:2] == ("inactive", "no_data")
|
|
# The resume and the firing it leads to share the sample's timestamp.
|
|
assert sorted(e[:2] for e in same_instant) == [("inactive", "firing"),
|
|
("no_data", "inactive")]
|
|
assert any((e[2] or "").startswith("data resumed after ") for e in same_instant)
|
|
assert [t for _, t, _ in sent if t != "firing"] == ["no_data", "data_resumed"]
|
|
|
|
# A firing alert that goes silent stays firing, marked.
|
|
async with app.db_sessionmaker() as s:
|
|
async with s.begin():
|
|
await s.execute(update(AlertState).where(AlertState.rule_id == ids["loud"])
|
|
.values(last_sample_at=now - timedelta(hours=2)))
|
|
await s.execute(update(AlertRule).where(AlertRule.id == ids["hushed"])
|
|
.values(enabled=False))
|
|
await alerts.check_silence(app)
|
|
st = await state("loud")
|
|
assert st.state == AlertStateEnum.firing and st.no_data_since is not None
|
|
assert (await events("loud"))[-1][:2] == ("firing", "no_data")
|
|
assert len(await events("hushed")) == 3 # disabled: not watched
|
|
|
|
with patch.object(alerts, "_dispatch_notification", fake_dispatch):
|
|
asyncio.run(_go())
|