fix(alerts): one silent agent is one alert and one finding (#5721)
VDNT-Docker03's silent agent read as three overview findings (agent offline,
agent stale, the disk rule's no-data) and every drop emailed the disk rule's
silence and return on top of agent offline.
- The host agent registers its heartbeat (seconds_since_seen) and the
sources it carries (host_agent, docker). While a host's agent is past its
offline window, its other rules' no_data is recorded "while its agent is
offline" and not emailed, nor is their quiet return; a return past a
threshold still fires.
- The overview folds those rules under the host's agent-offline finding
("also silent with it: ..."), and a stale agent whose agent-offline alert
is active is that alert's finding, not a second one.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -189,3 +189,70 @@ def test_a_series_quiet_only_while_steward_was_down_is_not_silent(app):
|
||||
with patch.object(alerts, "_dispatch_notification", fake_dispatch):
|
||||
asyncio.run(_go())
|
||||
|
||||
|
||||
|
||||
def test_a_rule_silent_with_its_agent_says_nothing_of_its_own(app):
|
||||
"""#5721: while a host's agent is offline, "agent offline" is the one
|
||||
email. The host's other rules going silent is recorded, marked as with
|
||||
its agent, and not emailed; their quiet return isn't either, but a
|
||||
return past the threshold still fires."""
|
||||
from sqlalchemy import select
|
||||
from steward.alerts import service as alert_svc
|
||||
from steward.core import alerts
|
||||
from steward.core.watch import watch
|
||||
from steward.models.alerts import AlertEvent, AlertState
|
||||
from steward.models.metrics import PluginMetric, PluginMetricLatest
|
||||
from steward.models.users import User, UserRole
|
||||
|
||||
alerts.agent_heartbeat("host_agent", "seconds_since_seen", lambda: 180.0)
|
||||
alerts.agent_fed("host_agent", lambda resource: resource.split(":", 1)[0])
|
||||
sent = []
|
||||
|
||||
async def fake_dispatch(app_, notif_type, rule, value, event_id, note=None):
|
||||
sent.append((rule.id, notif_type))
|
||||
|
||||
async def _go():
|
||||
tag = uuid.uuid4().hex[:8]
|
||||
host = f"gone-{tag}"
|
||||
now = datetime.now(timezone.utc)
|
||||
async with app.db_sessionmaker() as s:
|
||||
async with s.begin():
|
||||
user = User(username=f"ag-{tag}", email=f"ag-{tag}@test", password_hash="x",
|
||||
role=UserRole.admin)
|
||||
s.add(user)
|
||||
await alerts.record_metrics(s, [PluginMetric(
|
||||
source_module="host_agent", resource_name=f"{host}:/", metric_name="disk_used_pct",
|
||||
value=50.0, recorded_at=now - timedelta(minutes=20))])
|
||||
await s.flush()
|
||||
fields = await alert_svc.rule_fields(
|
||||
s, name=f"disk {tag}", source_module="host_agent", resource_name=f"{host}:/",
|
||||
metric_name="disk_used_pct", operator=">", threshold=90, notify_no_data=True)
|
||||
rule_id = (await alert_svc.add_rule(s, fields, created_by=user.id)).id
|
||||
await s.flush()
|
||||
st = (await s.execute(select(AlertState).where(AlertState.rule_id == rule_id))).scalar_one()
|
||||
st.last_sample_at, st.sample_interval_s = now - timedelta(minutes=20), 60.0
|
||||
# The agent has been quiet for 20 minutes: offline.
|
||||
await s.merge(PluginMetricLatest(source_module="host_agent", resource_name=host,
|
||||
metric_name="seconds_since_seen", value=1200.0,
|
||||
recorded_at=now))
|
||||
|
||||
w = watch(app, "alert_silence")
|
||||
w.since, w.last = now - timedelta(days=1), datetime.now(timezone.utc)
|
||||
await alerts.check_silence(app)
|
||||
await asyncio.sleep(0)
|
||||
async with app.db_sessionmaker() as s:
|
||||
async with s.begin():
|
||||
[note] = (await s.execute(select(AlertEvent.note).where(
|
||||
AlertEvent.rule_id == rule_id, AlertEvent.to_state == "no_data"))).scalars().all()
|
||||
assert alerts.AGENT_OFFLINE in note
|
||||
assert sent == [] # agent offline is the email
|
||||
|
||||
# The agent comes back and the disk is fine: back quietly.
|
||||
async with app.db_sessionmaker() as s:
|
||||
async with s.begin():
|
||||
await alerts.record_metric(s, "host_agent", f"{host}:/", "disk_used_pct", 50.0)
|
||||
await asyncio.sleep(0)
|
||||
assert sent == []
|
||||
|
||||
with patch.object(alerts, "_dispatch_notification", fake_dispatch):
|
||||
asyncio.run(_go())
|
||||
|
||||
Reference in New Issue
Block a user