fix(alerts): one silent agent is one alert and one finding (#5721)
CI / lint (push) Successful in 2s
CI / integration (push) Successful in 1m19s
CI / unit (push) Successful in 1m29s
CI / publish (push) Successful in 1m8s

VDNT-Docker03's silent agent read as three overview findings (agent offline,
agent stale, the disk rule's no-data) and every drop emailed the disk rule's
silence and return on top of agent offline.

- The host agent registers its heartbeat (seconds_since_seen) and the
  sources it carries (host_agent, docker). While a host's agent is past its
  offline window, its other rules' no_data is recorded "while its agent is
  offline" and not emailed, nor is their quiet return; a return past a
  threshold still fires.
- The overview folds those rules under the host's agent-offline finding
  ("also silent with it: ..."), and a stale agent whose agent-offline alert
  is active is that alert's finding, not a second one.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
2026-10-14 16:49:29 -04:00
co-authored by Claude Opus 5.5
parent 76d75961f8
commit d0c563abd0
6 changed files with 239 additions and 12 deletions
+67
View File
@@ -189,3 +189,70 @@ def test_a_series_quiet_only_while_steward_was_down_is_not_silent(app):
with patch.object(alerts, "_dispatch_notification", fake_dispatch):
asyncio.run(_go())
def test_a_rule_silent_with_its_agent_says_nothing_of_its_own(app):
"""#5721: while a host's agent is offline, "agent offline" is the one
email. The host's other rules going silent is recorded, marked as with
its agent, and not emailed; their quiet return isn't either, but a
return past the threshold still fires."""
from sqlalchemy import select
from steward.alerts import service as alert_svc
from steward.core import alerts
from steward.core.watch import watch
from steward.models.alerts import AlertEvent, AlertState
from steward.models.metrics import PluginMetric, PluginMetricLatest
from steward.models.users import User, UserRole
alerts.agent_heartbeat("host_agent", "seconds_since_seen", lambda: 180.0)
alerts.agent_fed("host_agent", lambda resource: resource.split(":", 1)[0])
sent = []
async def fake_dispatch(app_, notif_type, rule, value, event_id, note=None):
sent.append((rule.id, notif_type))
async def _go():
tag = uuid.uuid4().hex[:8]
host = f"gone-{tag}"
now = datetime.now(timezone.utc)
async with app.db_sessionmaker() as s:
async with s.begin():
user = User(username=f"ag-{tag}", email=f"ag-{tag}@test", password_hash="x",
role=UserRole.admin)
s.add(user)
await alerts.record_metrics(s, [PluginMetric(
source_module="host_agent", resource_name=f"{host}:/", metric_name="disk_used_pct",
value=50.0, recorded_at=now - timedelta(minutes=20))])
await s.flush()
fields = await alert_svc.rule_fields(
s, name=f"disk {tag}", source_module="host_agent", resource_name=f"{host}:/",
metric_name="disk_used_pct", operator=">", threshold=90, notify_no_data=True)
rule_id = (await alert_svc.add_rule(s, fields, created_by=user.id)).id
await s.flush()
st = (await s.execute(select(AlertState).where(AlertState.rule_id == rule_id))).scalar_one()
st.last_sample_at, st.sample_interval_s = now - timedelta(minutes=20), 60.0
# The agent has been quiet for 20 minutes: offline.
await s.merge(PluginMetricLatest(source_module="host_agent", resource_name=host,
metric_name="seconds_since_seen", value=1200.0,
recorded_at=now))
w = watch(app, "alert_silence")
w.since, w.last = now - timedelta(days=1), datetime.now(timezone.utc)
await alerts.check_silence(app)
await asyncio.sleep(0)
async with app.db_sessionmaker() as s:
async with s.begin():
[note] = (await s.execute(select(AlertEvent.note).where(
AlertEvent.rule_id == rule_id, AlertEvent.to_state == "no_data"))).scalars().all()
assert alerts.AGENT_OFFLINE in note
assert sent == [] # agent offline is the email
# The agent comes back and the disk is fine: back quietly.
async with app.db_sessionmaker() as s:
async with s.begin():
await alerts.record_metric(s, "host_agent", f"{host}:/", "disk_used_pct", 50.0)
await asyncio.sleep(0)
assert sent == []
with patch.object(alerts, "_dispatch_notification", fake_dispatch):
asyncio.run(_go())