fix(alerts): one silent agent is one alert and one finding (#5721)
VDNT-Docker03's silent agent read as three overview findings (agent offline,
agent stale, the disk rule's no-data) and every drop emailed the disk rule's
silence and return on top of agent offline.
- The host agent registers its heartbeat (seconds_since_seen) and the
sources it carries (host_agent, docker). While a host's agent is past its
offline window, its other rules' no_data is recorded "while its agent is
offline" and not emailed, nor is their quiet return; a return past a
threshold still fires.
- The overview folds those rules under the host's agent-offline finding
("also silent with it: ..."), and a stale agent whose agent-offline alert
is active is that alert's finding, not a second one.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -283,3 +283,37 @@ def test_get_host_groups_parts_by_kind_and_summarises_cores():
|
||||
assert got["temperatures"]["items"][0] == {"sensor": "Package id 0", "part": "temp:Package id 0",
|
||||
"temp_c": 61.0}
|
||||
assert "other" not in got
|
||||
|
||||
|
||||
def test_one_silent_agent_is_one_finding():
|
||||
"""#5721: VDNT-Docker03's silent agent read as three findings. Its agent-
|
||||
offline alert speaks for it, the stale-agent finding goes, and its rules
|
||||
that only went quiet are listed under it; one still firing stays listed."""
|
||||
from steward.core import alerts
|
||||
alerts.agent_heartbeat("host_agent", "seconds_since_seen", lambda: 180.0)
|
||||
alerts.agent_fed("host_agent", lambda resource: resource.split(":", 1)[0])
|
||||
|
||||
def rule(name, metric, resource):
|
||||
return NS(id=name, name=name, metric_name=metric, operator=AlertOperator.gt, threshold=90.0,
|
||||
source_module="host_agent", resource_name=resource)
|
||||
|
||||
gone = NOW - timedelta(hours=2)
|
||||
active = [
|
||||
(NS(state=NS(value="firing"), last_transition_at=gone, no_data_since=None),
|
||||
rule("Agent offline: d3", "seconds_since_seen", "d3")),
|
||||
(NS(state=NS(value="inactive"), last_transition_at=gone, no_data_since=gone),
|
||||
rule("Disk nearly full: d3", "disk_used_pct", "d3:/")),
|
||||
(NS(state=NS(value="firing"), last_transition_at=gone, no_data_since=gone),
|
||||
rule("Memory high: d3", "mem_used_pct", "d3")),
|
||||
]
|
||||
agent = {"d3": {"fresh": False, "last_at": gone}}
|
||||
listed, silent, speaking = overview.fold_agent_silence(active, agent)
|
||||
assert [r.name for _s, r in listed] == ["Agent offline: d3", "Memory high: d3"]
|
||||
assert silent == {"d3": ["Disk nearly full: d3"]} and speaking == {"d3"}
|
||||
found = overview.alert_findings(listed, silent) + overview.agent_findings(
|
||||
[NS(id="h3", name="d3")], agent, THRESHOLDS, silent, speaking)
|
||||
assert [f["subject"] for f in found] == ["Agent offline: d3", "Memory high: d3"]
|
||||
assert "also silent with it: Disk nearly full: d3" in found[0]["summary"]
|
||||
# Without its alert, the stale agent is the finding, and says the same.
|
||||
[stale] = overview.agent_findings([NS(id="h3", name="d3")], agent, THRESHOLDS, silent)
|
||||
assert stale["kind"] == "agent_stale" and "Disk nearly full: d3" in stale["summary"]
|
||||
|
||||
Reference in New Issue
Block a user