bvandeusenandClaude Opus 5.5
264d44ac81
Counting every agent's silence from when Steward started watching again (#5441) read an agent that was already gone as just seen: its agent-offline alert resolved at 0 s and fired again three minutes later, two false emails per restart (VDNT-Docker03, four times in a day). An agent whose last recorded seconds_since_seen was already past the offline window now keeps counting its whole silence; one that was fine when Steward stopped still counts from the restart, so Steward's own downtime still fires nothing. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
94 lines
4.0 KiB
Python
94 lines
4.0 KiB
Python
"""Pure-function tests for the host_agent seconds_since_seen reconciler."""
|
|
from dataclasses import dataclass
|
|
from datetime import datetime, timedelta, timezone
|
|
|
|
from plugins.host_agent.scheduler import seconds_since_seen
|
|
|
|
|
|
@dataclass
|
|
class FakeReg:
|
|
host_id: str
|
|
last_seen_at: datetime | None
|
|
created_at: datetime | None
|
|
|
|
|
|
NOW = datetime(2026, 4, 15, 12, 0, 0, tzinfo=timezone.utc)
|
|
NAMES = {"h1": "nas", "h2": "web", "h3": "db"}
|
|
|
|
|
|
def test_seconds_since_each_agent_last_reported():
|
|
regs = [FakeReg("h1", NOW - timedelta(seconds=30), NOW - timedelta(days=9)),
|
|
FakeReg("h2", NOW - timedelta(minutes=10), NOW - timedelta(days=9))]
|
|
assert seconds_since_seen(regs, NAMES, NOW) == [("nas", 30.0), ("web", 600.0)]
|
|
|
|
|
|
def test_an_agent_that_never_reported_counts_from_registration():
|
|
regs = [FakeReg("h3", None, NOW - timedelta(minutes=5))]
|
|
assert seconds_since_seen(regs, NAMES, NOW) == [("db", 300.0)]
|
|
|
|
|
|
def test_a_registration_without_its_host_is_skipped():
|
|
regs = [FakeReg("gone", NOW, NOW)]
|
|
assert seconds_since_seen(regs, NAMES, NOW) == []
|
|
|
|
|
|
def test_a_clock_ahead_of_ours_reads_as_just_seen():
|
|
regs = [FakeReg("h1", NOW + timedelta(seconds=5), NOW)]
|
|
assert seconds_since_seen(regs, NAMES, NOW) == [("nas", 0.0)]
|
|
|
|
|
|
def test_empty_input():
|
|
assert seconds_since_seen([], NAMES, NOW) == []
|
|
|
|
|
|
def test_steward_s_own_outage_is_not_the_agent_s_silence():
|
|
"""#5441: agents that last reported 10 minutes ago, while Steward has only
|
|
been back for a minute, are a minute old; an agent heard since then keeps
|
|
its own age."""
|
|
back = NOW - timedelta(minutes=1)
|
|
regs = [FakeReg("h1", NOW - timedelta(minutes=10), NOW - timedelta(days=9)),
|
|
FakeReg("h2", NOW - timedelta(seconds=20), NOW - timedelta(days=9)),
|
|
FakeReg("h3", None, NOW - timedelta(days=9))]
|
|
assert seconds_since_seen(regs, NAMES, NOW, back) == [("nas", 60.0), ("web", 20.0), ("db", 60.0)]
|
|
|
|
|
|
def test_a_post_that_arrived_but_was_not_stored_counts_as_seen():
|
|
"""#5514: Steward's storage stalled and dropped the agents' posts, so
|
|
last_seen_at (written with them) stayed old, and agents posting every few
|
|
seconds read as offline. Arriving is the agent's part; the later of the
|
|
two counts. An agent with no arrival keeps its stored age."""
|
|
regs = [FakeReg("h1", NOW - timedelta(minutes=4), NOW - timedelta(days=9)),
|
|
FakeReg("h2", NOW - timedelta(minutes=4), NOW - timedelta(days=9)),
|
|
FakeReg("h3", NOW - timedelta(seconds=10), NOW - timedelta(days=9))]
|
|
arrived = {"h1": NOW - timedelta(seconds=5), "h3": NOW - timedelta(minutes=2)}
|
|
assert seconds_since_seen(regs, NAMES, NOW, arrived=arrived) == [
|
|
("nas", 5.0), ("web", 240.0), ("db", 10.0)]
|
|
|
|
|
|
def test_arrivals_are_kept_per_app():
|
|
from types import SimpleNamespace
|
|
from plugins.host_agent.scheduler import arrivals
|
|
app, other = SimpleNamespace(), SimpleNamespace()
|
|
arrivals(app)["h1"] = NOW
|
|
assert arrivals(app) == {"h1": NOW} and arrivals(other) == {}
|
|
|
|
|
|
def test_an_agent_already_offline_stays_offline_through_a_restart():
|
|
"""#5719: VDNT-Docker03's agent was silent for hours when Steward
|
|
restarted, and counting it from the restart read it as just seen: its
|
|
alert resolved, then fired again three minutes later, two emails a
|
|
restart. An agent already past the offline window keeps its whole
|
|
silence; one that was fine when Steward stopped still counts from the
|
|
restart (#5441)."""
|
|
back = NOW - timedelta(minutes=1)
|
|
regs = [FakeReg("h1", NOW - timedelta(hours=5), NOW - timedelta(days=9)),
|
|
FakeReg("h2", NOW - timedelta(minutes=10), NOW - timedelta(days=9))]
|
|
assert seconds_since_seen(regs, NAMES, NOW, back, already_silent={"nas"}) == [
|
|
("nas", 5 * 3600.0), ("web", 60.0)]
|
|
|
|
|
|
def test_an_offline_agent_that_reports_after_the_restart_is_back():
|
|
back = NOW - timedelta(minutes=1)
|
|
regs = [FakeReg("h1", NOW - timedelta(seconds=15), NOW - timedelta(days=9))]
|
|
assert seconds_since_seen(regs, NAMES, NOW, back, already_silent={"nas"}) == [("nas", 15.0)]
|