bvandeusenandClaude Opus 5.5
c7ff5bc556
#5756 "The agent says when its host rebooted and when it lost contact, so downtime is recorded as what it was". Agent 1.14.0 posts events with the next post that gets through: started (with the host's boot time), stopping (on SIGTERM, with whether systemd says the host is going down) and lost_contact (from its first unanswered post to the one carrying it). Steward keeps them as the host's downtime (host_downtime): reboot, agent_stop, unclean (a boot after the agent was last heard that nobody announced) or lost_contact. Retried posts apply once. An announced stop explains the silence for reboot_grace_seconds (Settings -> Plugins -> host_agent, 10 minutes): seconds_since_seen reads 0 until then, and the host's other rules go quiet with it (agents_away beside #5721's registry). Past the grace, agent offline fires. The host's Agent card lists the last 7 days of downtime. Rows are pruned at data.retention_days. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
104 lines
4.5 KiB
Python
104 lines
4.5 KiB
Python
"""Pure-function tests for the host_agent seconds_since_seen reconciler."""
|
|
from dataclasses import dataclass
|
|
from datetime import datetime, timedelta, timezone
|
|
|
|
from plugins.host_agent.scheduler import seconds_since_seen
|
|
|
|
|
|
@dataclass
|
|
class FakeReg:
|
|
host_id: str
|
|
last_seen_at: datetime | None
|
|
created_at: datetime | None
|
|
|
|
|
|
NOW = datetime(2026, 4, 15, 12, 0, 0, tzinfo=timezone.utc)
|
|
NAMES = {"h1": "nas", "h2": "web", "h3": "db"}
|
|
|
|
|
|
def test_seconds_since_each_agent_last_reported():
|
|
regs = [FakeReg("h1", NOW - timedelta(seconds=30), NOW - timedelta(days=9)),
|
|
FakeReg("h2", NOW - timedelta(minutes=10), NOW - timedelta(days=9))]
|
|
assert seconds_since_seen(regs, NAMES, NOW) == [("nas", 30.0), ("web", 600.0)]
|
|
|
|
|
|
def test_an_agent_that_never_reported_counts_from_registration():
|
|
regs = [FakeReg("h3", None, NOW - timedelta(minutes=5))]
|
|
assert seconds_since_seen(regs, NAMES, NOW) == [("db", 300.0)]
|
|
|
|
|
|
def test_a_registration_without_its_host_is_skipped():
|
|
regs = [FakeReg("gone", NOW, NOW)]
|
|
assert seconds_since_seen(regs, NAMES, NOW) == []
|
|
|
|
|
|
def test_a_clock_ahead_of_ours_reads_as_just_seen():
|
|
regs = [FakeReg("h1", NOW + timedelta(seconds=5), NOW)]
|
|
assert seconds_since_seen(regs, NAMES, NOW) == [("nas", 0.0)]
|
|
|
|
|
|
def test_empty_input():
|
|
assert seconds_since_seen([], NAMES, NOW) == []
|
|
|
|
|
|
def test_steward_s_own_outage_is_not_the_agent_s_silence():
|
|
"""#5441: agents that last reported 10 minutes ago, while Steward has only
|
|
been back for a minute, are a minute old; an agent heard since then keeps
|
|
its own age."""
|
|
back = NOW - timedelta(minutes=1)
|
|
regs = [FakeReg("h1", NOW - timedelta(minutes=10), NOW - timedelta(days=9)),
|
|
FakeReg("h2", NOW - timedelta(seconds=20), NOW - timedelta(days=9)),
|
|
FakeReg("h3", None, NOW - timedelta(days=9))]
|
|
assert seconds_since_seen(regs, NAMES, NOW, back) == [("nas", 60.0), ("web", 20.0), ("db", 60.0)]
|
|
|
|
|
|
def test_a_post_that_arrived_but_was_not_stored_counts_as_seen():
|
|
"""#5514: Steward's storage stalled and dropped the agents' posts, so
|
|
last_seen_at (written with them) stayed old, and agents posting every few
|
|
seconds read as offline. Arriving is the agent's part; the later of the
|
|
two counts. An agent with no arrival keeps its stored age."""
|
|
regs = [FakeReg("h1", NOW - timedelta(minutes=4), NOW - timedelta(days=9)),
|
|
FakeReg("h2", NOW - timedelta(minutes=4), NOW - timedelta(days=9)),
|
|
FakeReg("h3", NOW - timedelta(seconds=10), NOW - timedelta(days=9))]
|
|
arrived = {"h1": NOW - timedelta(seconds=5), "h3": NOW - timedelta(minutes=2)}
|
|
assert seconds_since_seen(regs, NAMES, NOW, arrived=arrived) == [
|
|
("nas", 5.0), ("web", 240.0), ("db", 10.0)]
|
|
|
|
|
|
def test_arrivals_are_kept_per_app():
|
|
from types import SimpleNamespace
|
|
from plugins.host_agent.scheduler import arrivals
|
|
app, other = SimpleNamespace(), SimpleNamespace()
|
|
arrivals(app)["h1"] = NOW
|
|
assert arrivals(app) == {"h1": NOW} and arrivals(other) == {}
|
|
|
|
|
|
def test_an_agent_already_offline_stays_offline_through_a_restart():
|
|
"""#5719: VDNT-Docker03's agent was silent for hours when Steward
|
|
restarted, and counting it from the restart read it as just seen: its
|
|
alert resolved, then fired again three minutes later, two emails a
|
|
restart. An agent already past the offline window keeps its whole
|
|
silence; one that was fine when Steward stopped still counts from the
|
|
restart (#5441)."""
|
|
back = NOW - timedelta(minutes=1)
|
|
regs = [FakeReg("h1", NOW - timedelta(hours=5), NOW - timedelta(days=9)),
|
|
FakeReg("h2", NOW - timedelta(minutes=10), NOW - timedelta(days=9))]
|
|
assert seconds_since_seen(regs, NAMES, NOW, back, already_silent={"nas"}) == [
|
|
("nas", 5 * 3600.0), ("web", 60.0)]
|
|
|
|
|
|
def test_an_offline_agent_that_reports_after_the_restart_is_back():
|
|
back = NOW - timedelta(minutes=1)
|
|
regs = [FakeReg("h1", NOW - timedelta(seconds=15), NOW - timedelta(days=9))]
|
|
assert seconds_since_seen(regs, NAMES, NOW, back, already_silent={"nas"}) == [("nas", 15.0)]
|
|
|
|
|
|
def test_a_reboot_the_agent_announced_reads_as_just_seen_until_its_grace_ends():
|
|
"""#5756: an agent that said its host was rebooting isn't offline while
|
|
the reboot's grace lasts; another host silent as long still is."""
|
|
regs = [FakeReg("h1", NOW - timedelta(minutes=5), NOW - timedelta(days=9)),
|
|
FakeReg("h2", NOW - timedelta(minutes=5), NOW - timedelta(days=9))]
|
|
explained = {"h1": NOW + timedelta(minutes=5)}
|
|
assert seconds_since_seen(regs, NAMES, NOW, explained_until=explained) == [
|
|
("nas", 0.0), ("web", 300.0)]
|