bvandeusenandClaude Opus 5.5
400e91a0d8
At 20:25-20:39Z the NFS storage under steward_db slowed badly (checkpoint sync 4.6 s, then 70.7 s with one file at 21.9 s, against the usual 0.002 s). Ingest commits stalled past the agent's 10 s timeout and were dropped, and last_seen_at, written in that same transaction, went with them. Agents posting every few seconds read as silent for 3 minutes, and "Agent offline" fired and emailed for all three hosts. Ingest now notes each agent's arrival in memory right after the (read-only) registration lookup, before any write; seconds_since_seen counts from the later of last_seen_at and that arrival. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
74 lines
3.0 KiB
Python
74 lines
3.0 KiB
Python
"""Pure-function tests for the host_agent seconds_since_seen reconciler."""
|
|
from dataclasses import dataclass
|
|
from datetime import datetime, timedelta, timezone
|
|
|
|
from plugins.host_agent.scheduler import seconds_since_seen
|
|
|
|
|
|
@dataclass
|
|
class FakeReg:
|
|
host_id: str
|
|
last_seen_at: datetime | None
|
|
created_at: datetime | None
|
|
|
|
|
|
NOW = datetime(2026, 4, 15, 12, 0, 0, tzinfo=timezone.utc)
|
|
NAMES = {"h1": "nas", "h2": "web", "h3": "db"}
|
|
|
|
|
|
def test_seconds_since_each_agent_last_reported():
|
|
regs = [FakeReg("h1", NOW - timedelta(seconds=30), NOW - timedelta(days=9)),
|
|
FakeReg("h2", NOW - timedelta(minutes=10), NOW - timedelta(days=9))]
|
|
assert seconds_since_seen(regs, NAMES, NOW) == [("nas", 30.0), ("web", 600.0)]
|
|
|
|
|
|
def test_an_agent_that_never_reported_counts_from_registration():
|
|
regs = [FakeReg("h3", None, NOW - timedelta(minutes=5))]
|
|
assert seconds_since_seen(regs, NAMES, NOW) == [("db", 300.0)]
|
|
|
|
|
|
def test_a_registration_without_its_host_is_skipped():
|
|
regs = [FakeReg("gone", NOW, NOW)]
|
|
assert seconds_since_seen(regs, NAMES, NOW) == []
|
|
|
|
|
|
def test_a_clock_ahead_of_ours_reads_as_just_seen():
|
|
regs = [FakeReg("h1", NOW + timedelta(seconds=5), NOW)]
|
|
assert seconds_since_seen(regs, NAMES, NOW) == [("nas", 0.0)]
|
|
|
|
|
|
def test_empty_input():
|
|
assert seconds_since_seen([], NAMES, NOW) == []
|
|
|
|
|
|
def test_steward_s_own_outage_is_not_the_agent_s_silence():
|
|
"""#5441: agents that last reported 10 minutes ago, while Steward has only
|
|
been back for a minute, are a minute old; an agent heard since then keeps
|
|
its own age."""
|
|
back = NOW - timedelta(minutes=1)
|
|
regs = [FakeReg("h1", NOW - timedelta(minutes=10), NOW - timedelta(days=9)),
|
|
FakeReg("h2", NOW - timedelta(seconds=20), NOW - timedelta(days=9)),
|
|
FakeReg("h3", None, NOW - timedelta(days=9))]
|
|
assert seconds_since_seen(regs, NAMES, NOW, back) == [("nas", 60.0), ("web", 20.0), ("db", 60.0)]
|
|
|
|
|
|
def test_a_post_that_arrived_but_was_not_stored_counts_as_seen():
|
|
"""#5514: Steward's storage stalled and dropped the agents' posts, so
|
|
last_seen_at (written with them) stayed old, and agents posting every few
|
|
seconds read as offline. Arriving is the agent's part; the later of the
|
|
two counts. An agent with no arrival keeps its stored age."""
|
|
regs = [FakeReg("h1", NOW - timedelta(minutes=4), NOW - timedelta(days=9)),
|
|
FakeReg("h2", NOW - timedelta(minutes=4), NOW - timedelta(days=9)),
|
|
FakeReg("h3", NOW - timedelta(seconds=10), NOW - timedelta(days=9))]
|
|
arrived = {"h1": NOW - timedelta(seconds=5), "h3": NOW - timedelta(minutes=2)}
|
|
assert seconds_since_seen(regs, NAMES, NOW, arrived=arrived) == [
|
|
("nas", 5.0), ("web", 240.0), ("db", 10.0)]
|
|
|
|
|
|
def test_arrivals_are_kept_per_app():
|
|
from types import SimpleNamespace
|
|
from plugins.host_agent.scheduler import arrivals
|
|
app, other = SimpleNamespace(), SimpleNamespace()
|
|
arrivals(app)["h1"] = NOW
|
|
assert arrivals(app) == {"h1": NOW} and arrivals(other) == {}
|