bvandeusenandClaude Opus 5.5
41fc8073db
On 2026-10-17 the NFS under Postgres stalled for ~15 minutes. Two things went wrong for the readings that would have explained it: - The SNMP poll stored before it polled again, so while the database hung it stopped polling. It now collects on schedule and hands each round to a Holding (steward/core/holding.py): one writer stores what is held, oldest first, each reading at the time it was collected. Bounded at two hours of polls, with a 30-minute deadline on a store; the device list read falls back to the last one read after 10 s. - "Mount not answering" (10 checks in a row) went pending on Docker02 and Docker03 and never fired: the agent sent each post's backlog with its newest, healthy sample, and that sample ended every post's run. Agent 1.22.0 queues the new sample behind the backlog, so Steward receives a host's samples in the order they were taken. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
65 lines
2.2 KiB
Python
65 lines
2.2 KiB
Python
"""Collected rounds held in memory until the database takes them (#5995):
|
|
stored oldest first, one writer at a time, bounded, and kept through a
|
|
store that fails or never answers."""
|
|
import asyncio
|
|
from types import SimpleNamespace as NS
|
|
|
|
from steward.core.holding import Holding, holding
|
|
|
|
|
|
def test_rounds_held_while_the_store_hangs_are_stored_in_order_once_it_answers():
|
|
async def main():
|
|
gate, stored = asyncio.Event(), []
|
|
|
|
async def store(rounds):
|
|
await gate.wait()
|
|
stored.append(list(rounds))
|
|
|
|
held = Holding("t", store, limit=10)
|
|
held.hold("a")
|
|
assert await held.flush(0.05) is False # the store hangs: the caller goes on
|
|
held.hold("b")
|
|
assert await held.flush(0.05) is False and len(held) == 2
|
|
gate.set()
|
|
assert await held.flush(1.0) is True and len(held) == 0
|
|
return stored
|
|
|
|
assert asyncio.run(main()) == [["a"], ["b"]] # "b" waited for the store in flight
|
|
|
|
|
|
def test_past_its_limit_the_oldest_round_goes_and_is_counted():
|
|
held = Holding("t", store=None, limit=2)
|
|
for r in ("a", "b", "c"):
|
|
held.hold(r)
|
|
assert len(held) == 2 and held.dropped == 1 and [i for _, i in held._held] == ["b", "c"]
|
|
|
|
|
|
def test_a_store_that_fails_or_overruns_its_deadline_keeps_the_rounds_held():
|
|
async def main():
|
|
calls = []
|
|
|
|
async def failing(rounds):
|
|
calls.append(list(rounds))
|
|
raise RuntimeError("database refused")
|
|
|
|
held = Holding("t", failing, limit=10)
|
|
held.hold("a")
|
|
assert await held.flush(1.0) is False and len(held) == 1
|
|
|
|
async def never(rounds):
|
|
await asyncio.sleep(3600)
|
|
|
|
stuck = Holding("t", never, limit=10, deadline_s=0.05)
|
|
stuck.hold("a")
|
|
assert await stuck.flush(1.0) is False and len(stuck) == 1 # given up, not lost
|
|
return calls
|
|
|
|
assert asyncio.run(main()) == [["a"]]
|
|
|
|
|
|
def test_an_app_has_one_holding_per_collector():
|
|
app = NS()
|
|
first = holding(app, "SNMP poll", store=None, limit=5)
|
|
assert holding(app, "SNMP poll", store=None, limit=5) is first
|
|
assert holding(app, "UniFi poll", store=None, limit=5) is not first
|