bvandeusenandClaude Opus 5.5
41fc8073db
On 2026-10-17 the NFS under Postgres stalled for ~15 minutes. Two things went wrong for the readings that would have explained it: - The SNMP poll stored before it polled again, so while the database hung it stopped polling. It now collects on schedule and hands each round to a Holding (steward/core/holding.py): one writer stores what is held, oldest first, each reading at the time it was collected. Bounded at two hours of polls, with a 30-minute deadline on a store; the device list read falls back to the last one read after 10 s. - "Mount not answering" (10 checks in a row) went pending on Docker02 and Docker03 and never fired: the agent sent each post's backlog with its newest, healthy sample, and that sample ended every post's run. Agent 1.22.0 queues the new sample behind the backlog, so Steward receives a host's samples in the order they were taken. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
56 lines
2.2 KiB
Python
56 lines
2.2 KiB
Python
"""An SNMP poll collects on schedule while the database hangs, and what it
|
|
held is stored oldest first, each at its own time, once it answers (#5995)."""
|
|
import asyncio
|
|
from contextlib import asynccontextmanager
|
|
from types import SimpleNamespace as NS
|
|
|
|
import plugins.snmp.scheduler as scheduler
|
|
import plugins.snmp.service as service
|
|
from plugins.snmp.poller import Credentials, PollResult
|
|
|
|
|
|
def _device():
|
|
return NS(id="d1", name="nas", address="10.0.0.5", port=161, version="2c", presets=["system"],
|
|
interfaces="", custom=None, tables={}, tables_at=None, last_error=None, texts={})
|
|
|
|
|
|
def test_polls_go_on_while_the_database_hangs_and_are_stored_in_order_after(monkeypatch):
|
|
async def main():
|
|
stalled = asyncio.Event()
|
|
|
|
@asynccontextmanager
|
|
async def hung_session():
|
|
await stalled.wait() # the storage under Postgres has stalled
|
|
yield None
|
|
|
|
app = NS(db_sessionmaker=hung_session)
|
|
gate, stored = asyncio.Event(), []
|
|
|
|
async def store(app_, rounds):
|
|
await gate.wait()
|
|
stored.extend(rounds)
|
|
|
|
uptimes = iter([100.0, 200.0])
|
|
|
|
async def poll(address, port, creds, oids):
|
|
return PollResult(readings={"uptime": next(uptimes)})
|
|
|
|
monkeypatch.setattr(scheduler, "DEVICES", [_device()])
|
|
monkeypatch.setattr(scheduler, "DEVICES_READ_S", 0.05)
|
|
monkeypatch.setattr(scheduler, "STORE_WAIT_S", 0.05)
|
|
monkeypatch.setattr(scheduler, "_store", store)
|
|
monkeypatch.setattr(service, "credentials", lambda d: Credentials(community="public"))
|
|
scheduler.RATES = None
|
|
|
|
await scheduler._do_poll(app, poll=poll) # the device list is the last one read
|
|
await asyncio.sleep(0.01)
|
|
await scheduler._do_poll(app, poll=poll) # the first store hasn't returned: polled anyway
|
|
gate.set()
|
|
await app._holdings["SNMP poll"].flush(1.0)
|
|
return stored
|
|
|
|
rounds = asyncio.run(main())
|
|
assert [[(p.name, p.values) for p in r] for r in rounds] == [
|
|
[("nas", [("uptime", 100.0)])], [("nas", [("uptime", 200.0)])]]
|
|
assert rounds[0][0].at < rounds[1][0].at # each stored at the time it was collected
|