feat: readings kept through a storage stall; an agent's backlog arrives in order (#5995)

On 2026-10-17 the NFS under Postgres stalled for ~15 minutes. Two things
went wrong for the readings that would have explained it:

- The SNMP poll stored before it polled again, so while the database hung
  it stopped polling. It now collects on schedule and hands each round to a
  Holding (steward/core/holding.py): one writer stores what is held, oldest
  first, each reading at the time it was collected. Bounded at two hours of
  polls, with a 30-minute deadline on a store; the device list read falls
  back to the last one read after 10 s.
- "Mount not answering" (10 checks in a row) went pending on Docker02 and
  Docker03 and never fired: the agent sent each post's backlog with its
  newest, healthy sample, and that sample ended every post's run. Agent
  1.22.0 queues the new sample behind the backlog, so Steward receives a
  host's samples in the order they were taken.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

tests/plugins/host_agent/test_agent_build_payload.py

@@ -120,11 +120,11 @@ def test_backoff_schedule_stays_inside_the_stale_window():
     assert next_backoff(60) == 60    # capped below Steward's 180 s stale window (#5154)
 
 
-def test_a_backlog_drains_a_few_samples_a_post_oldest_first():
+def test_a_backlog_drains_a_few_samples_a_post_oldest_first_and_the_new_one_waits_its_turn():
     from plugins.host_agent.agent import MAX_SAMPLES_PER_POST, split_backlog
-    buffered = list(range(20))
-    send, held = split_backlog(buffered)
-    assert send == [0, 1, 2, 3] and held == list(range(4, 20))
-    assert len(send) + 1 == MAX_SAMPLES_PER_POST    # room for the new sample
-    assert split_backlog([7]) == ([7], [])
-    assert split_backlog([]) == ([], [])
+    queued = list(range(20)) + ["new"]
+    send, held = split_backlog(queued)
+    assert send == [0, 1, 2, 3, 4] and len(send) == MAX_SAMPLES_PER_POST
+    assert held == list(range(5, 20)) + ["new"]        # never ahead of an older sample
+    assert split_backlog(["new"]) == (["new"], [])
+    assert split_backlog([7, "new"]) == ([7, "new"], [])

tests/plugins/snmp/test_scheduler.py

@@ -0,0 +1,55 @@
+"""An SNMP poll collects on schedule while the database hangs, and what it
+held is stored oldest first, each at its own time, once it answers (#5995)."""
+import asyncio
+from contextlib import asynccontextmanager
+from types import SimpleNamespace as NS
+
+import plugins.snmp.scheduler as scheduler
+import plugins.snmp.service as service
+from plugins.snmp.poller import Credentials, PollResult
+
+
+def _device():
+    return NS(id="d1", name="nas", address="10.0.0.5", port=161, version="2c", presets=["system"],
+              interfaces="", custom=None, tables={}, tables_at=None, last_error=None, texts={})
+
+
+def test_polls_go_on_while_the_database_hangs_and_are_stored_in_order_after(monkeypatch):
+    async def main():
+        stalled = asyncio.Event()
+
+        @asynccontextmanager
+        async def hung_session():
+            await stalled.wait()                         # the storage under Postgres has stalled
+            yield None
+
+        app = NS(db_sessionmaker=hung_session)
+        gate, stored = asyncio.Event(), []
+
+        async def store(app_, rounds):
+            await gate.wait()
+            stored.extend(rounds)
+
+        uptimes = iter([100.0, 200.0])
+
+        async def poll(address, port, creds, oids):
+            return PollResult(readings={"uptime": next(uptimes)})
+
+        monkeypatch.setattr(scheduler, "DEVICES", [_device()])
+        monkeypatch.setattr(scheduler, "DEVICES_READ_S", 0.05)
+        monkeypatch.setattr(scheduler, "STORE_WAIT_S", 0.05)
+        monkeypatch.setattr(scheduler, "_store", store)
+        monkeypatch.setattr(service, "credentials", lambda d: Credentials(community="public"))
+        scheduler.RATES = None
+
+        await scheduler._do_poll(app, poll=poll)         # the device list is the last one read
+        await asyncio.sleep(0.01)
+        await scheduler._do_poll(app, poll=poll)         # the first store hasn't returned: polled anyway
+        gate.set()
+        await app._holdings["SNMP poll"].flush(1.0)
+        return stored
+
+    rounds = asyncio.run(main())
+    assert [[(p.name, p.values) for p in r] for r in rounds] == [
+        [("nas", [("uptime", 100.0)])], [("nas", [("uptime", 200.0)])]]
+    assert rounds[0][0].at < rounds[1][0].at              # each stored at the time it was collected