feat: readings kept through a storage stall; an agent's backlog arrives in order (#5995)

On 2026-10-17 the NFS under Postgres stalled for ~15 minutes. Two things
went wrong for the readings that would have explained it:

- The SNMP poll stored before it polled again, so while the database hung
  it stopped polling. It now collects on schedule and hands each round to a
  Holding (steward/core/holding.py): one writer stores what is held, oldest
  first, each reading at the time it was collected. Bounded at two hours of
  polls, with a 30-minute deadline on a store; the device list read falls
  back to the last one read after 10 s.
- "Mount not answering" (10 checks in a row) went pending on Docker02 and
  Docker03 and never fired: the agent sent each post's backlog with its
  newest, healthy sample, and that sample ended every post's run. Agent
  1.22.0 queues the new sample behind the backlog, so Steward receives a
  host's samples in the order they were taken.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

tests/core/test_holding.py

@@ -0,0 +1,64 @@
+"""Collected rounds held in memory until the database takes them (#5995):
+stored oldest first, one writer at a time, bounded, and kept through a
+store that fails or never answers."""
+import asyncio
+from types import SimpleNamespace as NS
+
+from steward.core.holding import Holding, holding
+
+
+def test_rounds_held_while_the_store_hangs_are_stored_in_order_once_it_answers():
+    async def main():
+        gate, stored = asyncio.Event(), []
+
+        async def store(rounds):
+            await gate.wait()
+            stored.append(list(rounds))
+
+        held = Holding("t", store, limit=10)
+        held.hold("a")
+        assert await held.flush(0.05) is False           # the store hangs: the caller goes on
+        held.hold("b")
+        assert await held.flush(0.05) is False and len(held) == 2
+        gate.set()
+        assert await held.flush(1.0) is True and len(held) == 0
+        return stored
+
+    assert asyncio.run(main()) == [["a"], ["b"]]         # "b" waited for the store in flight
+
+
+def test_past_its_limit_the_oldest_round_goes_and_is_counted():
+    held = Holding("t", store=None, limit=2)
+    for r in ("a", "b", "c"):
+        held.hold(r)
+    assert len(held) == 2 and held.dropped == 1 and [i for _, i in held._held] == ["b", "c"]
+
+
+def test_a_store_that_fails_or_overruns_its_deadline_keeps_the_rounds_held():
+    async def main():
+        calls = []
+
+        async def failing(rounds):
+            calls.append(list(rounds))
+            raise RuntimeError("database refused")
+
+        held = Holding("t", failing, limit=10)
+        held.hold("a")
+        assert await held.flush(1.0) is False and len(held) == 1
+
+        async def never(rounds):
+            await asyncio.sleep(3600)
+
+        stuck = Holding("t", never, limit=10, deadline_s=0.05)
+        stuck.hold("a")
+        assert await stuck.flush(1.0) is False and len(stuck) == 1   # given up, not lost
+        return calls
+
+    assert asyncio.run(main()) == [["a"]]
+
+
+def test_an_app_has_one_holding_per_collector():
+    app = NS()
+    first = holding(app, "SNMP poll", store=None, limit=5)
+    assert holding(app, "SNMP poll", store=None, limit=5) is first
+    assert holding(app, "UniFi poll", store=None, limit=5) is not first