feat: readings kept through a storage stall; an agent's backlog arrives in order (#5995)

On 2026-10-17 the NFS under Postgres stalled for ~15 minutes. Two things
went wrong for the readings that would have explained it:

- The SNMP poll stored before it polled again, so while the database hung
  it stopped polling. It now collects on schedule and hands each round to a
  Holding (steward/core/holding.py): one writer stores what is held, oldest
  first, each reading at the time it was collected. Bounded at two hours of
  polls, with a 30-minute deadline on a store; the device list read falls
  back to the last one read after 10 s.
- "Mount not answering" (10 checks in a row) went pending on Docker02 and
  Docker03 and never fired: the agent sent each post's backlog with its
  newest, healthy sample, and that sample ended every post's run. Agent
  1.22.0 queues the new sample behind the backlog, so Steward receives a
  host's samples in the order they were taken.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

steward/core/holding.py

@@ -0,0 +1,99 @@
+"""Collected readings held in memory until the database takes them (#5995).
+
+Steward's Postgres sits on the NFS storage it watches. When that storage
+stalls, every write waits, and a collector that stores what it found before
+collecting again stops collecting for as long as the stall lasts: on
+2026-10-17 the NAS readings went blind for the fifteen minutes they mattered.
+
+A collector built on a Holding collects on schedule whatever the database is
+doing, and hands each round to the holding. One writer at a time stores what
+is held, oldest first, each item at the time it was collected, so a stall
+delays the readings instead of losing them, and alert rules count them in the
+order they happened.
+
+    held = holding(app, "SNMP poll", store=_store, limit=120)
+    held.hold(round)                     # one collected round
+    await held.flush(wait_s=20)          # store it, or leave it to the writer
+
+The holding is bounded: past `limit` rounds the oldest is dropped, and that
+is logged with how many have gone. A store has `deadline_s`: generous, past
+any stall seen so far, so a connection that will never answer is given up
+and the rounds stay held for a fresh one rather than waiting forever.
+"""
+from __future__ import annotations
+
+import asyncio
+import logging
+from collections import deque
+from typing import Awaitable, Callable
+
+logger = logging.getLogger(__name__)
+
+
+class Holding:
+    def __init__(self, name: str, store: Callable[[list], Awaitable[None]], limit: int,
+                 deadline_s: float = 1800.0) -> None:
+        self.name = name
+        self.store = store
+        self.limit = limit
+        self.deadline_s = deadline_s
+        self._held: deque[tuple[int, object]] = deque()
+        self._seq = 0
+        self._writer: asyncio.Task | None = None
+        self.dropped = 0
+
+    def __len__(self) -> int:
+        return len(self._held)
+
+    def hold(self, item) -> None:
+        """Hold one collected round until it is stored."""
+        self._seq += 1
+        self._held.append((self._seq, item))
+        while len(self._held) > self.limit:
+            self._held.popleft()
+            self.dropped += 1
+            logger.warning("%s: the database hasn't taken a round in %d; dropped the oldest "
+                           "(%d dropped so far)", self.name, self.limit, self.dropped)
+
+    async def flush(self, wait_s: float) -> bool:
+        """Start the writer if it isn't running, and wait up to `wait_s` for
+        it. True when nothing is left held. A writer still waiting on the
+        database is left to finish: the next round joins what it stores next."""
+        writer = self._writer
+        if writer is None or writer.done() or writer.get_loop() is not asyncio.get_running_loop():
+            if not self._held:
+                return True
+            writer = self._writer = asyncio.create_task(self._write())
+        done, _ = await asyncio.wait({writer}, timeout=wait_s)
+        if not done:
+            logger.warning("%s: the database hasn't taken %d held round(s) in %.0f s; "
+                           "collecting on", self.name, len(self._held), wait_s)
+        return not self._held
+
+    async def _write(self) -> None:
+        while self._held:
+            batch = list(self._held)
+            try:
+                await asyncio.wait_for(self.store([item for _, item in batch]), self.deadline_s)
+            except TimeoutError:
+                logger.error("%s: storing %d held round(s) took over %.0f s; given up, they stay "
+                             "held (the database may still finish it)", self.name, len(batch),
+                             self.deadline_s)
+                return
+            except Exception:
+                logger.exception("%s: storing %d held round(s) failed; they stay held",
+                                 self.name, len(batch))
+                return
+            last = batch[-1][0]
+            while self._held and self._held[0][0] <= last:
+                self._held.popleft()
+
+
+def holding(app, name: str, store: Callable[[list], Awaitable[None]], limit: int) -> Holding:
+    """The app's Holding for one collector, made on first use."""
+    holdings = getattr(app, "_holdings", None)
+    if holdings is None:
+        holdings = app._holdings = {}
+    if name not in holdings:
+        holdings[name] = Holding(name, store, limit)
+    return holdings[name]