feat: monitors, UniFi and Traefik keep collecting through a storage stall (#5995)

The same Holding as the SNMP poll, for the rest of Steward's own
collectors:

- Monitors: the list is read within 10 s or the last one read stands in;
  a start the database doesn't take is kept in memory, so a monitor isn't
  checked twice; each result is held and stored at the time it was
  checked. A stall shows on the monitors as the checks it saw, not a gap.
- UniFi and Traefik: each poll's writes are held (hold_write) and run
  oldest first in one transaction; every reading is stamped with its scrape.
- A held round the database refuses (as opposed to can't be reached) is
  stored alone and let go, so it can't hold up every round after it.

The Prometheus scrape is left as it is: its one target, Fabled Archives,
keeps its own database on the same NFS.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

steward/core/holding.py

@@ -80,13 +80,42 @@ class Holding:
                              "held (the database may still finish it)", self.name, len(batch),
                              self.deadline_s)
                 return
-            except Exception:
-                logger.exception("%s: storing %d held round(s) failed; they stay held",
+            except Exception as exc:
+                if unreachable(exc):
+                    logger.warning("%s: the database can't be reached (%s); %d round(s) stay held",
+                                   self.name, exc, len(batch))
+                    return
+                # A round the database refuses would otherwise be retried, and
+                # refused, ahead of every round after it: store each on its own
+                # and let go of the ones refused.
+                logger.exception("%s: storing %d held round(s) failed; storing each alone",
                                  self.name, len(batch))
-                return
-            last = batch[-1][0]
-            while self._held and self._held[0][0] <= last:
-                self._held.popleft()
+                for seq, item in batch:
+                    try:
+                        await asyncio.wait_for(self.store([item]), self.deadline_s)
+                    except Exception as one:
+                        if isinstance(one, TimeoutError) or unreachable(one):
+                            return
+                        logger.error("%s: a round collected was refused and is dropped: %s",
+                                     self.name, one)
+                    self._drop_through(seq)
+                continue
+            self._drop_through(batch[-1][0])
+
+    def _drop_through(self, seq: int) -> None:
+        while self._held and self._held[0][0] <= seq:
+            self._held.popleft()
+
+
+def unreachable(exc: BaseException) -> bool:
+    """Whether a store failed because the database couldn't be reached, as
+    opposed to refusing what it was given."""
+    from sqlalchemy.exc import DBAPIError, InterfaceError, OperationalError
+
+    from steward.database import DatabaseUnavailable
+    if isinstance(exc, (OSError, DatabaseUnavailable, InterfaceError, OperationalError)):
+        return True
+    return isinstance(exc, DBAPIError) and bool(exc.connection_invalidated)
 
 
 def holding(app, name: str, store: Callable[[list], Awaitable[None]], limit: int) -> Holding:
@@ -97,3 +126,21 @@ def holding(app, name: str, store: Callable[[list], Awaitable[None]], limit: int
     if name not in holdings:
         holdings[name] = Holding(name, store, limit)
     return holdings[name]
+
+
+async def hold_write(app, name: str, write: Callable[[object], Awaitable[None]],
+                     limit: int = 120, wait_s: float = 20.0) -> bool:
+    """Hold one collected round as `write(session)` and store it: the held
+    writes run oldest first, in one transaction. For a collector whose round
+    is easiest said as the writes it makes. True when it was stored within
+    `wait_s`."""
+    held = holding(app, name, store=lambda writes: _run_writes(app, writes), limit=limit)
+    held.hold(write)
+    return await held.flush(wait_s)
+
+
+async def _run_writes(app, writes: list) -> None:
+    async with app.db_sessionmaker() as session:
+        async with session.begin():
+            for write in writes:
+                await write(session)