feat: monitors, UniFi and Traefik keep collecting through a storage stall (#5995)

The same Holding as the SNMP poll, for the rest of Steward's own
collectors:

- Monitors: the list is read within 10 s or the last one read stands in;
  a start the database doesn't take is kept in memory, so a monitor isn't
  checked twice; each result is held and stored at the time it was
  checked. A stall shows on the monitors as the checks it saw, not a gap.
- UniFi and Traefik: each poll's writes are held (hold_write) and run
  oldest first in one transaction; every reading is stamped with its scrape.
- A held round the database refuses (as opposed to can't be reached) is
  stored alone and let go, so it can't hold up every round after it.

The Prometheus scrape is left as it is: its one target, Fabled Archives,
keeps its own database on the same NFS.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

tests/core/test_holding.py

@@ -34,15 +34,12 @@ def test_past_its_limit_the_oldest_round_goes_and_is_counted():
     assert len(held) == 2 and held.dropped == 1 and [i for _, i in held._held] == ["b", "c"]
 
 
-def test_a_store_that_fails_or_overruns_its_deadline_keeps_the_rounds_held():
+def test_a_store_that_cant_reach_the_database_or_overruns_its_deadline_keeps_the_rounds_held():
     async def main():
-        calls = []
+        async def unreachable(rounds):
+            raise ConnectionRefusedError("database down")
 
-        async def failing(rounds):
-            calls.append(list(rounds))
-            raise RuntimeError("database refused")
-
-        held = Holding("t", failing, limit=10)
+        held = Holding("t", unreachable, limit=10)
         held.hold("a")
         assert await held.flush(1.0) is False and len(held) == 1
 
@@ -52,9 +49,26 @@ def test_a_store_that_fails_or_overruns_its_deadline_keeps_the_rounds_held():
         stuck = Holding("t", never, limit=10, deadline_s=0.05)
         stuck.hold("a")
         assert await stuck.flush(1.0) is False and len(stuck) == 1   # given up, not lost
-        return calls
 
-    assert asyncio.run(main()) == [["a"]]
+    asyncio.run(main())
+
+
+def test_a_round_the_database_refuses_is_dropped_and_holds_up_nobody_after_it():
+    async def main():
+        stored = []
+
+        async def store(rounds):
+            if "bad" in rounds:
+                raise ValueError("value out of range")
+            stored.extend(rounds)
+
+        held = Holding("t", store, limit=10)
+        for r in ("a", "bad", "c"):
+            held.hold(r)
+        assert await held.flush(1.0) is True
+        return stored
+
+    assert asyncio.run(main()) == ["a", "c"]
 
 
 def test_an_app_has_one_holding_per_collector():

tests/core/test_monitor_schedule.py

@@ -152,3 +152,41 @@ async def test_nothing_due_stamps_nothing(monkeypatch):
     app = _app(store)
     await sched.run_due_monitors(app)
     assert store.stamped == [] and store.added == []
+
+
+class _HungSession(_Session):
+    """The storage under Postgres has stalled: nothing answers until `gate`."""
+
+    def __init__(self, store, gate):
+        super().__init__(store)
+        self.gate = gate
+
+    async def __aenter__(self):
+        await self.gate.wait()
+        return self
+
+
+async def test_checks_go_on_while_the_database_hangs_and_are_stored_at_their_own_time(monkeypatch):
+    async def fake_run(m):
+        return dict(sched._BLANK, is_up=False, error_msg="502 Bad Gateway")
+
+    monkeypatch.setattr(sched, "run_monitor", fake_run)
+    monkeypatch.setattr(sched, "MetricBatch", _Batch)
+    monkeypatch.setattr(sched, "MONITORS_READ_S", 0.05)
+    monkeypatch.setattr(sched, "STORE_WAIT_S", 0.05)
+    store, gate = _Store([_m("app")]), asyncio.Event()
+    app = NS(config={"MONITORS_POLL_INTERVAL": 60}, db_sessionmaker=lambda: _HungSession(store, gate))
+    sched._memory(app).monitors = list(store.monitors)        # as last read, before the stall
+
+    await sched.run_due_monitors(app)                          # the list and the stamp time out
+    await _settle(app)                                         # the result is held, the monitor freed
+    assert store.added == [] and sched._state(app)[0] == set()
+    await sched.run_due_monitors(app)
+    await _settle(app)
+    assert store.added == []                                   # its start was kept in memory: not due again
+
+    checked_by = datetime.now(timezone.utc)
+    gate.set()                                                 # the database answers
+    await app._holdings["Monitor checks"].flush(1.0)
+    [res] = store.added
+    assert res.is_up is False and res.checked_at < checked_by  # stored at the time it was checked