"""Collected readings held in memory until the database takes them (#5995).

Steward's Postgres sits on the NFS storage it watches. When that storage
stalls, every write waits, and a collector that stores what it found before
collecting again stops collecting for as long as the stall lasts: on
2026-10-17 the NAS readings went blind for the fifteen minutes they mattered.

A collector built on a Holding collects on schedule whatever the database is
doing, and hands each round to the holding. One writer at a time stores what
is held, oldest first, each item at the time it was collected, so a stall
delays the readings instead of losing them, and alert rules count them in the
order they happened.

    held = holding(app, "SNMP poll", store=_store, limit=120)
    held.hold(round)                     # one collected round
    await held.flush(wait_s=20)          # store it, or leave it to the writer

The holding is bounded: past `limit` rounds the oldest is dropped, and that
is logged with how many have gone. A store has `deadline_s`: generous, past
any stall seen so far, so a connection that will never answer is given up
and the rounds stay held for a fresh one rather than waiting forever.
"""
from __future__ import annotations

import asyncio
import logging
from collections import deque
from typing import Awaitable, Callable

logger = logging.getLogger(__name__)


class Holding:
    def __init__(self, name: str, store: Callable[[list], Awaitable[None]], limit: int,
                 deadline_s: float = 1800.0) -> None:
        self.name = name
        self.store = store
        self.limit = limit
        self.deadline_s = deadline_s
        self._held: deque[tuple[int, object]] = deque()
        self._seq = 0
        self._writer: asyncio.Task | None = None
        self.dropped = 0

    def __len__(self) -> int:
        return len(self._held)

    def hold(self, item) -> None:
        """Hold one collected round until it is stored."""
        self._seq += 1
        self._held.append((self._seq, item))
        while len(self._held) > self.limit:
            self._held.popleft()
            self.dropped += 1
            logger.warning("%s: the database hasn't taken a round in %d; dropped the oldest "
                           "(%d dropped so far)", self.name, self.limit, self.dropped)

    async def flush(self, wait_s: float) -> bool:
        """Start the writer if it isn't running, and wait up to `wait_s` for
        it. True when nothing is left held. A writer still waiting on the
        database is left to finish: the next round joins what it stores next."""
        writer = self._writer
        if writer is None or writer.done() or writer.get_loop() is not asyncio.get_running_loop():
            if not self._held:
                return True
            writer = self._writer = asyncio.create_task(self._write())
        done, _ = await asyncio.wait({writer}, timeout=wait_s)
        if not done:
            logger.warning("%s: the database hasn't taken %d held round(s) in %.0f s; "
                           "collecting on", self.name, len(self._held), wait_s)
        return not self._held

    async def _write(self) -> None:
        while self._held:
            batch = list(self._held)
            try:
                await asyncio.wait_for(self.store([item for _, item in batch]), self.deadline_s)
            except TimeoutError:
                logger.error("%s: storing %d held round(s) took over %.0f s; given up, they stay "
                             "held (the database may still finish it)", self.name, len(batch),
                             self.deadline_s)
                return
            except Exception:
                logger.exception("%s: storing %d held round(s) failed; they stay held",
                                 self.name, len(batch))
                return
            last = batch[-1][0]
            while self._held and self._held[0][0] <= last:
                self._held.popleft()


def holding(app, name: str, store: Callable[[list], Awaitable[None]], limit: int) -> Holding:
    """The app's Holding for one collector, made on first use."""
    holdings = getattr(app, "_holdings", None)
    if holdings is None:
        holdings = app._holdings = {}
    if name not in holdings:
        holdings[name] = Holding(name, store, limit)
    return holdings[name]