feat(reliability): fd-leak rails — self-watchdog, poll-overlap guard, fd tests

Guardrails so the fd-leak class of bug (Errno 24 lockup; recent SNMP + UniFi
fixes) surfaces early or can't compound, instead of silently killing the app.

- Self-fd watchdog (steward/core/self_monitor.py): records open_fds and
  open_fds_pct (% of soft RLIMIT_NOFILE) as "steward"/"process" metrics each
  minute through the normal alert pipeline, so the operator can alert on them
  via the existing alert-rules UI. Built-in WARNING floor at 80% gives a
  zero-config early signal. Stdlib-only (/proc + resource); degrades to a no-op
  off Linux. Registered as a core ScheduledTask in app.py.

- Poll-overlap guard (steward/core/scheduler.py): extract a pure _DueTracker
  that skips a tick while a task's prior run is still in flight, so a hung poll
  can't stack overlapping runs (which amplify per-poll resource/fd use). A
  skipped task isn't penalised — it retries the next tick after it completes.

- fd-stability tests (tests/core/): _DueTracker overlap policy, the watchdog
  metric/warning/degradation paths, and a real-fd canary that hammers tcp_check
  and asserts /proc/self/fd doesn't grow.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

steward/core/scheduler.py

@@ -1,7 +1,7 @@
 from __future__ import annotations
 import asyncio
 import logging
-from dataclasses import dataclass
+from dataclasses import dataclass, field
 from typing import Callable, Coroutine
 
 logger = logging.getLogger(__name__)
@@ -15,28 +15,65 @@ class ScheduledTask:
     run_on_startup: bool = False
 
 
+@dataclass
+class _DueTracker:
+    """Decides which scheduled tasks are due, with a self-overlap guard.
+
+    Pure (no asyncio, no clock of its own — `now` is passed in) so the
+    scheduling policy is unit-testable without timing races. A task whose prior
+    run is still in flight is NOT re-fired: overlapping poll runs stack up open
+    connections/subprocesses and amplify any per-poll resource use — the same
+    failure mode behind the fd-leak lockups. The skipped task is retried on the
+    next tick once it completes (its last_run isn't advanced while skipped).
+    """
+    last_run: dict[str, float] = field(default_factory=dict)
+    in_flight: set[str] = field(default_factory=set)
+
+    def due(self, tasks: list[ScheduledTask], now: float) -> list[ScheduledTask]:
+        ready: list[ScheduledTask] = []
+        for task in tasks:
+            if now - self.last_run.get(task.name, 0) < task.interval_seconds:
+                continue
+            if task.name in self.in_flight:
+                logger.warning(
+                    "Scheduled task %r still running — skipping this tick",
+                    task.name)
+                continue
+            ready.append(task)
+        return ready
+
+    def mark_started(self, task: ScheduledTask, now: float) -> None:
+        self.in_flight.add(task.name)
+        self.last_run[task.name] = now
+
+    def mark_done(self, name: str) -> None:
+        self.in_flight.discard(name)
+
+
 async def start_scheduler(tasks: list[ScheduledTask]) -> None:
     """Run scheduled tasks in a loop. Call with asyncio.create_task()."""
-    last_run: dict[str, float] = {}
+    tracker = _DueTracker()
+
+    def _spawn(task: ScheduledTask, now: float) -> None:
+        tracker.mark_started(task, now)
+        asyncio.create_task(_run_task(task, tracker))
 
     for task in tasks:
         if task.run_on_startup:
             logger.info(f"Startup task: {task.name}")
-            asyncio.create_task(_run_task(task))
-            last_run[task.name] = asyncio.get_event_loop().time()
+            _spawn(task, asyncio.get_event_loop().time())
 
     while True:
         now = asyncio.get_event_loop().time()
-        for task in tasks:
-            last = last_run.get(task.name, 0)
-            if now - last >= task.interval_seconds:
-                asyncio.create_task(_run_task(task))
-                last_run[task.name] = now
+        for task in tracker.due(tasks, now):
+            _spawn(task, now)
         await asyncio.sleep(1)
 
 
-async def _run_task(task: ScheduledTask) -> None:
+async def _run_task(task: ScheduledTask, tracker: _DueTracker) -> None:
     try:
         await task.coro_factory()
     except Exception:
         logger.exception(f"Scheduled task {task.name!r} raised an exception")
+    finally:
+        tracker.mark_done(task.name)

steward/core/self_monitor.py

@@ -0,0 +1,116 @@
+"""Self-monitoring: record Steward's own open file-descriptor usage.
+
+A leaked socket/file handle in a poll loop (see the SNMP and UniFi fd-leak
+issues) used to fail silently until the process hit its fd ceiling and
+`socket.accept()` started raising `OSError: [Errno 24] Too many open files` —
+taking the whole app down with no early warning.
+
+This turns that failure mode into an observable signal. Steward monitors other
+things; it should monitor itself. We record two metrics each tick under
+`source_module="steward"`:
+
+  • ``open_fds``      — raw count of open descriptors
+  • ``open_fds_pct``  — that count as a percentage of the soft RLIMIT_NOFILE
+
+Both flow through the normal alert pipeline, so the operator can attach an alert
+rule to either via the existing alert-rules UI. As a zero-config floor we also
+log a WARNING once usage crosses ``FD_WARN_PCT`` — a leak becomes visible even
+before any rule is set up.
+"""
+from __future__ import annotations
+import logging
+import os
+
+logger = logging.getLogger(__name__)
+
+# Stdlib-only fd counting via /proc keeps this dependency-free. resource is
+# POSIX-only but always present on the Linux runtime image; guard anyway so
+# imports never explode on a dev machine.
+try:
+    import resource
+except ImportError:  # pragma: no cover - non-POSIX
+    resource = None  # type: ignore[assignment]
+
+# Warn (without needing a configured alert rule) once we're using this fraction
+# of the soft fd limit. 80% leaves headroom to act before accepts start failing.
+FD_WARN_PCT = 80.0
+
+_warned = False  # de-dupe the WARNING so a sustained leak doesn't spam the log
+
+
+def count_open_fds() -> int | None:
+    """Number of open file descriptors for this process, or None if unknown.
+
+    Reads ``/proc/self/fd`` (Linux). Returns None where /proc isn't available
+    (e.g. a macOS dev box) so callers degrade to a no-op rather than guessing.
+    """
+    try:
+        return len(os.listdir("/proc/self/fd"))
+    except OSError:
+        return None
+
+
+def fd_soft_limit() -> int | None:
+    """Soft RLIMIT_NOFILE for this process, or None if it can't be read.
+
+    None when the limit is unknown or 'unlimited' (RLIM_INFINITY) — a percentage
+    against an unbounded ceiling is meaningless, so we skip the pct metric then.
+    """
+    if resource is None:
+        return None
+    try:
+        soft, _hard = resource.getrlimit(resource.RLIMIT_NOFILE)
+    except (ValueError, OSError):
+        return None
+    if soft <= 0 or soft == resource.RLIM_INFINITY:
+        return None
+    return soft
+
+
+async def record_self_metrics(app) -> None:
+    """Record open-fd usage as Steward's own metrics; warn past the floor.
+
+    No-op (logged at debug) when fd accounting isn't available on this platform,
+    so it's safe to schedule unconditionally.
+    """
+    global _warned
+
+    fds = count_open_fds()
+    if fds is None:
+        logger.debug("self_monitor: /proc/self/fd unavailable — skipping fd metrics")
+        return
+
+    from .alerts import record_metric
+
+    soft = fd_soft_limit()
+    pct = (fds / soft * 100.0) if soft else None
+
+    async with app.db_sessionmaker() as session:
+        async with session.begin():
+            await record_metric(
+                session=session,
+                source_module="steward",
+                resource_name="process",
+                metric_name="open_fds",
+                value=float(fds),
+            )
+            if pct is not None:
+                await record_metric(
+                    session=session,
+                    source_module="steward",
+                    resource_name="process",
+                    metric_name="open_fds_pct",
+                    value=pct,
+                )
+
+    if pct is not None and pct >= FD_WARN_PCT:
+        if not _warned:
+            logger.warning(
+                "Open file descriptors at %.0f%% of the soft limit (%d/%d) — "
+                "possible descriptor leak; the app will stop accepting "
+                "connections if this reaches 100%%.", pct, fds, soft)
+            _warned = True
+    else:
+        _warned = False  # recovered — re-arm the warning for the next breach
+
+    logger.debug("self_monitor: open_fds=%d soft_limit=%s pct=%s", fds, soft, pct)