feat(reliability): fd-leak rails — self-watchdog, poll-overlap guard, fd tests

Guardrails so the fd-leak class of bug (Errno 24 lockup; recent SNMP + UniFi
fixes) surfaces early or can't compound, instead of silently killing the app.

- Self-fd watchdog (steward/core/self_monitor.py): records open_fds and
  open_fds_pct (% of soft RLIMIT_NOFILE) as "steward"/"process" metrics each
  minute through the normal alert pipeline, so the operator can alert on them
  via the existing alert-rules UI. Built-in WARNING floor at 80% gives a
  zero-config early signal. Stdlib-only (/proc + resource); degrades to a no-op
  off Linux. Registered as a core ScheduledTask in app.py.

- Poll-overlap guard (steward/core/scheduler.py): extract a pure _DueTracker
  that skips a tick while a task's prior run is still in flight, so a hung poll
  can't stack overlapping runs (which amplify per-poll resource/fd use). A
  skipped task isn't penalised — it retries the next tick after it completes.

- fd-stability tests (tests/core/): _DueTracker overlap policy, the watchdog
  metric/warning/degradation paths, and a real-fd canary that hammers tcp_check
  and asserts /proc/self/fd doesn't grow.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

steward/core/scheduler.py

@@ -1,7 +1,7 @@
 from __future__ import annotations
 import asyncio
 import logging
-from dataclasses import dataclass
+from dataclasses import dataclass, field
 from typing import Callable, Coroutine
 
 logger = logging.getLogger(__name__)
@@ -15,28 +15,65 @@ class ScheduledTask:
     run_on_startup: bool = False
 
 
+@dataclass
+class _DueTracker:
+    """Decides which scheduled tasks are due, with a self-overlap guard.
+
+    Pure (no asyncio, no clock of its own — `now` is passed in) so the
+    scheduling policy is unit-testable without timing races. A task whose prior
+    run is still in flight is NOT re-fired: overlapping poll runs stack up open
+    connections/subprocesses and amplify any per-poll resource use — the same
+    failure mode behind the fd-leak lockups. The skipped task is retried on the
+    next tick once it completes (its last_run isn't advanced while skipped).
+    """
+    last_run: dict[str, float] = field(default_factory=dict)
+    in_flight: set[str] = field(default_factory=set)
+
+    def due(self, tasks: list[ScheduledTask], now: float) -> list[ScheduledTask]:
+        ready: list[ScheduledTask] = []
+        for task in tasks:
+            if now - self.last_run.get(task.name, 0) < task.interval_seconds:
+                continue
+            if task.name in self.in_flight:
+                logger.warning(
+                    "Scheduled task %r still running — skipping this tick",
+                    task.name)
+                continue
+            ready.append(task)
+        return ready
+
+    def mark_started(self, task: ScheduledTask, now: float) -> None:
+        self.in_flight.add(task.name)
+        self.last_run[task.name] = now
+
+    def mark_done(self, name: str) -> None:
+        self.in_flight.discard(name)
+
+
 async def start_scheduler(tasks: list[ScheduledTask]) -> None:
     """Run scheduled tasks in a loop. Call with asyncio.create_task()."""
-    last_run: dict[str, float] = {}
+    tracker = _DueTracker()
+
+    def _spawn(task: ScheduledTask, now: float) -> None:
+        tracker.mark_started(task, now)
+        asyncio.create_task(_run_task(task, tracker))
 
     for task in tasks:
         if task.run_on_startup:
             logger.info(f"Startup task: {task.name}")
-            asyncio.create_task(_run_task(task))
-            last_run[task.name] = asyncio.get_event_loop().time()
+            _spawn(task, asyncio.get_event_loop().time())
 
     while True:
         now = asyncio.get_event_loop().time()
-        for task in tasks:
-            last = last_run.get(task.name, 0)
-            if now - last >= task.interval_seconds:
-                asyncio.create_task(_run_task(task))
-                last_run[task.name] = now
+        for task in tracker.due(tasks, now):
+            _spawn(task, now)
         await asyncio.sleep(1)
 
 
-async def _run_task(task: ScheduledTask) -> None:
+async def _run_task(task: ScheduledTask, tracker: _DueTracker) -> None:
     try:
         await task.coro_factory()
     except Exception:
         logger.exception(f"Scheduled task {task.name!r} raised an exception")
+    finally:
+        tracker.mark_done(task.name)