"""Self-monitoring: record Steward's own open file-descriptor usage. A leaked socket/file handle in a poll loop (see the SNMP and UniFi fd-leak issues) used to fail silently until the process hit its fd ceiling and `socket.accept()` started raising `OSError: [Errno 24] Too many open files` — taking the whole app down with no early warning. This turns that failure mode into an observable signal. Steward monitors other things; it should monitor itself. We record two metrics each tick under `source_module="steward"`: • ``open_fds`` — raw count of open descriptors • ``open_fds_pct`` — that count as a percentage of the soft RLIMIT_NOFILE Both flow through the normal alert pipeline, so the operator can attach an alert rule to either via the existing alert-rules UI. As a zero-config floor we also log a WARNING once usage crosses ``FD_WARN_PCT`` — a leak becomes visible even before any rule is set up. """ from __future__ import annotations import logging import os logger = logging.getLogger(__name__) # Stdlib-only fd counting via /proc keeps this dependency-free. resource is # POSIX-only but always present on the Linux runtime image; guard anyway so # imports never explode on a dev machine. try: import resource except ImportError: # pragma: no cover - non-POSIX resource = None # type: ignore[assignment] # Warn (without needing a configured alert rule) once we're using this fraction # of the soft fd limit. 80% leaves headroom to act before accepts start failing. FD_WARN_PCT = 80.0 _warned = False # de-dupe the WARNING so a sustained leak doesn't spam the log def count_open_fds() -> int | None: """Number of open file descriptors for this process, or None if unknown. Reads ``/proc/self/fd`` (Linux). Returns None where /proc isn't available (e.g. a macOS dev box) so callers degrade to a no-op rather than guessing. """ try: return len(os.listdir("/proc/self/fd")) except OSError: return None def fd_soft_limit() -> int | None: """Soft RLIMIT_NOFILE for this process, or None if it can't be read. None when the limit is unknown or 'unlimited' (RLIM_INFINITY) — a percentage against an unbounded ceiling is meaningless, so we skip the pct metric then. """ if resource is None: return None try: soft, _hard = resource.getrlimit(resource.RLIMIT_NOFILE) except (ValueError, OSError): return None if soft <= 0 or soft == resource.RLIM_INFINITY: return None return soft async def record_self_metrics(app) -> None: """Record open-fd usage as Steward's own metrics; warn past the floor. No-op (logged at debug) when fd accounting isn't available on this platform, so it's safe to schedule unconditionally. """ global _warned fds = count_open_fds() if fds is None: logger.debug("self_monitor: /proc/self/fd unavailable — skipping fd metrics") return from .alerts import record_metric soft = fd_soft_limit() pct = (fds / soft * 100.0) if soft else None async with app.db_sessionmaker() as session: async with session.begin(): await record_metric( session=session, source_module="steward", resource_name="process", metric_name="open_fds", value=float(fds), ) if pct is not None: await record_metric( session=session, source_module="steward", resource_name="process", metric_name="open_fds_pct", value=pct, ) if pct is not None and pct >= FD_WARN_PCT: if not _warned: logger.warning( "Open file descriptors at %.0f%% of the soft limit (%d/%d) — " "possible descriptor leak; the app will stop accepting " "connections if this reaches 100%%.", pct, fds, soft) _warned = True else: _warned = False # recovered — re-arm the warning for the next breach logger.debug("self_monitor: open_fds=%d soft_limit=%s pct=%s", fds, soft, pct)