"""Self-monitoring: record Steward's own health each minute. A leaked socket/file handle in a poll loop (see the SNMP and UniFi fd-leak issues) used to fail silently until the process hit its fd ceiling and `socket.accept()` started raising `OSError: [Errno 24] Too many open files` — taking the whole app down with no early warning. This turns that failure mode into an observable signal. Steward monitors other things; it should monitor itself. We record two metrics each tick under `source_module="steward"`: • ``open_fds`` — raw count of open descriptors • ``open_fds_pct`` — that count as a percentage of the soft RLIMIT_NOFILE Both flow through the normal alert pipeline, so the operator can attach an alert rule to either via the existing alert-rules UI. As a zero-config floor we also log a WARNING once usage crosses ``FD_WARN_PCT`` — a leak becomes visible even before any rule is set up. The same tick records the rest of Steward's health (#5467), under the same source and resource: • ``ingest_posts`` / ``ingest_failed`` / ``ingest_slowest_ms`` — the host-agent posts since the last tick (core/health.py), how many didn't land, the slowest • ``db_roundtrip_ms`` — a timed ``SELECT 1``, connection checkout included • ``unwatched_secs`` — how long Steward wasn't recording before this tick (a deploy, an outage, a stalled loop), from its own last sample; 0 when unbroken • ``restart_gap_secs`` — once, on a fresh process's first tick: how long since the last process's last sample, however short. A quick redeploy never breaks the minute long enough to show as unwatched; this marks it anyway, so get_anomaly can show Steward's own restarts beside an anomaly (#5862) ``cleanup_secs`` is recorded by the hourly cleanup itself when it finishes. """ from __future__ import annotations import logging import os import time from datetime import datetime, timezone from sqlalchemy import func logger = logging.getLogger(__name__) # Stdlib-only fd counting via /proc keeps this dependency-free. resource is # POSIX-only but always present on the Linux runtime image; guard anyway so # imports never explode on a dev machine. try: import resource except ImportError: # pragma: no cover - non-POSIX resource = None # type: ignore[assignment] # Warn (without needing a configured alert rule) once we're using this fraction # of the soft fd limit. 80% leaves headroom to act before accepts start failing. FD_WARN_PCT = 80.0 _warned = False # de-dupe the WARNING so a sustained leak doesn't spam the log _started = False # this process has ticked: restart_gap_secs is recorded on the first tick only def count_open_fds() -> int | None: """Number of open file descriptors for this process, or None if unknown. Reads ``/proc/self/fd`` (Linux). Returns None where /proc isn't available (e.g. a macOS dev box) so callers degrade to a no-op rather than guessing. """ try: return len(os.listdir("/proc/self/fd")) except OSError: return None def fd_soft_limit() -> int | None: """Soft RLIMIT_NOFILE for this process, or None if it can't be read. None when the limit is unknown or 'unlimited' (RLIM_INFINITY) — a percentage against an unbounded ceiling is meaningless, so we skip the pct metric then. """ if resource is None: return None try: soft, _hard = resource.getrlimit(resource.RLIMIT_NOFILE) except (ValueError, OSError): return None if soft <= 0 or soft == resource.RLIM_INFINITY: return None return soft def unwatched_secs(last: datetime | None, now: datetime) -> float: """How long Steward wasn't recording before `now`, given its last sample: 0 when the ticks are unbroken (a gap no longer than core.watch.GAP_S).""" from .watch import GAP_S if last is None: return 0.0 gap = (now - last).total_seconds() return gap if gap > GAP_S else 0.0 async def record_self_metrics(app) -> None: """Record Steward's own health as metrics; warn past the open-file floor. The open-file pair is skipped (logged at debug) where fd accounting isn't available, so it's safe to schedule unconditionally. """ global _warned, _started from sqlalchemy import select, text from steward.models.metrics import PluginMetricLatest from .alerts import MetricBatch from .health import ingest fds = count_open_fds() if fds is None: logger.debug("self_monitor: /proc/self/fd unavailable — skipping fd metrics") soft = fd_soft_limit() pct = (fds / soft * 100.0) if (fds is not None and soft) else None tally = ingest.take() now = datetime.now(timezone.utc) started = time.monotonic() async with app.db_sessionmaker() as session: async with session.begin(): await session.execute(text("SELECT 1")) roundtrip_ms = (time.monotonic() - started) * 1000.0 # The last tick's sample, whatever restarts came between. last = (await session.execute(select(func.max(PluginMetricLatest.recorded_at)).where( PluginMetricLatest.source_module == "steward", PluginMetricLatest.resource_name == "process", PluginMetricLatest.metric_name == "db_roundtrip_ms"))).scalar() batch = MetricBatch() # Literal names, so tests/core/test_metric_catalog.py sees each one. for metric_name, value in [ ("ingest_posts", tally.posts), ("ingest_failed", tally.failed), ("ingest_slowest_ms", tally.slowest_ms), ("db_roundtrip_ms", roundtrip_ms), ("unwatched_secs", unwatched_secs(last, now)), ("restart_gap_secs", None if _started or last is None else (now - last).total_seconds()), ("open_fds", fds), ("open_fds_pct", pct)]: if value is not None: batch.add(source_module="steward", resource_name="process", metric_name=metric_name, value=value, recorded_at=now) await batch.record(session) _started = True if pct is not None and pct >= FD_WARN_PCT: if not _warned: logger.warning( "Open file descriptors at %.0f%% of the soft limit (%d/%d) — " "possible descriptor leak; the app will stop accepting " "connections if this reaches 100%%.", pct, fds, soft) _warned = True else: _warned = False # recovered — re-arm the warning for the next breach logger.debug("self_monitor: open_fds=%s soft_limit=%s pct=%s ingest=%s db=%.0fms", fds, soft, pct, tally, roundtrip_ms) async def record_cleanup_secs(app, secs: float) -> None: """How long the hourly cleanup just took (#5467), as Steward's own metric.""" from .alerts import MetricBatch async with app.db_sessionmaker() as session: async with session.begin(): batch = MetricBatch() batch.add(source_module="steward", resource_name="process", metric_name="cleanup_secs", value=secs) await batch.record(session)