FabledSteward/steward/core/self_monitor.py

"""Self-monitoring: record Steward's own health each minute.

A leaked socket/file handle in a poll loop (see the SNMP and UniFi fd-leak
issues) used to fail silently until the process hit its fd ceiling and
`socket.accept()` started raising `OSError: [Errno 24] Too many open files` —
taking the whole app down with no early warning.

This turns that failure mode into an observable signal. Steward monitors other
things; it should monitor itself. We record two metrics each tick under
`source_module="steward"`:

  • ``open_fds``      — raw count of open descriptors
  • ``open_fds_pct``  — that count as a percentage of the soft RLIMIT_NOFILE

Both flow through the normal alert pipeline, so the operator can attach an alert
rule to either via the existing alert-rules UI. As a zero-config floor we also
log a WARNING once usage crosses ``FD_WARN_PCT`` — a leak becomes visible even
before any rule is set up.

The same tick records the rest of Steward's health (#5467), under the same
source and resource:

  • ``ingest_posts`` / ``ingest_failed`` / ``ingest_slowest_ms`` — the host-agent
    posts since the last tick (core/health.py), how many didn't land, the slowest
  • ``db_roundtrip_ms`` — a timed ``SELECT 1``, connection checkout included
  • ``unwatched_secs`` — how long Steward wasn't recording before this tick (a
    deploy, an outage, a stalled loop), from its own last sample; 0 when unbroken

``cleanup_secs`` is recorded by the hourly cleanup itself when it finishes.
"""
from __future__ import annotations
import logging
import os
import time
from datetime import datetime, timezone

from sqlalchemy import func

logger = logging.getLogger(__name__)

# Stdlib-only fd counting via /proc keeps this dependency-free. resource is
# POSIX-only but always present on the Linux runtime image; guard anyway so
# imports never explode on a dev machine.
try:
    import resource
except ImportError:  # pragma: no cover - non-POSIX
    resource = None  # type: ignore[assignment]

# Warn (without needing a configured alert rule) once we're using this fraction
# of the soft fd limit. 80% leaves headroom to act before accepts start failing.
FD_WARN_PCT = 80.0

_warned = False  # de-dupe the WARNING so a sustained leak doesn't spam the log


def count_open_fds() -> int | None:
    """Number of open file descriptors for this process, or None if unknown.

    Reads ``/proc/self/fd`` (Linux). Returns None where /proc isn't available
    (e.g. a macOS dev box) so callers degrade to a no-op rather than guessing.
    """
    try:
        return len(os.listdir("/proc/self/fd"))
    except OSError:
        return None


def fd_soft_limit() -> int | None:
    """Soft RLIMIT_NOFILE for this process, or None if it can't be read.

    None when the limit is unknown or 'unlimited' (RLIM_INFINITY) — a percentage
    against an unbounded ceiling is meaningless, so we skip the pct metric then.
    """
    if resource is None:
        return None
    try:
        soft, _hard = resource.getrlimit(resource.RLIMIT_NOFILE)
    except (ValueError, OSError):
        return None
    if soft <= 0 or soft == resource.RLIM_INFINITY:
        return None
    return soft


def unwatched_secs(last: datetime | None, now: datetime) -> float:
    """How long Steward wasn't recording before `now`, given its last sample:
    0 when the ticks are unbroken (a gap no longer than core.watch.GAP_S)."""
    from .watch import GAP_S
    if last is None:
        return 0.0
    gap = (now - last).total_seconds()
    return gap if gap > GAP_S else 0.0


async def record_self_metrics(app) -> None:
    """Record Steward's own health as metrics; warn past the open-file floor.

    The open-file pair is skipped (logged at debug) where fd accounting isn't
    available, so it's safe to schedule unconditionally.
    """
    global _warned

    from sqlalchemy import select, text
    from steward.models.metrics import PluginMetricLatest
    from .alerts import MetricBatch
    from .health import ingest

    fds = count_open_fds()
    if fds is None:
        logger.debug("self_monitor: /proc/self/fd unavailable — skipping fd metrics")
    soft = fd_soft_limit()
    pct = (fds / soft * 100.0) if (fds is not None and soft) else None
    tally = ingest.take()

    now = datetime.now(timezone.utc)
    started = time.monotonic()
    async with app.db_sessionmaker() as session:
        async with session.begin():
            await session.execute(text("SELECT 1"))
            roundtrip_ms = (time.monotonic() - started) * 1000.0
            # The last tick's sample, whatever restarts came between.
            last = (await session.execute(select(func.max(PluginMetricLatest.recorded_at)).where(
                PluginMetricLatest.source_module == "steward",
                PluginMetricLatest.resource_name == "process",
                PluginMetricLatest.metric_name == "db_roundtrip_ms"))).scalar()
            batch = MetricBatch()
            # Literal names, so tests/core/test_metric_catalog.py sees each one.
            for metric_name, value in [
                    ("ingest_posts", tally.posts), ("ingest_failed", tally.failed),
                    ("ingest_slowest_ms", tally.slowest_ms), ("db_roundtrip_ms", roundtrip_ms),
                    ("unwatched_secs", unwatched_secs(last, now)),
                    ("open_fds", fds), ("open_fds_pct", pct)]:
                if value is not None:
                    batch.add(source_module="steward", resource_name="process",
                              metric_name=metric_name, value=value, recorded_at=now)
            await batch.record(session)

    if pct is not None and pct >= FD_WARN_PCT:
        if not _warned:
            logger.warning(
                "Open file descriptors at %.0f%% of the soft limit (%d/%d) — "
                "possible descriptor leak; the app will stop accepting "
                "connections if this reaches 100%%.", pct, fds, soft)
            _warned = True
    else:
        _warned = False  # recovered — re-arm the warning for the next breach

    logger.debug("self_monitor: open_fds=%s soft_limit=%s pct=%s ingest=%s db=%.0fms",
                 fds, soft, pct, tally, roundtrip_ms)


async def record_cleanup_secs(app, secs: float) -> None:
    """How long the hourly cleanup just took (#5467), as Steward's own metric."""
    from .alerts import MetricBatch
    async with app.db_sessionmaker() as session:
        async with session.begin():
            batch = MetricBatch()
            batch.add(source_module="steward", resource_name="process",
                      metric_name="cleanup_secs", value=secs)
            await batch.record(session)