bvandeusenandClaude Opus 5.5
58bfc7c361
#5060 "One evaluation path: record_metrics batch API; host-agent ingest evaluates rules". - record_metrics(session, rows) writes metric rows and evaluates the rules that watch them, reading rules, states and maintenance windows once per batch. Each rule sees its key's samples oldest first, so an agent catching up after an outage counts consecutive checks as they happened. - Host-agent ingest now goes through it. Before, it wrote plugin_metrics directly and no host_agent rule could ever fire. The catalog gains the per-mount disk_used_pct ("<host>:<mount>"). - MetricBatch gathers a transaction's samples; the docker ingest, traefik, unifi, snmp, monitor and self-monitor writers record one batch each instead of a rule query per sample. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
117 lines
4.1 KiB
Python
117 lines
4.1 KiB
Python
"""Self-monitoring: record Steward's own open file-descriptor usage.
|
|
|
|
A leaked socket/file handle in a poll loop (see the SNMP and UniFi fd-leak
|
|
issues) used to fail silently until the process hit its fd ceiling and
|
|
`socket.accept()` started raising `OSError: [Errno 24] Too many open files` —
|
|
taking the whole app down with no early warning.
|
|
|
|
This turns that failure mode into an observable signal. Steward monitors other
|
|
things; it should monitor itself. We record two metrics each tick under
|
|
`source_module="steward"`:
|
|
|
|
• ``open_fds`` — raw count of open descriptors
|
|
• ``open_fds_pct`` — that count as a percentage of the soft RLIMIT_NOFILE
|
|
|
|
Both flow through the normal alert pipeline, so the operator can attach an alert
|
|
rule to either via the existing alert-rules UI. As a zero-config floor we also
|
|
log a WARNING once usage crosses ``FD_WARN_PCT`` — a leak becomes visible even
|
|
before any rule is set up.
|
|
"""
|
|
from __future__ import annotations
|
|
import logging
|
|
import os
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
# Stdlib-only fd counting via /proc keeps this dependency-free. resource is
|
|
# POSIX-only but always present on the Linux runtime image; guard anyway so
|
|
# imports never explode on a dev machine.
|
|
try:
|
|
import resource
|
|
except ImportError: # pragma: no cover - non-POSIX
|
|
resource = None # type: ignore[assignment]
|
|
|
|
# Warn (without needing a configured alert rule) once we're using this fraction
|
|
# of the soft fd limit. 80% leaves headroom to act before accepts start failing.
|
|
FD_WARN_PCT = 80.0
|
|
|
|
_warned = False # de-dupe the WARNING so a sustained leak doesn't spam the log
|
|
|
|
|
|
def count_open_fds() -> int | None:
|
|
"""Number of open file descriptors for this process, or None if unknown.
|
|
|
|
Reads ``/proc/self/fd`` (Linux). Returns None where /proc isn't available
|
|
(e.g. a macOS dev box) so callers degrade to a no-op rather than guessing.
|
|
"""
|
|
try:
|
|
return len(os.listdir("/proc/self/fd"))
|
|
except OSError:
|
|
return None
|
|
|
|
|
|
def fd_soft_limit() -> int | None:
|
|
"""Soft RLIMIT_NOFILE for this process, or None if it can't be read.
|
|
|
|
None when the limit is unknown or 'unlimited' (RLIM_INFINITY) — a percentage
|
|
against an unbounded ceiling is meaningless, so we skip the pct metric then.
|
|
"""
|
|
if resource is None:
|
|
return None
|
|
try:
|
|
soft, _hard = resource.getrlimit(resource.RLIMIT_NOFILE)
|
|
except (ValueError, OSError):
|
|
return None
|
|
if soft <= 0 or soft == resource.RLIM_INFINITY:
|
|
return None
|
|
return soft
|
|
|
|
|
|
async def record_self_metrics(app) -> None:
|
|
"""Record open-fd usage as Steward's own metrics; warn past the floor.
|
|
|
|
No-op (logged at debug) when fd accounting isn't available on this platform,
|
|
so it's safe to schedule unconditionally.
|
|
"""
|
|
global _warned
|
|
|
|
fds = count_open_fds()
|
|
if fds is None:
|
|
logger.debug("self_monitor: /proc/self/fd unavailable — skipping fd metrics")
|
|
return
|
|
|
|
from .alerts import MetricBatch
|
|
|
|
soft = fd_soft_limit()
|
|
pct = (fds / soft * 100.0) if soft else None
|
|
|
|
async with app.db_sessionmaker() as session:
|
|
async with session.begin():
|
|
batch = MetricBatch()
|
|
batch.add(
|
|
source_module="steward",
|
|
resource_name="process",
|
|
metric_name="open_fds",
|
|
value=float(fds),
|
|
)
|
|
if pct is not None:
|
|
batch.add(
|
|
source_module="steward",
|
|
resource_name="process",
|
|
metric_name="open_fds_pct",
|
|
value=pct,
|
|
)
|
|
await batch.record(session)
|
|
|
|
if pct is not None and pct >= FD_WARN_PCT:
|
|
if not _warned:
|
|
logger.warning(
|
|
"Open file descriptors at %.0f%% of the soft limit (%d/%d) — "
|
|
"possible descriptor leak; the app will stop accepting "
|
|
"connections if this reaches 100%%.", pct, fds, soft)
|
|
_warned = True
|
|
else:
|
|
_warned = False # recovered — re-arm the warning for the next breach
|
|
|
|
logger.debug("self_monitor: open_fds=%d soft_limit=%s pct=%s", fds, soft, pct)
|