bvandeusen
f40063a74d
Guardrails so the fd-leak class of bug (Errno 24 lockup; recent SNMP + UniFi fixes) surfaces early or can't compound, instead of silently killing the app. - Self-fd watchdog (steward/core/self_monitor.py): records open_fds and open_fds_pct (% of soft RLIMIT_NOFILE) as "steward"/"process" metrics each minute through the normal alert pipeline, so the operator can alert on them via the existing alert-rules UI. Built-in WARNING floor at 80% gives a zero-config early signal. Stdlib-only (/proc + resource); degrades to a no-op off Linux. Registered as a core ScheduledTask in app.py. - Poll-overlap guard (steward/core/scheduler.py): extract a pure _DueTracker that skips a tick while a task's prior run is still in flight, so a hung poll can't stack overlapping runs (which amplify per-poll resource/fd use). A skipped task isn't penalised — it retries the next tick after it completes. - fd-stability tests (tests/core/): _DueTracker overlap policy, the watchdog metric/warning/degradation paths, and a real-fd canary that hammers tcp_check and asserts /proc/self/fd doesn't grow. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
116 lines
3.9 KiB
Python
116 lines
3.9 KiB
Python
"""Unit tests for the self-fd watchdog (no DB, no network).
|
|
|
|
Exercises the fd accounting, the percentage/warning floor, and that the
|
|
recorder degrades to a no-op where fd accounting isn't available. The DB session
|
|
and the alert pipeline are faked — we only assert that the right metrics get
|
|
handed to record_metric.
|
|
"""
|
|
import asyncio
|
|
import logging
|
|
|
|
import steward.core.alerts as alerts
|
|
import steward.core.self_monitor as sm
|
|
|
|
|
|
# ── fd accounting ────────────────────────────────────────────────────────────
|
|
|
|
def test_count_open_fds_sane_or_none():
|
|
n = sm.count_open_fds()
|
|
# On the Linux CI image /proc exists; stdin/stdout/stderr are always open.
|
|
assert n is None or n >= 3
|
|
|
|
|
|
def test_fd_soft_limit_positive_or_none():
|
|
soft = sm.fd_soft_limit()
|
|
assert soft is None or soft > 0
|
|
|
|
|
|
# ── fake session / app plumbing ──────────────────────────────────────────────
|
|
|
|
class _Ctx:
|
|
async def __aenter__(self):
|
|
return self
|
|
|
|
async def __aexit__(self, *_):
|
|
return False
|
|
|
|
|
|
class _FakeSession(_Ctx):
|
|
def begin(self):
|
|
return _Ctx()
|
|
|
|
|
|
class _FakeApp:
|
|
def db_sessionmaker(self):
|
|
return _FakeSession()
|
|
|
|
|
|
def _patch_recorder(monkeypatch):
|
|
recorded: list[tuple] = []
|
|
|
|
async def fake_record_metric(session, source_module, resource_name, metric_name, value):
|
|
recorded.append((source_module, resource_name, metric_name, value))
|
|
|
|
monkeypatch.setattr(alerts, "record_metric", fake_record_metric)
|
|
return recorded
|
|
|
|
|
|
# ── record_self_metrics ──────────────────────────────────────────────────────
|
|
|
|
def test_records_both_fd_metrics(monkeypatch):
|
|
recorded = _patch_recorder(monkeypatch)
|
|
monkeypatch.setattr(sm, "count_open_fds", lambda: 50)
|
|
monkeypatch.setattr(sm, "fd_soft_limit", lambda: 1000)
|
|
|
|
asyncio.run(sm.record_self_metrics(_FakeApp()))
|
|
|
|
assert ("steward", "process", "open_fds", 50.0) in recorded
|
|
assert ("steward", "process", "open_fds_pct", 5.0) in recorded
|
|
|
|
|
|
def test_no_pct_metric_when_limit_unknown(monkeypatch):
|
|
recorded = _patch_recorder(monkeypatch)
|
|
monkeypatch.setattr(sm, "count_open_fds", lambda: 50)
|
|
monkeypatch.setattr(sm, "fd_soft_limit", lambda: None)
|
|
|
|
asyncio.run(sm.record_self_metrics(_FakeApp()))
|
|
|
|
metric_names = [m for (_s, _r, m, _v) in recorded]
|
|
assert "open_fds" in metric_names
|
|
assert "open_fds_pct" not in metric_names # meaningless without a ceiling
|
|
|
|
|
|
def test_warns_past_floor(monkeypatch, caplog):
|
|
_patch_recorder(monkeypatch)
|
|
monkeypatch.setattr(sm, "count_open_fds", lambda: 900)
|
|
monkeypatch.setattr(sm, "fd_soft_limit", lambda: 1000) # 90% ≥ FD_WARN_PCT
|
|
monkeypatch.setattr(sm, "_warned", False)
|
|
|
|
with caplog.at_level(logging.WARNING, logger=sm.logger.name):
|
|
asyncio.run(sm.record_self_metrics(_FakeApp()))
|
|
|
|
assert any("soft limit" in r.message for r in caplog.records)
|
|
|
|
|
|
def test_no_warning_below_floor(monkeypatch, caplog):
|
|
_patch_recorder(monkeypatch)
|
|
monkeypatch.setattr(sm, "count_open_fds", lambda: 100)
|
|
monkeypatch.setattr(sm, "fd_soft_limit", lambda: 1000) # 10%
|
|
monkeypatch.setattr(sm, "_warned", False)
|
|
|
|
with caplog.at_level(logging.WARNING, logger=sm.logger.name):
|
|
asyncio.run(sm.record_self_metrics(_FakeApp()))
|
|
|
|
assert not any("soft limit" in r.message for r in caplog.records)
|
|
|
|
|
|
def test_noop_when_fd_count_unavailable(monkeypatch):
|
|
# /proc absent → must return before touching the DB.
|
|
monkeypatch.setattr(sm, "count_open_fds", lambda: None)
|
|
|
|
class _Boom:
|
|
def db_sessionmaker(self):
|
|
raise AssertionError("must not open a session when fds are unknown")
|
|
|
|
asyncio.run(sm.record_self_metrics(_Boom())) # no raise
|