bvandeusenandClaude Opus 5.5
58bfc7c361
#5060 "One evaluation path: record_metrics batch API; host-agent ingest evaluates rules". - record_metrics(session, rows) writes metric rows and evaluates the rules that watch them, reading rules, states and maintenance windows once per batch. Each rule sees its key's samples oldest first, so an agent catching up after an outage counts consecutive checks as they happened. - Host-agent ingest now goes through it. Before, it wrote plugin_metrics directly and no host_agent rule could ever fire. The catalog gains the per-mount disk_used_pct ("<host>:<mount>"). - MetricBatch gathers a transaction's samples; the docker ingest, traefik, unifi, snmp, monitor and self-monitor writers record one batch each instead of a rule query per sample. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
116 lines
3.9 KiB
Python
116 lines
3.9 KiB
Python
"""Unit tests for the self-fd watchdog (no DB, no network).
|
|
|
|
Exercises the fd accounting, the percentage/warning floor, and that the
|
|
recorder degrades to a no-op where fd accounting isn't available. The DB session
|
|
and the alert pipeline are faked — we only assert that the right metrics get
|
|
handed to record_metrics.
|
|
"""
|
|
import asyncio
|
|
import logging
|
|
|
|
import steward.core.alerts as alerts
|
|
import steward.core.self_monitor as sm
|
|
|
|
|
|
# ── fd accounting ────────────────────────────────────────────────────────────
|
|
|
|
def test_count_open_fds_sane_or_none():
|
|
n = sm.count_open_fds()
|
|
# On the Linux CI image /proc exists; stdin/stdout/stderr are always open.
|
|
assert n is None or n >= 3
|
|
|
|
|
|
def test_fd_soft_limit_positive_or_none():
|
|
soft = sm.fd_soft_limit()
|
|
assert soft is None or soft > 0
|
|
|
|
|
|
# ── fake session / app plumbing ──────────────────────────────────────────────
|
|
|
|
class _Ctx:
|
|
async def __aenter__(self):
|
|
return self
|
|
|
|
async def __aexit__(self, *_):
|
|
return False
|
|
|
|
|
|
class _FakeSession(_Ctx):
|
|
def begin(self):
|
|
return _Ctx()
|
|
|
|
|
|
class _FakeApp:
|
|
def db_sessionmaker(self):
|
|
return _FakeSession()
|
|
|
|
|
|
def _patch_recorder(monkeypatch):
|
|
recorded: list[tuple] = []
|
|
|
|
async def fake_record_metrics(session, rows):
|
|
recorded.extend((r.source_module, r.resource_name, r.metric_name, r.value) for r in rows)
|
|
|
|
monkeypatch.setattr(alerts, "record_metrics", fake_record_metrics)
|
|
return recorded
|
|
|
|
|
|
# ── record_self_metrics ──────────────────────────────────────────────────────
|
|
|
|
def test_records_both_fd_metrics(monkeypatch):
|
|
recorded = _patch_recorder(monkeypatch)
|
|
monkeypatch.setattr(sm, "count_open_fds", lambda: 50)
|
|
monkeypatch.setattr(sm, "fd_soft_limit", lambda: 1000)
|
|
|
|
asyncio.run(sm.record_self_metrics(_FakeApp()))
|
|
|
|
assert ("steward", "process", "open_fds", 50.0) in recorded
|
|
assert ("steward", "process", "open_fds_pct", 5.0) in recorded
|
|
|
|
|
|
def test_no_pct_metric_when_limit_unknown(monkeypatch):
|
|
recorded = _patch_recorder(monkeypatch)
|
|
monkeypatch.setattr(sm, "count_open_fds", lambda: 50)
|
|
monkeypatch.setattr(sm, "fd_soft_limit", lambda: None)
|
|
|
|
asyncio.run(sm.record_self_metrics(_FakeApp()))
|
|
|
|
metric_names = [m for (_s, _r, m, _v) in recorded]
|
|
assert "open_fds" in metric_names
|
|
assert "open_fds_pct" not in metric_names # meaningless without a ceiling
|
|
|
|
|
|
def test_warns_past_floor(monkeypatch, caplog):
|
|
_patch_recorder(monkeypatch)
|
|
monkeypatch.setattr(sm, "count_open_fds", lambda: 900)
|
|
monkeypatch.setattr(sm, "fd_soft_limit", lambda: 1000) # 90% ≥ FD_WARN_PCT
|
|
monkeypatch.setattr(sm, "_warned", False)
|
|
|
|
with caplog.at_level(logging.WARNING, logger=sm.logger.name):
|
|
asyncio.run(sm.record_self_metrics(_FakeApp()))
|
|
|
|
assert any("soft limit" in r.message for r in caplog.records)
|
|
|
|
|
|
def test_no_warning_below_floor(monkeypatch, caplog):
|
|
_patch_recorder(monkeypatch)
|
|
monkeypatch.setattr(sm, "count_open_fds", lambda: 100)
|
|
monkeypatch.setattr(sm, "fd_soft_limit", lambda: 1000) # 10%
|
|
monkeypatch.setattr(sm, "_warned", False)
|
|
|
|
with caplog.at_level(logging.WARNING, logger=sm.logger.name):
|
|
asyncio.run(sm.record_self_metrics(_FakeApp()))
|
|
|
|
assert not any("soft limit" in r.message for r in caplog.records)
|
|
|
|
|
|
def test_noop_when_fd_count_unavailable(monkeypatch):
|
|
# /proc absent → must return before touching the DB.
|
|
monkeypatch.setattr(sm, "count_open_fds", lambda: None)
|
|
|
|
class _Boom:
|
|
def db_sessionmaker(self):
|
|
raise AssertionError("must not open a session when fds are unknown")
|
|
|
|
asyncio.run(sm.record_self_metrics(_Boom())) # no raise
|