"""Unit tests for the self-monitor: open files and Steward's own health (no DB, no network). Exercises the fd accounting, the percentage/warning floor, and that the recorder degrades to a no-op where fd accounting isn't available. The DB session and the alert pipeline are faked — we only assert that the right metrics get handed to record_metrics. """ import asyncio import logging import steward.core.alerts as alerts import steward.core.self_monitor as sm # ── fd accounting ──────────────────────────────────────────────────────────── def test_count_open_fds_sane_or_none(): n = sm.count_open_fds() # On the Linux CI image /proc exists; stdin/stdout/stderr are always open. assert n is None or n >= 3 def test_fd_soft_limit_positive_or_none(): soft = sm.fd_soft_limit() assert soft is None or soft > 0 # ── fake session / app plumbing ────────────────────────────────────────────── class _Ctx: async def __aenter__(self): return self async def __aexit__(self, *_): return False class _FakeSession(_Ctx): last = None # the latest steward sample the unwatched gap is counted from def begin(self): return _Ctx() async def execute(self, stmt): last = self.last return type("R", (), {"scalar": staticmethod(lambda: last)})() class _FakeApp: def db_sessionmaker(self): return _FakeSession() def _patch_recorder(monkeypatch): recorded: list[tuple] = [] async def fake_record_metrics(session, rows): recorded.extend((r.source_module, r.resource_name, r.metric_name, r.value) for r in rows) monkeypatch.setattr(alerts, "record_metrics", fake_record_metrics) return recorded # ── record_self_metrics ────────────────────────────────────────────────────── def test_records_both_fd_metrics(monkeypatch): recorded = _patch_recorder(monkeypatch) monkeypatch.setattr(sm, "count_open_fds", lambda: 50) monkeypatch.setattr(sm, "fd_soft_limit", lambda: 1000) asyncio.run(sm.record_self_metrics(_FakeApp())) assert ("steward", "process", "open_fds", 50.0) in recorded assert ("steward", "process", "open_fds_pct", 5.0) in recorded def test_no_pct_metric_when_limit_unknown(monkeypatch): recorded = _patch_recorder(monkeypatch) monkeypatch.setattr(sm, "count_open_fds", lambda: 50) monkeypatch.setattr(sm, "fd_soft_limit", lambda: None) asyncio.run(sm.record_self_metrics(_FakeApp())) metric_names = [m for (_s, _r, m, _v) in recorded] assert "open_fds" in metric_names assert "open_fds_pct" not in metric_names # meaningless without a ceiling def test_warns_past_floor(monkeypatch, caplog): _patch_recorder(monkeypatch) monkeypatch.setattr(sm, "count_open_fds", lambda: 900) monkeypatch.setattr(sm, "fd_soft_limit", lambda: 1000) # 90% ≥ FD_WARN_PCT monkeypatch.setattr(sm, "_warned", False) with caplog.at_level(logging.WARNING, logger=sm.logger.name): asyncio.run(sm.record_self_metrics(_FakeApp())) assert any("soft limit" in r.message for r in caplog.records) def test_no_warning_below_floor(monkeypatch, caplog): _patch_recorder(monkeypatch) monkeypatch.setattr(sm, "count_open_fds", lambda: 100) monkeypatch.setattr(sm, "fd_soft_limit", lambda: 1000) # 10% monkeypatch.setattr(sm, "_warned", False) with caplog.at_level(logging.WARNING, logger=sm.logger.name): asyncio.run(sm.record_self_metrics(_FakeApp())) assert not any("soft limit" in r.message for r in caplog.records) def test_without_fd_accounting_the_rest_is_still_recorded(monkeypatch): # /proc absent: no open-file pair, but Steward's other health still lands. recorded = _patch_recorder(monkeypatch) monkeypatch.setattr(sm, "count_open_fds", lambda: None) asyncio.run(sm.record_self_metrics(_FakeApp())) names = {m for (_s, _r, m, _v) in recorded} assert "db_roundtrip_ms" in names and not names & {"open_fds", "open_fds_pct"} def test_the_minute_s_ingests_are_recorded_and_the_tally_starts_again(monkeypatch): """#5467: posts, failures and the slowest post since the last tick.""" from steward.core import health recorded = _patch_recorder(monkeypatch) monkeypatch.setattr(sm, "count_open_fds", lambda: 50) monkeypatch.setattr(health, "ingest", health.IngestTally()) for ms, ok in ((300.0, True), (9800.0, False), (450.0, True)): health.ingest.done(ms, ok) asyncio.run(sm.record_self_metrics(_FakeApp())) got = {m: v for (s, r, m, v) in recorded if (s, r) == ("steward", "process")} assert (got["ingest_posts"], got["ingest_failed"], got["ingest_slowest_ms"]) == (3.0, 1.0, 9800.0) assert got["unwatched_secs"] == 0.0 and got["db_roundtrip_ms"] >= 0 assert health.ingest.take() == health.IngestTally() # taken: the next minute starts at zero def test_unwatched_counts_only_a_real_gap(): from datetime import datetime, timedelta, timezone now = datetime(2026, 10, 9, 12, 0, tzinfo=timezone.utc) assert sm.unwatched_secs(None, now) == 0.0 # first ever tick assert sm.unwatched_secs(now - timedelta(seconds=61), now) == 0.0 # an ordinary minute assert sm.unwatched_secs(now - timedelta(minutes=7), now) == 420.0 # a deploy or outage def test_a_fresh_process_marks_its_restart_once(monkeypatch): """#5862: the first tick records how long since the last process's last sample, however short; later ticks and a first-ever start don't.""" from datetime import datetime, timedelta, timezone recorded = _patch_recorder(monkeypatch) monkeypatch.setattr(sm, "_started", False) monkeypatch.setattr(_FakeSession, "last", datetime.now(timezone.utc) - timedelta(seconds=40)) asyncio.run(sm.record_self_metrics(_FakeApp())) asyncio.run(sm.record_self_metrics(_FakeApp())) gaps = [v for (_s, _r, m, v) in recorded if m == "restart_gap_secs"] assert len(gaps) == 1 and 39 <= gaps[0] <= 60 assert dict((m, v) for (_s, _r, m, v) in recorded)["unwatched_secs"] == 0.0 recorded.clear() monkeypatch.setattr(sm, "_started", False) monkeypatch.setattr(_FakeSession, "last", None) asyncio.run(sm.record_self_metrics(_FakeApp())) assert "restart_gap_secs" not in {m for (_s, _r, m, _v) in recorded}