FabledSteward/tests/core/test_self_monitor.py

"""Unit tests for the self-monitor: open files and Steward's own health (no DB, no network).

Exercises the fd accounting, the percentage/warning floor, and that the
recorder degrades to a no-op where fd accounting isn't available. The DB session
and the alert pipeline are faked — we only assert that the right metrics get
handed to record_metrics.
"""
import asyncio
import logging

import steward.core.alerts as alerts
import steward.core.self_monitor as sm


# ── fd accounting ────────────────────────────────────────────────────────────

def test_count_open_fds_sane_or_none():
    n = sm.count_open_fds()
    # On the Linux CI image /proc exists; stdin/stdout/stderr are always open.
    assert n is None or n >= 3


def test_fd_soft_limit_positive_or_none():
    soft = sm.fd_soft_limit()
    assert soft is None or soft > 0


# ── fake session / app plumbing ──────────────────────────────────────────────

class _Ctx:
    async def __aenter__(self):
        return self

    async def __aexit__(self, *_):
        return False


class _FakeSession(_Ctx):
    last = None   # the latest steward sample the unwatched gap is counted from

    def begin(self):
        return _Ctx()

    async def execute(self, stmt):
        last = self.last
        return type("R", (), {"scalar": staticmethod(lambda: last)})()


class _FakeApp:
    def db_sessionmaker(self):
        return _FakeSession()


def _patch_recorder(monkeypatch):
    recorded: list[tuple] = []

    async def fake_record_metrics(session, rows):
        recorded.extend((r.source_module, r.resource_name, r.metric_name, r.value) for r in rows)

    monkeypatch.setattr(alerts, "record_metrics", fake_record_metrics)
    return recorded


# ── record_self_metrics ──────────────────────────────────────────────────────

def test_records_both_fd_metrics(monkeypatch):
    recorded = _patch_recorder(monkeypatch)
    monkeypatch.setattr(sm, "count_open_fds", lambda: 50)
    monkeypatch.setattr(sm, "fd_soft_limit", lambda: 1000)

    asyncio.run(sm.record_self_metrics(_FakeApp()))

    assert ("steward", "process", "open_fds", 50.0) in recorded
    assert ("steward", "process", "open_fds_pct", 5.0) in recorded


def test_no_pct_metric_when_limit_unknown(monkeypatch):
    recorded = _patch_recorder(monkeypatch)
    monkeypatch.setattr(sm, "count_open_fds", lambda: 50)
    monkeypatch.setattr(sm, "fd_soft_limit", lambda: None)

    asyncio.run(sm.record_self_metrics(_FakeApp()))

    metric_names = [m for (_s, _r, m, _v) in recorded]
    assert "open_fds" in metric_names
    assert "open_fds_pct" not in metric_names   # meaningless without a ceiling


def test_warns_past_floor(monkeypatch, caplog):
    _patch_recorder(monkeypatch)
    monkeypatch.setattr(sm, "count_open_fds", lambda: 900)
    monkeypatch.setattr(sm, "fd_soft_limit", lambda: 1000)   # 90% ≥ FD_WARN_PCT
    monkeypatch.setattr(sm, "_warned", False)

    with caplog.at_level(logging.WARNING, logger=sm.logger.name):
        asyncio.run(sm.record_self_metrics(_FakeApp()))

    assert any("soft limit" in r.message for r in caplog.records)


def test_no_warning_below_floor(monkeypatch, caplog):
    _patch_recorder(monkeypatch)
    monkeypatch.setattr(sm, "count_open_fds", lambda: 100)
    monkeypatch.setattr(sm, "fd_soft_limit", lambda: 1000)   # 10%
    monkeypatch.setattr(sm, "_warned", False)

    with caplog.at_level(logging.WARNING, logger=sm.logger.name):
        asyncio.run(sm.record_self_metrics(_FakeApp()))

    assert not any("soft limit" in r.message for r in caplog.records)


def test_without_fd_accounting_the_rest_is_still_recorded(monkeypatch):
    # /proc absent: no open-file pair, but Steward's other health still lands.
    recorded = _patch_recorder(monkeypatch)
    monkeypatch.setattr(sm, "count_open_fds", lambda: None)

    asyncio.run(sm.record_self_metrics(_FakeApp()))

    names = {m for (_s, _r, m, _v) in recorded}
    assert "db_roundtrip_ms" in names and not names & {"open_fds", "open_fds_pct"}


def test_the_minute_s_ingests_are_recorded_and_the_tally_starts_again(monkeypatch):
    """#5467: posts, failures and the slowest post since the last tick."""
    from steward.core import health
    recorded = _patch_recorder(monkeypatch)
    monkeypatch.setattr(sm, "count_open_fds", lambda: 50)
    monkeypatch.setattr(health, "ingest", health.IngestTally())
    for ms, ok in ((300.0, True), (9800.0, False), (450.0, True)):
        health.ingest.done(ms, ok)

    asyncio.run(sm.record_self_metrics(_FakeApp()))

    got = {m: v for (s, r, m, v) in recorded if (s, r) == ("steward", "process")}
    assert (got["ingest_posts"], got["ingest_failed"], got["ingest_slowest_ms"]) == (3.0, 1.0, 9800.0)
    assert got["unwatched_secs"] == 0.0 and got["db_roundtrip_ms"] >= 0
    assert health.ingest.take() == health.IngestTally()   # taken: the next minute starts at zero


def test_unwatched_counts_only_a_real_gap():
    from datetime import datetime, timedelta, timezone
    now = datetime(2026, 10, 9, 12, 0, tzinfo=timezone.utc)
    assert sm.unwatched_secs(None, now) == 0.0                          # first ever tick
    assert sm.unwatched_secs(now - timedelta(seconds=61), now) == 0.0   # an ordinary minute
    assert sm.unwatched_secs(now - timedelta(minutes=7), now) == 420.0  # a deploy or outage


def test_a_fresh_process_marks_its_restart_once(monkeypatch):
    """#5862: the first tick records how long since the last process's last
    sample, however short; later ticks and a first-ever start don't."""
    from datetime import datetime, timedelta, timezone
    recorded = _patch_recorder(monkeypatch)
    monkeypatch.setattr(sm, "_started", False)
    monkeypatch.setattr(_FakeSession, "last", datetime.now(timezone.utc) - timedelta(seconds=40))

    asyncio.run(sm.record_self_metrics(_FakeApp()))
    asyncio.run(sm.record_self_metrics(_FakeApp()))

    gaps = [v for (_s, _r, m, v) in recorded if m == "restart_gap_secs"]
    assert len(gaps) == 1 and 39 <= gaps[0] <= 60
    assert dict((m, v) for (_s, _r, m, v) in recorded)["unwatched_secs"] == 0.0

    recorded.clear()
    monkeypatch.setattr(sm, "_started", False)
    monkeypatch.setattr(_FakeSession, "last", None)
    asyncio.run(sm.record_self_metrics(_FakeApp()))
    assert "restart_gap_secs" not in {m for (_s, _r, m, _v) in recorded}