FabledSteward/tests/core/test_steward_health.py

"""Settings → System (#5467): Steward's own health, a verdict then each measure
now and over the day. The verdict names what is wrong, and only that."""
from datetime import datetime, timezone

import pytest
from quart import render_template, session

from steward.core import health

AT = datetime(2026, 10, 9, 12, 0, tzinfo=timezone.utc)
STORAGE = {"total": 3 * 2**30, "rest": {"count": 0, "bytes": 0},
           "tables": [{"name": "plugin_metrics", "bytes": 2 * 2**30, "rows": 41_000_000},
                      {"name": "docker_logs", "bytes": 512 * 2**20, "rows": None}]}


NO_EMAIL = {"rows": [], "more": 0, "sent": 0, "problems": []}


async def _page(app, ctx, storage=STORAGE, cleanup=None, email=NO_EMAIL):
    from steward.core.outbox import KIND_LABELS
    async with app.test_request_context("/settings/system/"):
        session["user_role"] = "admin"
        page = await render_template("settings/system.html", **ctx, storage=storage, cleanup=cleanup,
                                     email=email, kind_labels=KIND_LABELS)
    return " ".join(page.split())


def _previews(**over):
    base = {m: {"now": 0.0, "at": AT, "series": [0.0, 0.0], "low": 0.0, "high": 0.0}
            for m in health.PROCESS_METRICS}
    base["db_roundtrip_ms"] = {"now": 4.0, "at": AT, "series": [3.0, 5.0], "low": 2.0, "high": 9.0}
    for m, v in over.items():
        base[m] = {**base[m], **v}
    return base


@pytest.fixture
def previews(monkeypatch):
    def use(**over):
        got = _previews(**over)

        async def fake(db, source, resource, metric, now=None):
            assert (source, resource) == ("steward", "process")
            return got[metric]
        monkeypatch.setattr("steward.alerts.service.metric_preview", fake)
    return use


async def test_a_healthy_steward_has_no_problems(previews):
    previews()
    got = await health.process_health(None)
    assert got["seen"] and got["problems"] == []
    assert [r["metric"] for r in got["rows"]] == health.PROCESS_METRICS


async def test_failures_and_gaps_are_named(previews):
    previews(ingest_failed={"now": 2.0, "high": 2.0}, unwatched_secs={"high": 420.0},
             open_fds_pct={"now": 85.0})
    problems = (await health.process_health(None))["problems"]
    assert problems[0] == "2 agent posts failed in the last minute"
    assert "wasn't watching for up to 7m" in problems[1]
    assert problems[2] == "open files at 85% of the limit"


async def test_the_page_reads_as_a_verdict_then_each_measure(app, previews):
    previews(unwatched_secs={"high": 420.0})
    got = await health.process_health(None)
    for r in got["rows"]:
        r["spark"] = ""
    page = await _page(app, got)
    assert '<span class="tone-warn">Steward wasn&#39;t watching for up to 7m' in page
    assert 'title="<' not in page          # a <time> element never lands inside an attribute
    assert "Database round trip" in page and "4 ms" in page and "2 ms – 9 ms" in page
    assert page.index("page-verdict") < page.index("Agent posts per minute")


# ── Storage and the last cleanup (#5469) ─────────────────────────────────────

async def test_tables_largest_first_then_what_the_cleanup_removed(app, previews):
    previews()
    got = await health.process_health(None)
    for r in got["rows"]:
        r["spark"] = ""
    cleanup = {"at": AT, "secs": 12.5, "passes": 14,
               "removed": [("plugin_metrics: raw samples rolled into hourly", 86_400)]}
    page = await _page(app, got, cleanup=cleanup)
    assert "The database takes 3.0 GiB on disk." in page
    assert page.index(">plugin_metrics<") < page.index(">docker_logs<")
    assert "41,000,000" in page and "512.0 MiB" in page
    assert "took 12s" in page and "86,400" in page
    assert page.index("Steward itself") < page.index("Storage") < page.index("Last cleanup")


async def test_a_quiet_cleanup_and_one_not_yet_run_each_say_so(app, previews):
    previews()
    got = await health.process_health(None)
    for r in got["rows"]:
        r["spark"] = ""
    quiet = await _page(app, got, cleanup={"at": AT, "secs": 1.0, "passes": 14, "removed": []})
    assert "Nothing was past its window (14 passes)." in quiet
    assert "None has finished since this version started" in await _page(app, got)


async def test_the_last_cleanup_keeps_only_what_went(monkeypatch):
    stored = {"at": "2026-10-09T12:00:00+00:00", "secs": 3.2,
              "removed": [["audit_events: older than 365 days", 0], ["docker_logs: past the age window", 7]]}

    async def fake(db, key):
        assert key == "cleanup.last_run"
        return stored
    monkeypatch.setattr("steward.core.settings.get_setting", fake)
    got = await health.last_cleanup(None)
    assert got["at"] == AT and got["passes"] == 2
    assert got["removed"] == [("docker_logs: past the age window", 7)]
    stored = {}
    assert await health.last_cleanup(None) is None


# ── Outgoing email (#5509) ────────────────────────────────────────────────────

def _mail(**kw):
    from steward.models.outbox import EmailOutbox
    base = dict(kind="alert", recipient="ann@example.com", state="pending", attempts=0,
                created_at=AT, send_after=None, next_attempt_at=None, last_error=None)
    return EmailOutbox(**{**base, **kw})


async def test_outgoing_email_lists_what_has_not_gone_failures_first(app, previews):
    previews()
    got = await health.process_health(None)
    for r in got["rows"]:
        r["spark"] = ""
    email = {"sent": 1234, "more": 3, "problems": ["1 email failed in the last 7 days"], "rows": [
        _mail(state="failed", kind="invite", recipient="bob@example.com", attempts=1,
              last_error="550 no such user"),
        _mail(state="pending", attempts=3, next_attempt_at=AT, last_error="451 greylisted"),
        _mail(state="held", kind="alert", send_after=AT)]}
    got["problems"] = email["problems"] + got["problems"]
    page = await _page(app, got, email=email)
    # The verdict leads with it; the section says each one's state and why.
    assert '<p class="page-verdict"><span class="tone-warn">1 email failed in the last 7 days.' in page
    assert "1,234 sent in the last 7 days." in page and "Nothing waiting" not in page
    assert page.index("bob@example.com") < page.index("451 greylisted")
    assert "Invite" in page and "550 no such user" in page
    assert "retrying</span> <span class=\"tone-dim\">attempt 3, next" in page
    assert "gathering</span> <span class=\"tone-dim\">sends" in page
    assert "+3 more not yet sent" in page
    assert 'title="<' not in page
    assert page.index("Steward itself") < page.index("Outgoing email") < page.index("Storage")


async def test_nothing_waiting_says_so(app, previews):
    previews()
    got = await health.process_health(None)
    for r in got["rows"]:
        r["spark"] = ""
    page = await _page(app, got, email={**NO_EMAIL, "sent": 2})
    assert "Nothing waiting to send. 2 sent in the last 7 days." in page
    assert "Steward is running normally." in page