FabledSteward/tests/core/test_outbox.py

"""The email outbox's decisions (#5506): which failures are worth another
try, how long to wait, when to give up, and what one connection's batch
reports per message. The table and the sender's ticks are in
tests/integration/test_email_outbox.py."""
import smtplib
from datetime import datetime, timedelta, timezone
from email.message import EmailMessage
from zoneinfo import ZoneInfo

import pytest

from steward.core import mail, outbox
from steward.models.outbox import EmailOutbox

NOW = datetime(2026, 10, 9, 12, 0, tzinfo=timezone.utc)


def _row(**kw) -> EmailOutbox:
    base = dict(kind="report", recipient="ann@example.com", retry=True, attempts=0,
                state="sending", first_attempt_at=None, claimed_at=NOW)
    return EmailOutbox(**{**base, **kw})


@pytest.mark.parametrize("exc, kind", [
    (smtplib.SMTPRecipientsRefused({"a@x": (550, b"no such user")}), "permanent"),
    (smtplib.SMTPRecipientsRefused({"a@x": (550, b"no"), "b@x": (451, b"later")}), "temporary"),
    (smtplib.SMTPDataError(554, b"rejected as spam"), "permanent"),
    (smtplib.SMTPSenderRefused(553, b"not yours", "steward@x"), "permanent"),
    (smtplib.SMTPAuthenticationError(535, b"bad credentials"), "permanent"),
    (smtplib.SMTPDataError(451, b"greylisted, try later"), "temporary"),
    (smtplib.SMTPServerDisconnected("gone"), "temporary"),
    (ConnectionRefusedError(111, "refused"), "temporary"),
    (TimeoutError(), "temporary"),
    (smtplib.SMTPNotSupportedError("STARTTLS extension not supported"), "permanent"),
    (ValueError("no words"), "permanent"),
])
def test_a_refusal_is_final_and_anything_that_may_pass_is_retried(exc, kind):
    assert outbox.classify(exc) == kind


def test_backoff_grows_then_holds_at_its_last_step():
    assert [outbox.backoff(n) for n in range(1, 9)] == [30, 60, 120, 300, 600, 900, 900, 900]


def test_reason_is_what_the_server_said():
    assert outbox.reason(smtplib.SMTPDataError(554, b"5.7.1  rejected\r\n as spam")) \
        == "554 5.7.1 rejected as spam"
    assert outbox.reason(smtplib.SMTPRecipientsRefused({"a@x": (550, b"no such user")})) \
        == "550 no such user"
    assert outbox.reason(smtplib.SMTPRecipientsRefused({"a@x": (550, b"no"), "b@x": (450, b"busy")})) \
        == "a@x: 550 no; b@x: 450 busy"
    assert outbox.reason(TimeoutError()) == "The mail server didn't answer in time"


def test_accepted_is_sent():
    row = _row()
    outbox.settle(row, None, NOW)
    assert (row.state, row.sent_at, row.attempts, row.claimed_at) == ("sent", NOW, 1, None)
    assert row.first_attempt_at == NOW and row.next_attempt_at is None


def test_a_refusal_fails_at_once_with_the_servers_words():
    row = _row()
    outbox.settle(row, smtplib.SMTPRecipientsRefused({"ann@example.com": (550, b"no such user")}), NOW)
    assert row.state == "failed" and row.next_attempt_at is None
    assert row.last_error == "550 no such user"


def test_a_temporary_failure_waits_and_tries_again():
    row = _row()
    outbox.settle(row, smtplib.SMTPDataError(451, b"try later"), NOW)
    assert row.state == "pending" and row.next_attempt_at == NOW + timedelta(seconds=30)
    later = NOW + timedelta(seconds=30)
    outbox.settle(row, ConnectionRefusedError(), later)
    assert row.attempts == 2 and row.next_attempt_at == later + timedelta(seconds=60)
    assert row.first_attempt_at == NOW


def test_the_last_try_lands_at_the_hour_and_then_it_fails():
    row = _row(attempts=7, first_attempt_at=NOW)
    near = NOW + timedelta(seconds=3300)
    outbox.settle(row, TimeoutError(), near)
    # 900 s would overshoot the hour: the try is pulled back to it.
    assert row.state == "pending" and row.next_attempt_at == NOW + timedelta(seconds=3600)
    outbox.settle(row, TimeoutError(), NOW + timedelta(seconds=3600))
    assert row.state == "failed"
    assert row.last_error == "The mail server didn't answer in time (gave up after an hour of retries)"


def test_a_test_send_settles_on_its_first_answer():
    row = _row(kind="test", retry=False)
    outbox.settle(row, ConnectionRefusedError(), NOW)
    assert row.state == "failed" and row.last_error == "The mail server refused the connection"


def test_enqueue_keeps_the_readers_zone_and_holds_until_send_after():
    added = []

    class _Db:
        add = added.append

    ann = mail.Recipient("ann@example.com", ZoneInfo("America/New_York"), True, "ann")
    now_row = outbox.enqueue(_Db, kind="report", to=ann, subject="s", text="t", now=NOW)
    assert (now_row.state, now_row.next_attempt_at, now_row.tz, now_row.show_utc) == \
        ("pending", NOW, "America/New_York", True)
    held = outbox.enqueue(_Db, kind="alert", to="bob@example.com",
                          send_after=NOW + timedelta(minutes=2), now=NOW)
    assert (held.state, held.next_attempt_at, held.tz) == ("held", NOW + timedelta(minutes=2), "UTC")
    assert added == [now_row, held]


class _FakeSMTP:
    def __init__(self, answers):
        self.answers, self.sent = list(answers), []

    def __enter__(self):
        return self

    def __exit__(self, *exc):
        return False

    def send_message(self, msg):
        answer = self.answers.pop(0)
        if answer is not None:
            raise answer
        self.sent.append(msg)


def _msgs(n):
    out = []
    for i in range(n):
        m = EmailMessage()
        m["Subject"] = f"m{i}"
        out.append(m)
    return out


def test_one_connection_reports_each_message(monkeypatch):
    from steward.core import notifications
    smtp = _FakeSMTP([None, smtplib.SMTPRecipientsRefused({"b@x": (550, b"no")}), None])
    monkeypatch.setattr(notifications, "smtp_open", lambda cfg: smtp)
    out = outbox.deliver_sync({}, _msgs(3))
    assert out[0] is None and isinstance(out[1], smtplib.SMTPRecipientsRefused) and out[2] is None
    assert len(smtp.sent) == 2


def test_a_dropped_line_is_every_unsent_messages_outcome(monkeypatch):
    from steward.core import notifications
    gone = smtplib.SMTPServerDisconnected("gone")
    monkeypatch.setattr(notifications, "smtp_open", lambda cfg: _FakeSMTP([None, gone]))
    out = outbox.deliver_sync({}, _msgs(4))
    assert out[0] is None and out[1] is gone
    assert all(isinstance(o, smtplib.SMTPServerDisconnected) for o in out[2:]) and len(out) == 4


def test_no_connection_is_every_messages_outcome(monkeypatch):
    from steward.core import notifications
    refused = ConnectionRefusedError(111, "refused")

    def _open(cfg):
        raise refused

    monkeypatch.setattr(notifications, "smtp_open", _open)
    assert outbox.deliver_sync({}, _msgs(2)) == [refused, refused]


CFG = {"host": "mx", "port": 587, "from": "alerts@example.com", "base_url": "https://steward.example/"}
ANN = mail.Recipient("ann@example.com", ZoneInfo("America/New_York"), False, "ann")


def _alert(host: str, state: str = "FIRING", value: float = 400.0) -> dict:
    return {"rule_name": f"Agent offline: {host}", "state": state, "metric": "seconds_since_seen",
            "value": value, "operator": ">", "threshold": 180.0, "resource": host,
            "source_module": "host_agent", "note": None, "timestamp": "2026-10-09T12:00:00+00:00"}


def test_one_alert_alone_keeps_its_usual_email():
    assert outbox.alert_words(CFG, [_alert("web01")], ANN) == mail.alert_email(CFG, _alert("web01"), ANN)


def test_several_alerts_are_one_email_listing_each_oldest_first():
    burst = [_alert("web01"), _alert("web02"), _alert("db01", "RESOLVED", 12.0)]
    subject, text, html = outbox.alert_words(CFG, burst, ANN)
    assert subject == "[Steward] 3 alerts: 2 firing, 1 resolved"
    for host in ("web01", "web02", "db01"):
        assert f"Agent offline: {host}" in text and f"Agent offline: {host}" in html
    assert text.index("web01") < text.index("web02") < text.index("db01")
    assert "Fri 9 Oct 2026 08:00 EDT" in text                                # the reader's zone


def test_said_names_who_got_it_who_refused_and_who_is_still_being_tried():
    rows = [_row(recipient="a@x", state="sent"),
            _row(recipient="b@x", state="failed", last_error="550 no such user"),
            _row(recipient="c@x", state="pending", last_error="451 greylisted")]
    ok, said = outbox.said(rows)
    assert not ok
    assert said == ("Sent to a@x. Not sent to b@x: 550 no such user. Not sent yet to c@x "
                    "(451 greylisted); Steward will try again for up to an hour.")
    assert outbox.said(rows[:1]) == (True, "Sent to a@x.")
    assert outbox.said([])[0] is False


# The guard: every email goes through the queue (System 156 ruling), so the
# only code that opens a connection to the mail server is the outbox's.
def _smtp_openers(sources: dict[str, str]) -> set[str]:
    import re
    opens = re.compile(r"\bsmtp_open\(|\bsmtplib\.SMTP(_SSL)?\(")
    return {path for path, src in sources.items()
            if any(opens.search(line) and not line.lstrip().startswith(("#", "def "))
                   for line in src.splitlines())}


def test_only_the_outbox_talks_to_the_mail_server():
    from pathlib import Path
    root = Path(__file__).resolve().parents[2]          # the checkout
    sources = {str(p.relative_to(root)): p.read_text() for p in (root / "steward").rglob("*.py")}
    sources.update({str(p.relative_to(root)): p.read_text() for p in (root / "plugins").rglob("*.py")}
                   if (root / "plugins").is_dir() else {})
    assert _smtp_openers(sources) == {"steward/core/outbox.py", "steward/core/notifications.py"}


def test_the_smtp_guard_can_fail():
    assert _smtp_openers({"x.py": "    with smtplib.SMTP(host) as s:"}) == {"x.py"}
    assert _smtp_openers({"y.py": "    smtp = notifications.smtp_open(cfg)"}) == {"y.py"}
    assert _smtp_openers({"z.py": "# smtp_open(cfg) is the outbox's\ndef smtp_open(cfg):"}) == set()


# ── How an alert's email reads in history (#5509) ────────────────────────────

def test_an_alerts_email_reads_by_its_least_finished_copy():
    later, sent_at = NOW + timedelta(minutes=2), NOW + timedelta(seconds=5)
    held = _row(state="held", send_after=later, created_at=NOW)
    retrying = _row(recipient="b@x", state="pending", attempts=3, next_attempt_at=later,
                    last_error="451 greylisted", created_at=NOW)
    sent = _row(recipient="c@x", state="sent", attempts=1, sent_at=sent_at, created_at=NOW)
    failed = _row(recipient="d@x", state="failed", attempts=1, first_attempt_at=NOW,
                  last_error="550 no such user", created_at=NOW)
    first = _row(recipient="e@x", state="pending", attempts=0, next_attempt_at=NOW, created_at=NOW)
    assert outbox.delivery([sent, held, failed]) == {"state": "held", "at": later, "attempts": 0, "error": None}
    assert outbox.delivery([sent, retrying]) == {"state": "retrying", "at": later, "attempts": 3,
                                                 "error": "451 greylisted"}
    assert outbox.delivery([first])["state"] == "sending"
    assert outbox.delivery([sent, failed]) == {"state": "failed", "at": NOW, "attempts": 1,
                                               "error": "d@x: 550 no such user"}
    assert outbox.delivery([sent]) == {"state": "sent", "at": sent_at, "attempts": 1, "error": None}