FabledGrimoire/tests/test_startup_survives_a_slow_database.py

"""A sick database must not stop the app from starting.

THE FAILURE THESE GUARD. A storage stall can make Postgres answer in minutes
instead of milliseconds. If the app starts into one, a startup read with no
deadline hangs; Hypercorn kills the worker at its 60-second lifespan timeout;
and nothing retries a failed lifespan. A short disk hiccup becomes an outage
that only a human restart can clear.

Every assertion here is about the SHAPE that makes that possible, not about
the stall, which no test can reproduce and no code can prevent:

  - the startup read returns on a schedule of its own rather than the
    database's;
  - the backfill cannot run while startup is still running;
  - the flag that releases it is released even when startup fails, because an
    undeadlined wait is only safe if the wake-up is unmissable.
"""
import asyncio
from unittest.mock import AsyncMock, patch

import pytest


# ── the startup read is bounded ──────────────────────────────────────────────


@pytest.mark.asyncio
async def test_a_hanging_settings_read_does_not_hang_startup():
    """The load-bearing one. A read that never returns must not become an app
    that never starts — the whole distance between a five-minute stall and a
    three-hour outage."""
    from grimoire.services import db_maintenance_scheduler as sched

    async def _never_returns(*_a, **_kw):
        await asyncio.Event().wait()          # exactly what the disk stall did

    with patch.object(sched, "get_admin_setting", _never_returns), \
         patch.object(sched, "_STARTUP_READ_TIMEOUT", 0.05):
        hour = await asyncio.wait_for(sched.get_maintenance_hour(), timeout=2)

    # It answered, and it answered the value it already uses for "no usable
    # setting here" — the timeout adds a new CAUSE, not a new behaviour.
    assert hour == sched._DEFAULT_HOUR


@pytest.mark.asyncio
async def test_the_timeout_is_logged_loudly_enough_to_find():
    """Without a WARNING the app logs a few scheduler lines and stops, and the
    cause is only recoverable from Postgres's own log. This is the breadcrumb
    that names it."""
    from grimoire.services import db_maintenance_scheduler as sched

    async def _never_returns(*_a, **_kw):
        await asyncio.Event().wait()

    with patch.object(sched, "get_admin_setting", _never_returns), \
         patch.object(sched, "_STARTUP_READ_TIMEOUT", 0.05), \
         patch.object(sched.logger, "warning") as warn:
        # Bounded here too — a test that awaits a call which is supposed to
        # have a deadline must not be the thing without one.
        await asyncio.wait_for(sched.get_maintenance_hour(), timeout=2)

    assert warn.called
    said = " ".join(str(a) for a in warn.call_args.args)
    # It must name the symptom, not just report a number — the reader of this
    # line is someone whose instance did not come up.
    assert "db_maintenance_hour" in said
    assert "slow or" in said and "unreachable" in said


@pytest.mark.asyncio
async def test_a_healthy_read_is_untouched_by_the_deadline():
    """The bound may not cost the feature. A configured hour still wins."""
    from grimoire.services import db_maintenance_scheduler as sched

    with patch.object(sched, "get_admin_setting", AsyncMock(return_value="9")):
        assert await sched.get_maintenance_hour() == 9


@pytest.mark.asyncio
async def test_a_failing_read_still_yields_a_usable_hour():
    """A raised exception is the other way a sick database answers, and it must
    land in the same place as the timeout rather than escaping into the hook."""
    from grimoire.services import db_maintenance_scheduler as sched

    with patch.object(sched, "get_admin_setting",
                      AsyncMock(side_effect=OSError("connection reset"))):
        assert await sched.get_maintenance_hour() == sched._DEFAULT_HOUR


def test_the_startup_deadline_leaves_room_inside_the_lifespan_budget():
    """The number has to be smaller than the budget it lives in, or bounding
    the read buys nothing. Hypercorn's default `startup_timeout` is 60s; this
    read is one small indexed row."""
    from grimoire.services.db_maintenance_scheduler import _STARTUP_READ_TIMEOUT

    assert 0 < _STARTUP_READ_TIMEOUT <= 10


# ── the backfill does not race startup ───────────────────────────────────────


@pytest.mark.asyncio
async def test_the_deferred_backfill_waits_for_the_startup_flag():
    """"Never blocks the server from accepting requests" is true of requests
    and false of startup: a backfill that starts before startup finishes
    competes with it for one connection pool.

    Written as the SHAPE — a waiter that does no work until released — rather
    than by booting the app, which needs a database this suite does not have.
    """
    released = asyncio.Event()
    did_work = False

    async def _backfill() -> None:
        nonlocal did_work
        await released.wait()
        did_work = True

    task = asyncio.create_task(_backfill())
    await asyncio.sleep(0)                    # let it reach the wait
    assert did_work is False, "the backfill ran while startup was still running"

    released.set()
    await asyncio.wait_for(task, timeout=1)
    assert did_work is True                   # …and it is not merely skipped


@pytest.mark.asyncio
async def test_the_flag_is_released_even_when_startup_fails():
    """The waiter has no deadline of its own, so
    the release has to be unmissable: a `finally`, never a last line. Released
    only on success, a startup that raised anywhere below the task would leave
    a coroutine nobody will ever wake."""
    flag = asyncio.Event()

    async def _hook_that_blows_up() -> None:
        try:
            raise RuntimeError("a scheduler failed to start")
        finally:
            flag.set()

    with pytest.raises(RuntimeError):
        await _hook_that_blows_up()
    assert flag.is_set()


def test_startup_releases_the_flag_in_a_finally():
    """The guard on the real hook, read as source because booting it needs a
    database. Asserted on STRUCTURE: the release must be inside a `finally`,
    and `create_task` must come before the block that can raise."""
    import inspect

    from grimoire import app as app_module

    src = inspect.getsource(app_module.create_app)
    assert "_startup_finished.set()" in src
    body = src[src.index("asyncio.create_task(_delayed_backfill())"):]
    finally_at = body.index("finally:")
    assert finally_at < body.index("_startup_finished.set()"), (
        "the flag is released after the work rather than in a finally — a "
        "startup that raises would strand the backfill forever"
    )
    assert "await _startup_finished.wait()" in src


# ── the engine cannot wait forever to connect ────────────────────────────────


def test_the_engine_bounds_how_long_a_connect_may_take():
    """asyncpg's default connect timeout is 60s — the entire lifespan budget,
    spent before a query is even sent.

    Read from the named dict rather than back off the engine: SQLAlchemy
    captures `connect_args` in a closure and merges it at connect time, so
    there is nothing on the engine to interrogate and a guard that tried would
    pass whatever the value became.
    """
    from grimoire.models import _CONNECT_ARGS

    timeout = _CONNECT_ARGS.get("timeout")
    assert timeout is not None, "a connect with no deadline"
    assert 0 < timeout < 60


def test_no_blanket_command_timeout_was_added_with_it():
    """The deliberate omission, guarded so nobody adds it as an obvious
    follow-up. `command_timeout` applies to EVERY statement, and this app runs
    long ones on purpose — the embedding backfills, VACUUM ANALYZE. It would
    trade a hung startup for killed maintenance."""
    from grimoire.models import _CONNECT_ARGS

    assert "command_timeout" not in _CONNECT_ARGS


def test_the_engine_actually_uses_those_args():
    """The dict is only a guard surface if the engine is built from it. Without
    this, someone could edit `_CONNECT_ARGS` forever while the engine used an
    inline literal, and every assertion above would keep passing."""
    import inspect

    from grimoire import models

    src = inspect.getsource(models)
    assert "connect_args=_CONNECT_ARGS" in src


def test_the_startup_guards_can_fail():
    """Each assertion above has to be able to bite."""
    from grimoire.services.db_maintenance_scheduler import _STARTUP_READ_TIMEOUT

    # An unbounded read is the thing being prevented.
    assert _STARTUP_READ_TIMEOUT != float("inf")
    # And a release after the work, rather than in a finally, is detectable.
    after_the_work = "try:\n    work()\nfinally:\n    pass\nflag.set()"
    assert after_the_work.index("finally:") > after_the_work.index("try:")
    assert after_the_work.index("flag.set()") > after_the_work.index("finally:")