bvandeusenandClaude Opus 5.5
cba2c24612
Brings the three compose files in line with rule 198 (milestone 544, step 6203, at the operator's request): - no healthcheck on the app or the database, so a slow pg_isready on stalled storage can no longer get a working database killed; - no deploy.restart_policy in the Swarm file, so Swarm's default (condition: any) restarts clean exits too, instead of on-failure with max_attempts leaving the app down; - depends_on is plain start order; restart: unless-stopped stays on the plain-Compose files (Swarm ignores it), and the dev app gains it. The orchestrator used to be what retried a container that started before Postgres was ready. That is now the app's job: alembic/env.py calls services/db_ready.wait_for_database before migrating. It retries refused connections, unresolved hosts, connect timeouts and "the database system is starting up" with 1-2-4-...-10s spacing for up to 300s, then exits non-zero; a wrong password or missing database fails at once. Per-request recovery is the engine's pool_pre_ping. Also fixes the dev compose database image: postgres:16-alpine has no vector extension, so migrations could not run against it. It now uses pgvector/pgvector:pg17 like the other two files. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
101 lines
3.1 KiB
Python
101 lines
3.1 KiB
Python
"""The startup wait for the database: retries what clears up, fails what won't."""
|
|
|
|
import asyncpg
|
|
import pytest
|
|
|
|
from scribe.services.db_ready import asyncpg_dsn, wait_for_database
|
|
|
|
URL = "postgresql+asyncpg://scribe:p%40ss@db:5432/scribe"
|
|
|
|
|
|
class _Clock:
|
|
"""A fake monotonic clock that only moves when the fake sleep is awaited."""
|
|
|
|
def __init__(self):
|
|
self.now = 0.0
|
|
self.sleeps: list[float] = []
|
|
|
|
def __call__(self) -> float:
|
|
return self.now
|
|
|
|
async def sleep(self, seconds: float) -> None:
|
|
self.sleeps.append(seconds)
|
|
self.now += seconds
|
|
|
|
|
|
class _Conn:
|
|
closed = False
|
|
|
|
async def close(self):
|
|
self.closed = True
|
|
|
|
|
|
def _connect_failing(errors):
|
|
"""A connect that raises each error in turn, then succeeds."""
|
|
calls = []
|
|
conn = _Conn()
|
|
|
|
async def connect(dsn, timeout):
|
|
calls.append(dsn)
|
|
if len(calls) <= len(errors):
|
|
raise errors[len(calls) - 1]
|
|
return conn
|
|
|
|
return connect, calls, conn
|
|
|
|
|
|
def test_dsn_drops_the_driver_and_keeps_the_escaped_password():
|
|
assert asyncpg_dsn(URL) == "postgresql://scribe:p%40ss@db:5432/scribe"
|
|
|
|
|
|
async def test_returns_at_once_when_the_database_is_up():
|
|
clock = _Clock()
|
|
connect, calls, conn = _connect_failing([])
|
|
await wait_for_database(URL, connect=connect, sleep=clock.sleep, clock=clock)
|
|
assert len(calls) == 1
|
|
assert clock.sleeps == []
|
|
assert conn.closed
|
|
|
|
|
|
async def test_waits_through_a_database_that_is_still_starting():
|
|
clock = _Clock()
|
|
connect, calls, conn = _connect_failing([
|
|
ConnectionRefusedError("refused"),
|
|
OSError("Name or service not known"),
|
|
asyncpg.CannotConnectNowError("the database system is starting up"),
|
|
])
|
|
await wait_for_database(URL, connect=connect, sleep=clock.sleep, clock=clock)
|
|
assert len(calls) == 4
|
|
assert clock.sleeps == [1.0, 2.0, 4.0]
|
|
assert conn.closed
|
|
|
|
|
|
async def test_backoff_is_capped_at_ten_seconds():
|
|
clock = _Clock()
|
|
connect, _, _ = _connect_failing([ConnectionRefusedError()] * 6)
|
|
await wait_for_database(URL, connect=connect, sleep=clock.sleep, clock=clock)
|
|
assert clock.sleeps == [1.0, 2.0, 4.0, 8.0, 10.0, 10.0]
|
|
|
|
|
|
async def test_gives_up_with_the_last_error_once_the_deadline_passes():
|
|
clock = _Clock()
|
|
connect, calls, _ = _connect_failing([ConnectionRefusedError("still down")] * 100)
|
|
with pytest.raises(ConnectionRefusedError, match="still down"):
|
|
await wait_for_database(
|
|
URL, deadline_s=30, connect=connect, sleep=clock.sleep, clock=clock,
|
|
)
|
|
# 25s waited; another 10s would pass the deadline, so it raises instead.
|
|
assert clock.sleeps == [1.0, 2.0, 4.0, 8.0, 10.0]
|
|
assert len(calls) == len(clock.sleeps) + 1
|
|
|
|
|
|
async def test_a_wrong_password_fails_at_once():
|
|
clock = _Clock()
|
|
connect, calls, _ = _connect_failing([
|
|
asyncpg.InvalidPasswordError("password authentication failed"),
|
|
])
|
|
with pytest.raises(asyncpg.InvalidPasswordError):
|
|
await wait_for_database(URL, connect=connect, sleep=clock.sleep, clock=clock)
|
|
assert len(calls) == 1
|
|
assert clock.sleeps == []
|