fix(db): survive a database restart and a not-yet-ready database

Two connection-recovery gaps, both surfaced by a Postgres restart that left
the app throwing tracebacks while the DB itself was healthy.

pool_pre_ping + pool_recycle on the app engine [#2626]: when the database
restarts, every connection already in the pool is dead at the socket level.
SQLAlchemy only discovered that by failing a real query, so the first
operation after a restart errored out on whatever triggered it. pre_ping
checks liveness on checkout and swaps the dead connection transparently;
pool_recycle caps connection age so a socket stranded by a NAT/conntrack
timeout or a Docker network rebuild is retired on a timer instead.

wait_for_database() gate at startup [#2627]: create_app touches the DB
synchronously (migrations, secret re-encryption, settings load) and assumed
it was both resolvable and accepting connections on the first try. Neither
holds after a host reboot (Docker DNS not yet serving `db` -> gaierror -2)
or an unclean shutdown (Postgres still replaying WAL -> "not yet accepting
connections"). Both are transient, so retry with capped backoff behind one
gate ahead of the first DB touch. Credential and missing-database errors
are classified by SQLSTATE and still fail immediately -- waiting cannot fix
those. Budget is bootstrap-configurable (STEWARD_DB_CONNECT_TIMEOUT /
database.connect_timeout, default 60s) since it governs reaching the DB and
so cannot live in the DB-backed settings.

Tests drive the retry loop off a fake clock, so backoff and timeout
behaviour are deterministic rather than wall-clock dependent.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>

steward/app.py

@@ -5,7 +5,7 @@ import logging
 from pathlib import Path
 from quart import Quart, render_template
 from .config import load_bootstrap
-from .database import init_db
+from .database import init_db, DB_CONNECT_TIMEOUT_SECONDS
 
 logger = logging.getLogger(__name__)
 
@@ -23,6 +23,7 @@ def create_app(
         bootstrap = {
             "database_url": "postgresql+asyncpg://test/test",
             "secret_key": "test-secret-key",
+            "db_connect_timeout": DB_CONNECT_TIMEOUT_SECONDS,
             "plugin_dirs": ["plugins"],
             "plugin_install_dir": "plugins",
         }
@@ -42,6 +43,19 @@ def create_app(
         from unittest.mock import MagicMock
         app.db_sessionmaker = MagicMock()
 
+    # ── 2b. Block until the database is actually reachable ────────────────────
+    # Everything from here down touches the DB synchronously (migrations,
+    # secret re-encryption, settings load). Gate all of it behind one readiness
+    # check so a DB that is merely slow to come up — WAL recovery after an
+    # unclean shutdown, or Docker DNS not yet serving the `db` name after a host
+    # reboot — is waited out instead of crashing the container on a traceback.
+    if not testing:
+        from .database import wait_for_database
+        wait_for_database(
+            app.config["DATABASE_URL"],
+            timeout_seconds=bootstrap["db_connect_timeout"],
+        )
+
     # ── 3. Core migrations only (creates app_settings table) ──────────────────
     if not testing:
         from .core.migration_runner import run_core_migrations

steward/config.py

@@ -49,6 +49,8 @@ def load_bootstrap(config_path: Path | str | None = None) -> dict[str, Any]:
 
     secret_key = _resolve_secret_key(raw)
 
+    db_connect_timeout = _resolve_db_connect_timeout(raw)
+
     # Plugin discovery spans two roots (see load_plugins / migration_runner):
     #   • bundled  — first-party plugins shipped inside the image at repo-root
     #     `plugins/`; they version atomically with core and are read-only at runtime.
@@ -68,12 +70,47 @@ def load_bootstrap(config_path: Path | str | None = None) -> dict[str, Any]:
     return {
         "database_url": database_url,
         "secret_key": secret_key,
+        "db_connect_timeout": db_connect_timeout,
         "plugin_dirs": plugin_dirs,
         # Installs/downloads target the external (writable, persistent) dir.
         "plugin_install_dir": external_plugin_dir or bundled_plugin_dir,
     }
 
 
+def _resolve_db_connect_timeout(raw: dict) -> float:
+    """How long to wait for the database at startup, in seconds.
+
+    Bootstrap-only by necessity: this governs reaching the DB, so it cannot
+    itself be read from the DB like the rest of Steward's settings.
+
+    A deployment whose database is slower to come up than the default (a large
+    cluster replaying WAL, a remote DB behind a link that takes a while) can
+    raise it rather than crash-looping the container.
+    """
+    from .database import DB_CONNECT_TIMEOUT_SECONDS
+
+    value = _env("DB_CONNECT_TIMEOUT") or raw.get("database", {}).get("connect_timeout")
+    if value is None or value == "":
+        return DB_CONNECT_TIMEOUT_SECONDS
+    try:
+        parsed = float(value)
+    except (TypeError, ValueError):
+        logger.warning(
+            "Invalid database connect timeout %r — using default %.0fs",
+            value, DB_CONNECT_TIMEOUT_SECONDS,
+        )
+        return DB_CONNECT_TIMEOUT_SECONDS
+    if parsed <= 0:
+        # 0/negative would mean "never wait", which is the broken behaviour this
+        # setting exists to fix — treat it as a mistake, not as an opt-out.
+        logger.warning(
+            "Database connect timeout %.0fs is not positive — using default %.0fs",
+            parsed, DB_CONNECT_TIMEOUT_SECONDS,
+        )
+        return DB_CONNECT_TIMEOUT_SECONDS
+    return parsed
+
+
 def _resolve_secret_key(raw: dict) -> str:
     """Resolve secret_key: env var → file → auto-generate.
 

steward/database.py

@@ -1,10 +1,39 @@
 from __future__ import annotations
+import asyncio
+import logging
+import time
 from typing import TYPE_CHECKING
+from sqlalchemy import text
 from sqlalchemy.ext.asyncio import create_async_engine, async_sessionmaker, AsyncSession
+from sqlalchemy.pool import NullPool
 
 if TYPE_CHECKING:
     from quart import Quart
 
+logger = logging.getLogger(__name__)
+
+# Recycle pooled connections after 30 minutes. Nothing in Postgres closes an
+# idle connection by default, but the path to it is not always durable: NAT and
+# conntrack tables drop idle flows, and a Docker network rebuild silently
+# strands existing sockets. Capping connection age means a stranded connection
+# is retired on a timer instead of surfacing as a failed query later.
+POOL_RECYCLE_SECONDS = 1800
+
+# Total budget for the database to become reachable at startup, and the backoff
+# ceiling between attempts. 60s comfortably covers Postgres WAL recovery after
+# an unclean shutdown (observed ~7s) plus Docker DNS coming up on a host reboot.
+DB_CONNECT_TIMEOUT_SECONDS = 60.0
+_BACKOFF_INITIAL_SECONDS = 0.5
+_BACKOFF_MAX_SECONDS = 5.0
+
+# Postgres SQLSTATEs that will never resolve by waiting — retrying these just
+# delays a clear error behind a full timeout budget.
+_FATAL_SQLSTATES = {
+    "28P01",  # invalid_password
+    "28000",  # invalid_authorization_specification
+    "3D000",  # invalid_catalog_name — database does not exist
+}
+
 
 def init_db(app: "Quart") -> None:
     """Create async engine and attach db_sessionmaker to app.
@@ -13,8 +42,120 @@ def init_db(app: "Quart") -> None:
     Does not create tables — Alembic handles migrations.
     """
     db_url: str = app.config["DATABASE_URL"]
-    engine = create_async_engine(db_url, echo=False)
+    engine = create_async_engine(
+        db_url,
+        echo=False,
+        # Check a pooled connection is still alive before handing it out. When
+        # the database restarts, every connection already in the pool is dead
+        # at the socket level; without this, SQLAlchemy only discovers that by
+        # failing a real query, so the first operation after a DB restart
+        # errors out on whatever triggered it (a request, a scheduled poll).
+        # The check is a cheap round-trip and it makes a DB restart invisible.
+        pool_pre_ping=True,
+        pool_recycle=POOL_RECYCLE_SECONDS,
+    )
     app.db_sessionmaker: async_sessionmaker[AsyncSession] = async_sessionmaker(
         engine, expire_on_commit=False
     )
     app._db_engine = engine
+
+
+def _fatal_sqlstate(exc: BaseException) -> str | None:
+    """Return the SQLSTATE if this error chain carries a non-retryable one."""
+    seen: set[int] = set()
+    cur: BaseException | None = exc
+    while cur is not None and id(cur) not in seen:
+        seen.add(id(cur))
+        sqlstate = getattr(cur, "sqlstate", None)
+        if sqlstate in _FATAL_SQLSTATES:
+            return sqlstate
+        cur = cur.__cause__ or cur.__context__
+    return None
+
+
+def _describe(exc: BaseException) -> str:
+    """Innermost cause, which is the part that says what actually went wrong.
+
+    SQLAlchemy wraps DBAPI errors several layers deep; the outer message is
+    boilerplate, so surface the root for the waiting-for-database log line.
+    """
+    cur: BaseException = exc
+    seen: set[int] = {id(cur)}
+    while True:
+        nxt = cur.__cause__ or cur.__context__
+        if nxt is None or id(nxt) in seen:
+            return f"{type(cur).__name__}: {cur}"
+        seen.add(id(nxt))
+        cur = nxt
+
+
+def wait_for_database(
+    db_url: str,
+    timeout_seconds: float = DB_CONNECT_TIMEOUT_SECONDS,
+) -> None:
+    """Block until the database accepts a connection, or raise after the budget.
+
+    Steward otherwise assumes the database is both resolvable and *ready* the
+    first time it asks, which is false in two ordinary situations:
+
+      • Host reboot / full stack restart — the app container can try to resolve
+        the `db` service name before Docker's embedded DNS has the record,
+        giving `gaierror -2 Name or service not known`.
+      • Unclean shutdown — Postgres is listening but still replaying WAL, and
+        refuses connections with "the database system is not yet accepting
+        connections" until recovery reaches a consistent state.
+
+    compose's `depends_on: service_healthy` covers ordering on a clean `up`,
+    but not either of the above. Both are transient and self-healing, so retry
+    rather than dumping a traceback and dying. Credential and missing-database
+    errors are NOT transient and fail immediately.
+    """
+    deadline = time.monotonic() + timeout_seconds
+    delay = _BACKOFF_INITIAL_SECONDS
+    attempt = 0
+    last_exc: BaseException | None = None
+
+    while True:
+        attempt += 1
+        try:
+            asyncio.run(_probe(db_url))
+        except Exception as exc:  # broad by design — classified just below
+            sqlstate = _fatal_sqlstate(exc)
+            if sqlstate is not None:
+                # Wrong password / missing database: waiting cannot fix it.
+                raise
+            last_exc = exc
+            remaining = deadline - time.monotonic()
+            if remaining <= 0:
+                break
+            logger.warning(
+                "Database not ready (attempt %d, %.0fs budget left): %s — "
+                "retrying in %.1fs",
+                attempt, remaining, _describe(exc), min(delay, remaining),
+            )
+            time.sleep(min(delay, remaining))
+            delay = min(delay * 2, _BACKOFF_MAX_SECONDS)
+        else:
+            if attempt > 1:
+                logger.info("Database ready after %d attempt(s)", attempt)
+            return
+
+    detail = f" Last error: {_describe(last_exc)}" if last_exc is not None else ""
+    raise RuntimeError(
+        f"Database did not become available within {timeout_seconds:.0f}s "
+        f"({attempt} attempts).{detail}"
+    ) from last_exc
+
+
+async def _probe(db_url: str) -> None:
+    """Open one throwaway connection and round-trip a trivial query.
+
+    Uses its own engine with NullPool: this runs before the app engine exists,
+    and a probe connection must never be left in a pool for real work to reuse.
+    """
+    engine = create_async_engine(db_url, echo=False, poolclass=NullPool)
+    try:
+        async with engine.connect() as conn:
+            await conn.execute(text("SELECT 1"))
+    finally:
+        await engine.dispose()