fix(db): ride out a brief database outage mid-request; explain it if it persists

The operator's 500 was NOT the startup gap I attributed it to. They found it by
logging in and getting Steward's own error page -- which means create_app had
completed and the app was serving, so the gaierror came from a request handler
acquiring a connection, not from boot. Neither prior fix covers that: the
startup retry never runs, and pool_pre_ping only helps once the database is
back, since its replacement connect fails too while the container is gone.

Adds a before_request gate that acquires a pooled connection with a short
bounded retry (~1.75s over 4 tries) so a database restart is ridden out
invisibly, and renders a distinct 503 "database unavailable" page when the
budget is exhausted. The request budget is deliberately far shorter than the
startup one: nobody watches a container boot, but somebody is watching this
page load, and a page that hangs is worse than one that says what is wrong.

/health and static are exempt -- a liveness probe must stay answerable while
the database is down, or a dependency outage triggers a restart loop.

Credential and missing-database errors still propagate rather than being
reported as "unavailable", which would send the operator chasing the wrong
problem entirely.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>

steward/app.py

@@ -3,9 +3,12 @@ from __future__ import annotations
 import asyncio
 import logging
 from pathlib import Path
-from quart import Quart, render_template
+from quart import Quart, render_template, request
 from .config import load_bootstrap
-from .database import init_db, DB_CONNECT_TIMEOUT_SECONDS
+from .database import (
+    init_db, ensure_database_reachable, DatabaseUnavailable,
+    DB_CONNECT_TIMEOUT_SECONDS,
+)
 
 logger = logging.getLogger(__name__)
 
@@ -245,6 +248,29 @@ def create_app(
     async def health():
         return {"status": "ok"}
 
+    # ── 11b. Database availability gate ────────────────────────────────────────
+    # Every page in Steward reads the database, so a database that has gone away
+    # under a running app turns each request into an opaque 500 (this is how a
+    # login attempt surfaced a bare gaierror). Acquire a connection up front,
+    # retrying briefly to ride out a restart, and answer honestly if it stays
+    # down rather than failing deep inside a handler with a generic error.
+    if not testing:
+        @app.before_request
+        async def _database_gate():
+            # /health is a liveness probe for the container itself — it must
+            # stay answerable while the database is down, or a restart loop
+            # gets triggered by a dependency outage. Static files need no DB.
+            if request.endpoint in ("health", "static"):
+                return None
+            try:
+                await ensure_database_reachable(app._db_engine)
+            except DatabaseUnavailable as exc:
+                logger.error("Database unreachable while serving %s: %s",
+                             request.path, exc)
+                return await render_template(
+                    "errors/database_unavailable.html"), 503
+            return None
+
     # ── 12. Error handlers ─────────────────────────────────────────────────────
     @app.errorhandler(404)
     async def not_found(_):

steward/database.py

@@ -4,7 +4,10 @@ import logging
 import time
 from typing import TYPE_CHECKING
 from sqlalchemy import text
-from sqlalchemy.ext.asyncio import create_async_engine, async_sessionmaker, AsyncSession
+from sqlalchemy.exc import SQLAlchemyError
+from sqlalchemy.ext.asyncio import (
+    create_async_engine, async_sessionmaker, AsyncEngine, AsyncSession,
+)
 from sqlalchemy.pool import NullPool
 
 if TYPE_CHECKING:
@@ -26,6 +29,13 @@ DB_CONNECT_TIMEOUT_SECONDS = 60.0
 _BACKOFF_INITIAL_SECONDS = 0.5
 _BACKOFF_MAX_SECONDS = 5.0
 
+# Per-REQUEST retry schedule (seconds between attempts), total ~1.75s across 4
+# tries. Far shorter than the startup budget on purpose: nobody is watching a
+# container boot, but somebody is watching this page load. Long enough to ride
+# out a pool blip or a fast reconnect, short enough that a genuinely-down
+# database gets an honest answer instead of a spinner.
+_REQUEST_RETRY_DELAYS = (0.25, 0.5, 1.0)
+
 # Postgres SQLSTATEs that will never resolve by waiting — retrying these just
 # delays a clear error behind a full timeout budget.
 _FATAL_SQLSTATES = {
@@ -147,6 +157,48 @@ def wait_for_database(
     ) from last_exc
 
 
+class DatabaseUnavailable(Exception):
+    """The database could not be reached while serving a request."""
+
+
+async def ensure_database_reachable(engine: AsyncEngine) -> None:
+    """Check out one pooled connection, retrying briefly, or raise.
+
+    Startup readiness (wait_for_database) does not help once the app is already
+    serving: if the database goes away underneath a running Steward, the next
+    request needs a connection, the pooled ones are dead, and establishing a new
+    one fails — which is how a login attempt turned into an opaque 500.
+
+    pool_pre_ping already makes recovery automatic *once the database is back*.
+    What it cannot do is wait: while the container is genuinely down, its
+    replacement connect fails too. So retry briefly here to ride out a restart,
+    then give up and let the caller render an honest "database unavailable"
+    page rather than a generic error.
+
+    The budget is deliberately short — a person is waiting on this request, and
+    a page that hangs for half a minute is worse than one that says plainly
+    what is wrong and offers a retry.
+    """
+    last_exc: BaseException | None = None
+    for attempt, delay in enumerate(_REQUEST_RETRY_DELAYS + (None,)):
+        try:
+            async with engine.connect():
+                if attempt:
+                    logger.info(
+                        "Database reachable again after %d retry attempt(s)", attempt)
+                return
+        except (SQLAlchemyError, OSError) as exc:
+            # Bad credentials / missing database will never resolve by waiting,
+            # and they are not what this guard is for — let them surface.
+            if _fatal_sqlstate(exc) is not None:
+                raise
+            last_exc = exc
+            if delay is not None:
+                await asyncio.sleep(delay)
+
+    raise DatabaseUnavailable(_describe(last_exc)) from last_exc
+
+
 async def _probe(db_url: str) -> None:
     """Open one throwaway connection and round-trip a trivial query.
 

steward/templates/errors/database_unavailable.html

@@ -0,0 +1,16 @@
+{% extends "base.html" %}
+{% block title %}Database unavailable — Steward{% endblock %}
+{% block content %}
+<div style="text-align:center; padding: 4rem 2rem; font-family: var(--font-serif);">
+  <h1 style="color: var(--yellow); font-size: 2.5rem; margin-bottom: 0.5rem;">503</h1>
+  <p style="color: var(--text); font-size: 1.2rem; margin-bottom: 0.25rem;">The records are out of reach.</p>
+  <p style="color: var(--text-dim); margin-bottom: 0.5rem; max-width: 34rem; margin-left:auto; margin-right:auto;">
+    Steward is running, but cannot reach its database. Nothing has been lost —
+    this page will work again as soon as the database is back.
+  </p>
+  <p style="color: var(--text-dim); font-size: 0.85rem; margin-bottom: 1.5rem;">
+    Steward already retried for a moment before showing this.
+  </p>
+  <a href="{{ request.path }}" class="btn">Try again</a>
+</div>
+{% endblock %}