ops: wait for the database on startup (no crash-loop on a slow DB)
New thoughtsync.dbwait polls the DB (SELECT 1) up to 60×1s before startup, logging each attempt, so a briefly slow/unready database no longer crash- loops the container. Wired as `python -m thoughtsync.dbwait &&` ahead of `alembic upgrade head` in the image CMD and the dev compose command; exits non-zero after the window so a restart policy can take over. Prod compose gains restart: unless-stopped as the complementary piece. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01FRgehjoz7Yv8LkUfADxACm
This commit is contained in:
+4
-3
@@ -30,6 +30,7 @@ ARG BUILD_VERSION=dev
|
|||||||
ENV APP_VERSION=$BUILD_VERSION
|
ENV APP_VERSION=$BUILD_VERSION
|
||||||
|
|
||||||
EXPOSE 5000
|
EXPOSE 5000
|
||||||
# Run migrations, then serve. Family convention (rule 82): schema is built by real
|
# Wait for the database, run migrations, then serve. The DB wait keeps a briefly
|
||||||
# migrations, never metadata.create_all.
|
# slow/unready database from crash-looping the container. Family convention
|
||||||
CMD ["sh", "-c", "alembic upgrade head && hypercorn 'thoughtsync.app:create_app()' --bind 0.0.0.0:5000 --keep-alive 600"]
|
# (rule 82): schema is built by real migrations, never metadata.create_all.
|
||||||
|
CMD ["sh", "-c", "python -m thoughtsync.dbwait && alembic upgrade head && hypercorn 'thoughtsync.app:create_app()' --bind 0.0.0.0:5000 --keep-alive 600"]
|
||||||
|
|||||||
@@ -40,9 +40,10 @@ services:
|
|||||||
- thoughtsync-dev-data:/var/thoughtsync
|
- thoughtsync-dev-data:/var/thoughtsync
|
||||||
ports:
|
ports:
|
||||||
- "5000:5000"
|
- "5000:5000"
|
||||||
# Install deps, run migrations, then serve with live reload on the mounted src.
|
# Install deps, wait for the DB, run migrations, then serve with live reload.
|
||||||
command: >
|
command: >
|
||||||
sh -c "pip install --quiet -e . &&
|
sh -c "pip install --quiet -e . &&
|
||||||
|
python -m thoughtsync.dbwait &&
|
||||||
alembic upgrade head &&
|
alembic upgrade head &&
|
||||||
hypercorn 'thoughtsync.app:create_app()' --bind 0.0.0.0:5000 --reload"
|
hypercorn 'thoughtsync.app:create_app()' --bind 0.0.0.0:5000 --reload"
|
||||||
|
|
||||||
|
|||||||
@@ -19,6 +19,7 @@ services:
|
|||||||
|
|
||||||
app:
|
app:
|
||||||
build: .
|
build: .
|
||||||
|
restart: unless-stopped
|
||||||
depends_on:
|
depends_on:
|
||||||
db:
|
db:
|
||||||
condition: service_healthy
|
condition: service_healthy
|
||||||
|
|||||||
@@ -0,0 +1,54 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import asyncio
|
||||||
|
import sys
|
||||||
|
|
||||||
|
from sqlalchemy import text
|
||||||
|
from sqlalchemy.ext.asyncio import create_async_engine
|
||||||
|
|
||||||
|
from .config import Config
|
||||||
|
|
||||||
|
ATTEMPTS = 60
|
||||||
|
DELAY_SECONDS = 1.0
|
||||||
|
|
||||||
|
|
||||||
|
async def _probe(url: str) -> None:
|
||||||
|
engine = create_async_engine(url)
|
||||||
|
try:
|
||||||
|
async with engine.connect() as conn:
|
||||||
|
await conn.execute(text("SELECT 1"))
|
||||||
|
finally:
|
||||||
|
await engine.dispose()
|
||||||
|
|
||||||
|
|
||||||
|
async def wait_for_db(attempts: int = ATTEMPTS, delay: float = DELAY_SECONDS) -> bool:
|
||||||
|
"""Poll the database until it accepts a connection, up to `attempts` tries.
|
||||||
|
|
||||||
|
Startup runs `alembic upgrade head` as the container's first DB touch, so a
|
||||||
|
database that is a moment slow to accept connections would otherwise crash the
|
||||||
|
container. This gives it a short window to come up, verifying once per `delay`
|
||||||
|
seconds, instead of failing on the first missed connection.
|
||||||
|
"""
|
||||||
|
url = Config.DATABASE_URL
|
||||||
|
for attempt in range(1, attempts + 1):
|
||||||
|
try:
|
||||||
|
await _probe(url)
|
||||||
|
print(f"[dbwait] database ready (attempt {attempt}/{attempts})", flush=True)
|
||||||
|
return True
|
||||||
|
except Exception as exc:
|
||||||
|
print(
|
||||||
|
f"[dbwait] not ready (attempt {attempt}/{attempts}): {exc.__class__.__name__}: {exc}",
|
||||||
|
flush=True,
|
||||||
|
)
|
||||||
|
if attempt < attempts:
|
||||||
|
await asyncio.sleep(delay)
|
||||||
|
print(f"[dbwait] database unreachable after {attempts} attempts; giving up", flush=True)
|
||||||
|
return False
|
||||||
|
|
||||||
|
|
||||||
|
def main() -> int:
|
||||||
|
return 0 if asyncio.run(wait_for_db()) else 1
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
sys.exit(main())
|
||||||
Reference in New Issue
Block a user