bvandeusen
7309d1d6d4
Every web replica runs 'alembic upgrade head' in its entrypoint, so under docker stack deploy two replicas can boot at once and race the same DDL — 0040 raced in prod (operator-flagged 2026-06-07): one backend wedged on the series_page lock while a second tried to re-CREATE series_chapter, and the loser died with AdminShutdown, crash-looping the web service. Wrap run_migrations() in a transaction-scoped pg_advisory_xact_lock acquired BEFORE the version table is read. The first replica to reach it migrates and holds the lock for the whole upgrade; siblings block, then find the version already at head and apply nothing. Works regardless of replica count and needs no Swarm depends_on ordering (which stack deploy ignores anyway). Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
75 lines
2.7 KiB
Python
75 lines
2.7 KiB
Python
"""Alembic environment — reads DATABASE_URL from app config."""
|
|
|
|
from logging.config import fileConfig
|
|
|
|
from sqlalchemy import engine_from_config, pool, text
|
|
|
|
from alembic import context
|
|
from backend.app.config import get_config
|
|
from backend.app.models import Base
|
|
|
|
# Arbitrary fixed 64-bit key for the session/transaction advisory lock that
|
|
# serializes concurrent `alembic upgrade head` runs. Every `web` replica runs
|
|
# migrations in its entrypoint, so under `docker stack deploy` two replicas can
|
|
# boot at once and race the same DDL — duplicate CREATE TABLE, then a crashed
|
|
# replica (operator-flagged 2026-06-07: 0040 raced; one backend died with
|
|
# AdminShutdown). The first replica to reach the lock migrates; the rest block,
|
|
# then find the version table already at head and apply nothing.
|
|
_MIGRATION_LOCK_KEY = 0xFCA1E35C
|
|
|
|
config = context.config
|
|
|
|
if config.config_file_name is not None:
|
|
fileConfig(config.config_file_name)
|
|
|
|
config.set_main_option("sqlalchemy.url", get_config().database_url_sync)
|
|
|
|
target_metadata = Base.metadata
|
|
|
|
|
|
def run_migrations_offline() -> None:
|
|
url = config.get_main_option("sqlalchemy.url")
|
|
context.configure(
|
|
url=url,
|
|
target_metadata=target_metadata,
|
|
literal_binds=True,
|
|
dialect_opts={"paramstyle": "named"},
|
|
compare_type=True,
|
|
)
|
|
with context.begin_transaction():
|
|
context.run_migrations()
|
|
|
|
|
|
def run_migrations_online() -> None:
|
|
connectable = engine_from_config(
|
|
config.get_section(config.config_ini_section, {}),
|
|
prefix="sqlalchemy.",
|
|
poolclass=pool.NullPool,
|
|
)
|
|
with connectable.connect() as connection:
|
|
context.configure(
|
|
connection=connection,
|
|
target_metadata=target_metadata,
|
|
compare_type=True,
|
|
)
|
|
with context.begin_transaction():
|
|
# Serialize concurrent migrators (see _MIGRATION_LOCK_KEY). A
|
|
# transaction-scoped advisory lock: the first replica to get here
|
|
# holds it for the whole upgrade and is auto-released when this
|
|
# transaction ends. A sibling replica blocks on this line, and only
|
|
# once the leader commits does it proceed to read the version table
|
|
# — now at head — so it runs zero migrations instead of re-applying
|
|
# the same DDL. The lock is acquired BEFORE run_migrations() reads
|
|
# the current revision, which is what makes the no-op correct.
|
|
connection.execute(
|
|
text("SELECT pg_advisory_xact_lock(:k)"),
|
|
{"k": _MIGRATION_LOCK_KEY},
|
|
)
|
|
context.run_migrations()
|
|
|
|
|
|
if context.is_offline_mode():
|
|
run_migrations_offline()
|
|
else:
|
|
run_migrations_online()
|