From 7309d1d6d4a0155731131c94b3505b531f491d95 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Sun, 7 Jun 2026 19:59:45 -0400 Subject: [PATCH] fix(alembic): serialize concurrent migrators with an advisory lock MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Every web replica runs 'alembic upgrade head' in its entrypoint, so under docker stack deploy two replicas can boot at once and race the same DDL — 0040 raced in prod (operator-flagged 2026-06-07): one backend wedged on the series_page lock while a second tried to re-CREATE series_chapter, and the loser died with AdminShutdown, crash-looping the web service. Wrap run_migrations() in a transaction-scoped pg_advisory_xact_lock acquired BEFORE the version table is read. The first replica to reach it migrates and holds the lock for the whole upgrade; siblings block, then find the version already at head and apply nothing. Works regardless of replica count and needs no Swarm depends_on ordering (which stack deploy ignores anyway). Co-Authored-By: Claude Opus 4.8 (1M context) --- alembic/env.py | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/alembic/env.py b/alembic/env.py index 0530946..b657465 100644 --- a/alembic/env.py +++ b/alembic/env.py @@ -2,12 +2,21 @@ from logging.config import fileConfig -from sqlalchemy import engine_from_config, pool +from sqlalchemy import engine_from_config, pool, text from alembic import context from backend.app.config import get_config from backend.app.models import Base +# Arbitrary fixed 64-bit key for the session/transaction advisory lock that +# serializes concurrent `alembic upgrade head` runs. Every `web` replica runs +# migrations in its entrypoint, so under `docker stack deploy` two replicas can +# boot at once and race the same DDL — duplicate CREATE TABLE, then a crashed +# replica (operator-flagged 2026-06-07: 0040 raced; one backend died with +# AdminShutdown). The first replica to reach the lock migrates; the rest block, +# then find the version table already at head and apply nothing. +_MIGRATION_LOCK_KEY = 0xFCA1E35C + config = context.config if config.config_file_name is not None: @@ -44,6 +53,18 @@ def run_migrations_online() -> None: compare_type=True, ) with context.begin_transaction(): + # Serialize concurrent migrators (see _MIGRATION_LOCK_KEY). A + # transaction-scoped advisory lock: the first replica to get here + # holds it for the whole upgrade and is auto-released when this + # transaction ends. A sibling replica blocks on this line, and only + # once the leader commits does it proceed to read the version table + # — now at head — so it runs zero migrations instead of re-applying + # the same DDL. The lock is acquired BEFORE run_migrations() reads + # the current revision, which is what makes the no-op correct. + connection.execute( + text("SELECT pg_advisory_xact_lock(:k)"), + {"k": _MIGRATION_LOCK_KEY}, + ) context.run_migrations()