alembic/env.py

@@ -1,5 +1,7 @@
 """Alembic environment — reads DATABASE_URL from app config."""
 
+import os
+import re
 from logging.config import fileConfig
 
 from sqlalchemy import engine_from_config, pool, text
@@ -8,14 +10,18 @@ from alembic import context
 from backend.app.config import get_config
 from backend.app.models import Base
 
-# Arbitrary fixed 64-bit key for the session/transaction advisory lock that
-# serializes concurrent `alembic upgrade head` runs. Every `web` replica runs
-# migrations in its entrypoint, so under `docker stack deploy` two replicas can
-# boot at once and race the same DDL — duplicate CREATE TABLE, then a crashed
-# replica (operator-flagged 2026-06-07: 0040 raced; one backend died with
-# AdminShutdown). The first replica to reach the lock migrates; the rest block,
-# then find the version table already at head and apply nothing.
-_MIGRATION_LOCK_KEY = 0xFCA1E35C
+# Fail a blocked migration FAST instead of hanging forever. Migrations run
+# against the live DB while workers hold locks; 0040's `ALTER series_page` queued
+# behind a tag-merge that held a series_page lock for minutes (the merge runs an
+# unindexed full scan over image_record while repointing series_page) and hung
+# with no timeout — silent, indefinite (operator-flagged 2026-06-07). With a
+# lock_timeout a blocked DDL errors ("canceling statement due to lock timeout")
+# and the entrypoint's `alembic upgrade head` exits non-zero, so the deploy
+# retries / surfaces loudly rather than wedging. Override via env when a known
+# slow-lock window is expected.
+_MIGRATION_LOCK_TIMEOUT = os.environ.get("MIGRATION_LOCK_TIMEOUT", "30s")
+if not re.fullmatch(r"\d+\s*(ms|s|min)?", _MIGRATION_LOCK_TIMEOUT.strip()):
+    _MIGRATION_LOCK_TIMEOUT = "30s"  # ignore a malformed override
 
 config = context.config
 
@@ -47,24 +53,21 @@ def run_migrations_online() -> None:
         poolclass=pool.NullPool,
     )
     with connectable.connect() as connection:
+        # Session-level lock_timeout for every DDL statement in this run. Set
+        # (and commit) before alembic opens its own transaction so the GUC
+        # persists on this connection regardless of how alembic structures its
+        # transactions. Value is from our own env, so f-string interpolation is
+        # safe (and it's been pattern-validated above); SET takes no bind params.
+        connection.execute(
+            text(f"SET lock_timeout = '{_MIGRATION_LOCK_TIMEOUT}'")
+        )
+        connection.commit()
         context.configure(
             connection=connection,
             target_metadata=target_metadata,
             compare_type=True,
         )
         with context.begin_transaction():
-            # Serialize concurrent migrators (see _MIGRATION_LOCK_KEY). A
-            # transaction-scoped advisory lock: the first replica to get here
-            # holds it for the whole upgrade and is auto-released when this
-            # transaction ends. A sibling replica blocks on this line, and only
-            # once the leader commits does it proceed to read the version table
-            # — now at head — so it runs zero migrations instead of re-applying
-            # the same DDL. The lock is acquired BEFORE run_migrations() reads
-            # the current revision, which is what makes the no-op correct.
-            connection.execute(
-                text("SELECT pg_advisory_xact_lock(:k)"),
-                {"k": _MIGRATION_LOCK_KEY},
-            )
             context.run_migrations()