bvandeusen
8e98e79968
Reverses the advisory-lock approach (7309d1d) — it treated a replica race that
wasn't the cause and added a new indefinite-hang mode (a sibling/stale migrator
holding the xact lock).
Real cause of the 0040 hang (operator-diagnosed 2026-06-07): web has always been
a single replica. The migration's ALTER series_page queued behind a concurrent
tag-merge that held a series_page lock for minutes — _do_merge repoints
series_page then runs _create_protective_aliases, an unindexed full scan of
image_record (JSON column, ~59k rows). Migrations ran with no lock_timeout, so
the DDL hung indefinitely and silently.
Fix: SET lock_timeout (default 30s, env-overridable) on the migration connection
before alembic's transaction. A blocked DDL now fails fast with 'canceling
statement due to lock timeout'; the entrypoint exits non-zero so the deploy
retries / surfaces loudly instead of wedging. General protection for every
future migration. (The slow _create_protective_aliases scan — the actual lock
holder — is the separate perf fix still under discussion.)
Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
78 lines
2.8 KiB
Python
78 lines
2.8 KiB
Python
"""Alembic environment — reads DATABASE_URL from app config."""
|
|
|
|
import os
|
|
import re
|
|
from logging.config import fileConfig
|
|
|
|
from sqlalchemy import engine_from_config, pool, text
|
|
|
|
from alembic import context
|
|
from backend.app.config import get_config
|
|
from backend.app.models import Base
|
|
|
|
# Fail a blocked migration FAST instead of hanging forever. Migrations run
|
|
# against the live DB while workers hold locks; 0040's `ALTER series_page` queued
|
|
# behind a tag-merge that held a series_page lock for minutes (the merge runs an
|
|
# unindexed full scan over image_record while repointing series_page) and hung
|
|
# with no timeout — silent, indefinite (operator-flagged 2026-06-07). With a
|
|
# lock_timeout a blocked DDL errors ("canceling statement due to lock timeout")
|
|
# and the entrypoint's `alembic upgrade head` exits non-zero, so the deploy
|
|
# retries / surfaces loudly rather than wedging. Override via env when a known
|
|
# slow-lock window is expected.
|
|
_MIGRATION_LOCK_TIMEOUT = os.environ.get("MIGRATION_LOCK_TIMEOUT", "30s")
|
|
if not re.fullmatch(r"\d+\s*(ms|s|min)?", _MIGRATION_LOCK_TIMEOUT.strip()):
|
|
_MIGRATION_LOCK_TIMEOUT = "30s" # ignore a malformed override
|
|
|
|
config = context.config
|
|
|
|
if config.config_file_name is not None:
|
|
fileConfig(config.config_file_name)
|
|
|
|
config.set_main_option("sqlalchemy.url", get_config().database_url_sync)
|
|
|
|
target_metadata = Base.metadata
|
|
|
|
|
|
def run_migrations_offline() -> None:
|
|
url = config.get_main_option("sqlalchemy.url")
|
|
context.configure(
|
|
url=url,
|
|
target_metadata=target_metadata,
|
|
literal_binds=True,
|
|
dialect_opts={"paramstyle": "named"},
|
|
compare_type=True,
|
|
)
|
|
with context.begin_transaction():
|
|
context.run_migrations()
|
|
|
|
|
|
def run_migrations_online() -> None:
|
|
connectable = engine_from_config(
|
|
config.get_section(config.config_ini_section, {}),
|
|
prefix="sqlalchemy.",
|
|
poolclass=pool.NullPool,
|
|
)
|
|
with connectable.connect() as connection:
|
|
# Session-level lock_timeout for every DDL statement in this run. Set
|
|
# (and commit) before alembic opens its own transaction so the GUC
|
|
# persists on this connection regardless of how alembic structures its
|
|
# transactions. Value is from our own env, so f-string interpolation is
|
|
# safe (and it's been pattern-validated above); SET takes no bind params.
|
|
connection.execute(
|
|
text(f"SET lock_timeout = '{_MIGRATION_LOCK_TIMEOUT}'")
|
|
)
|
|
connection.commit()
|
|
context.configure(
|
|
connection=connection,
|
|
target_metadata=target_metadata,
|
|
compare_type=True,
|
|
)
|
|
with context.begin_transaction():
|
|
context.run_migrations()
|
|
|
|
|
|
if context.is_offline_mode():
|
|
run_migrations_offline()
|
|
else:
|
|
run_migrations_online()
|