fix(tasks): share one sync engine per worker process to stop connection-pool leak

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
2026-05-23 10:44:16 -04:00
parent 8f25b27315
commit 7d8b9c3d90
6 changed files with 45 additions and 45 deletions
+36
View File
@@ -0,0 +1,36 @@
"""Process-level sync engine for Celery task modules.
Each task module used to call ``create_engine(...)`` on every invocation,
which leaked engines (and Postgres connections) — high-fire-rate tasks
like ``import_media_file`` would exhaust ``max_connections`` within
minutes during a bulk migration.
This module owns one engine per process. Celery prefork forks before any
task runs, so each worker process lazily initializes its own engine on
the first task and reuses it for the rest of its life.
"""
from sqlalchemy import create_engine
from sqlalchemy.orm import sessionmaker
from ..config import get_config
_ENGINE = None
_SESSIONMAKER = None
def sync_session_factory():
"""Return a process-wide ``sessionmaker`` bound to a single engine."""
global _ENGINE, _SESSIONMAKER
if _SESSIONMAKER is None:
cfg = get_config()
_ENGINE = create_engine(
cfg.database_url_sync,
future=True,
pool_pre_ping=True,
pool_size=5,
max_overflow=5,
pool_recycle=300,
)
_SESSIONMAKER = sessionmaker(_ENGINE, expire_on_commit=False)
return _SESSIONMAKER