fix(tasks): share one sync engine per worker process to stop connection-pool leak
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,36 @@
|
||||
"""Process-level sync engine for Celery task modules.
|
||||
|
||||
Each task module used to call ``create_engine(...)`` on every invocation,
|
||||
which leaked engines (and Postgres connections) — high-fire-rate tasks
|
||||
like ``import_media_file`` would exhaust ``max_connections`` within
|
||||
minutes during a bulk migration.
|
||||
|
||||
This module owns one engine per process. Celery prefork forks before any
|
||||
task runs, so each worker process lazily initializes its own engine on
|
||||
the first task and reuses it for the rest of its life.
|
||||
"""
|
||||
|
||||
from sqlalchemy import create_engine
|
||||
from sqlalchemy.orm import sessionmaker
|
||||
|
||||
from ..config import get_config
|
||||
|
||||
_ENGINE = None
|
||||
_SESSIONMAKER = None
|
||||
|
||||
|
||||
def sync_session_factory():
|
||||
"""Return a process-wide ``sessionmaker`` bound to a single engine."""
|
||||
global _ENGINE, _SESSIONMAKER
|
||||
if _SESSIONMAKER is None:
|
||||
cfg = get_config()
|
||||
_ENGINE = create_engine(
|
||||
cfg.database_url_sync,
|
||||
future=True,
|
||||
pool_pre_ping=True,
|
||||
pool_size=5,
|
||||
max_overflow=5,
|
||||
pool_recycle=300,
|
||||
)
|
||||
_SESSIONMAKER = sessionmaker(_ENGINE, expire_on_commit=False)
|
||||
return _SESSIONMAKER
|
||||
Reference in New Issue
Block a user