# app/celery_app.py """ Celery application factory and configuration. Provides task queue functionality for the ImageRepo import pipeline. """ import os from celery import Celery # Default broker/backend URLs DEFAULT_BROKER = 'redis://redis:6379/0' DEFAULT_BACKEND = 'redis://redis:6379/0' def make_celery(app=None): """ Create Celery instance with Flask app context support. Args: app: Flask application instance (optional) Returns: Configured Celery instance """ # Get broker/backend from app config or environment if app: broker = app.config.get('CELERY_BROKER_URL', DEFAULT_BROKER) backend = app.config.get('CELERY_RESULT_BACKEND', DEFAULT_BACKEND) else: broker = os.environ.get('CELERY_BROKER_URL', DEFAULT_BROKER) backend = os.environ.get('CELERY_RESULT_BACKEND', DEFAULT_BACKEND) celery = Celery( 'imagerepo', broker=broker, backend=backend, include=[ 'app.tasks.scan', 'app.tasks.import_file', 'app.tasks.thumbnail', 'app.tasks.sidecar' ] ) # Worker concurrency from environment concurrency = int(os.environ.get('CELERY_WORKER_CONCURRENCY', '2')) celery.conf.update( # Serialization task_serializer='json', accept_content=['json'], result_serializer='json', timezone='UTC', enable_utc=True, # Concurrency - intentionally low for steady background processing worker_concurrency=concurrency, worker_prefetch_multiplier=1, # One task at a time per worker process # Task routing - separate queues for different task types task_routes={ 'app.tasks.scan.*': {'queue': 'scan'}, 'app.tasks.import_file.*': {'queue': 'import'}, 'app.tasks.thumbnail.*': {'queue': 'thumbnail'}, 'app.tasks.sidecar.*': {'queue': 'sidecar'}, }, # Task default queue for unrouted tasks task_default_queue='default', # Result backend settings result_expires=86400, # 24 hours # Task execution settings for resume capability task_acks_late=True, # Acknowledge after task completes task_reject_on_worker_lost=True, # Requeue if worker dies # Time limits task_soft_time_limit=300, # 5 minutes soft limit task_time_limit=600, # 10 minutes hard limit (archives may need more) # Periodic task schedule (Celery Beat) beat_schedule={ 'periodic-import-scan': { 'task': 'app.tasks.scan.scan_directory', 'schedule': int(os.environ.get('IMPORT_EVERY_SECONDS', '28800')), # 8 hours default 'args': ('/import', '/images'), }, 'recover-interrupted-tasks': { 'task': 'app.tasks.scan.recover_interrupted_tasks', 'schedule': 300, # Every 5 minutes }, }, ) if app: # Don't pass Flask config directly to Celery - it contains old-style keys # that conflict with Celery's new lowercase format. # The broker/backend are already set above from app.config. # Create a task base class that runs within Flask app context class ContextTask(celery.Task): def __call__(self, *args, **kwargs): with app.app_context(): return self.run(*args, **kwargs) celery.Task = ContextTask return celery def create_celery_with_app(): """ Create Celery instance with Flask app context for standalone workers. This is used when running `celery -A app.celery_app:celery worker`. """ from app import create_app flask_app = create_app() return make_celery(flask_app) # Create celery instance with Flask app context # This ensures workers have access to the database and Flask extensions celery = create_celery_with_app()