Bryan Van Deusen
124 lines
3.9 KiB
Python
124 lines
3.9 KiB
Python
# app/celery_app.py
|
|
"""
|
|
Celery application factory and configuration.
|
|
Provides task queue functionality for the ImageRepo import pipeline.
|
|
"""
|
|
import os
|
|
from celery import Celery
|
|
|
|
# Default broker/backend URLs
|
|
DEFAULT_BROKER = 'redis://redis:6379/0'
|
|
DEFAULT_BACKEND = 'redis://redis:6379/0'
|
|
|
|
|
|
def make_celery(app=None):
|
|
"""
|
|
Create Celery instance with Flask app context support.
|
|
|
|
Args:
|
|
app: Flask application instance (optional)
|
|
|
|
Returns:
|
|
Configured Celery instance
|
|
"""
|
|
# Get broker/backend from app config or environment
|
|
if app:
|
|
broker = app.config.get('CELERY_BROKER_URL', DEFAULT_BROKER)
|
|
backend = app.config.get('CELERY_RESULT_BACKEND', DEFAULT_BACKEND)
|
|
else:
|
|
broker = os.environ.get('CELERY_BROKER_URL', DEFAULT_BROKER)
|
|
backend = os.environ.get('CELERY_RESULT_BACKEND', DEFAULT_BACKEND)
|
|
|
|
celery = Celery(
|
|
'imagerepo',
|
|
broker=broker,
|
|
backend=backend,
|
|
include=[
|
|
'app.tasks.scan',
|
|
'app.tasks.import_file',
|
|
'app.tasks.thumbnail',
|
|
'app.tasks.sidecar'
|
|
]
|
|
)
|
|
|
|
# Worker concurrency from environment
|
|
concurrency = int(os.environ.get('CELERY_WORKER_CONCURRENCY', '2'))
|
|
|
|
celery.conf.update(
|
|
# Serialization
|
|
task_serializer='json',
|
|
accept_content=['json'],
|
|
result_serializer='json',
|
|
timezone='UTC',
|
|
enable_utc=True,
|
|
|
|
# Concurrency - intentionally low for steady background processing
|
|
worker_concurrency=concurrency,
|
|
worker_prefetch_multiplier=1, # One task at a time per worker process
|
|
|
|
# Task routing - separate queues for different task types
|
|
task_routes={
|
|
'app.tasks.scan.*': {'queue': 'scan'},
|
|
'app.tasks.import_file.*': {'queue': 'import'},
|
|
'app.tasks.thumbnail.*': {'queue': 'thumbnail'},
|
|
'app.tasks.sidecar.*': {'queue': 'sidecar'},
|
|
},
|
|
|
|
# Task default queue for unrouted tasks
|
|
task_default_queue='default',
|
|
|
|
# Result backend settings
|
|
result_expires=86400, # 24 hours
|
|
|
|
# Task execution settings for resume capability
|
|
task_acks_late=True, # Acknowledge after task completes
|
|
task_reject_on_worker_lost=True, # Requeue if worker dies
|
|
|
|
# Time limits
|
|
task_soft_time_limit=300, # 5 minutes soft limit
|
|
task_time_limit=600, # 10 minutes hard limit (archives may need more)
|
|
|
|
# Periodic task schedule (Celery Beat)
|
|
beat_schedule={
|
|
'periodic-import-scan': {
|
|
'task': 'app.tasks.scan.scan_directory',
|
|
'schedule': int(os.environ.get('IMPORT_EVERY_SECONDS', '28800')), # 8 hours default
|
|
'args': ('/import', '/images'),
|
|
},
|
|
'recover-interrupted-tasks': {
|
|
'task': 'app.tasks.scan.recover_interrupted_tasks',
|
|
'schedule': 300, # Every 5 minutes
|
|
},
|
|
},
|
|
)
|
|
|
|
if app:
|
|
# Don't pass Flask config directly to Celery - it contains old-style keys
|
|
# that conflict with Celery's new lowercase format.
|
|
# The broker/backend are already set above from app.config.
|
|
|
|
# Create a task base class that runs within Flask app context
|
|
class ContextTask(celery.Task):
|
|
def __call__(self, *args, **kwargs):
|
|
with app.app_context():
|
|
return self.run(*args, **kwargs)
|
|
|
|
celery.Task = ContextTask
|
|
|
|
return celery
|
|
|
|
|
|
def create_celery_with_app():
|
|
"""
|
|
Create Celery instance with Flask app context for standalone workers.
|
|
This is used when running `celery -A app.celery_app:celery worker`.
|
|
"""
|
|
from app import create_app
|
|
flask_app = create_app()
|
|
return make_celery(flask_app)
|
|
|
|
|
|
# Create celery instance with Flask app context
|
|
# This ensures workers have access to the database and Flask extensions
|
|
celery = create_celery_with_app()
|