"""Celery configuration with separate queue lanes. Queues: import — filesystem import path (FC-2) ml — WD14 + SigLIP inference (FC-2; runs in the ml-worker image) thumbnail — image and video thumbnail generation (FC-2) download — gallery-dl tasks (FC-3) scan — periodic source checks (FC-3) — kept separate so long imports don't starve the scheduler maintenance — pHash recomputation, centroid rebuild, etc. (FC-2/FC-3) default — anything not explicitly routed """ from celery import Celery from .config import get_config def make_celery() -> Celery: cfg = get_config() app = Celery( "fabledcurator", broker=cfg.celery_broker_url, backend=cfg.celery_result_backend, include=[ "backend.app.tasks.smoke", "backend.app.tasks.scan", "backend.app.tasks.import_file", "backend.app.tasks.thumbnail", "backend.app.tasks.maintenance", "backend.app.tasks.ml", ], ) app.conf.update( task_default_queue="default", task_routes={ "backend.app.tasks.import_file.*": {"queue": "import"}, "backend.app.tasks.ml.*": {"queue": "ml"}, "backend.app.tasks.thumbnail.*": {"queue": "thumbnail"}, "backend.app.tasks.download.*": {"queue": "download"}, "backend.app.tasks.scan.*": {"queue": "scan"}, "backend.app.tasks.maintenance.*": {"queue": "maintenance"}, }, # Heavy ML tasks need fair dispatch — see ImageRepo's precedent. task_acks_late=True, worker_prefetch_multiplier=1, broker_connection_retry_on_startup=True, beat_schedule={ "recover-interrupted-tasks": { "task": "backend.app.tasks.maintenance.recover_interrupted_tasks", "schedule": 300.0, # every 5 minutes }, "cleanup-old-tasks": { "task": "backend.app.tasks.maintenance.cleanup_old_tasks", "schedule": 86400.0, # daily }, "ml-backfill-daily": { "task": "backend.app.tasks.ml.backfill", "schedule": 86400.0, }, "recompute-centroids-daily": { "task": "backend.app.tasks.ml.recompute_centroids", "schedule": 86400.0, }, "apply-allowlist-sweep-daily": { "task": "backend.app.tasks.ml.apply_allowlist_tags", "schedule": 86400.0, }, }, timezone="UTC", ) return app celery = make_celery()