bvandeusen
d3bb8c509a
scan_directory walks ImportSettings.import_scan_path, creates an ImportBatch, enumerates supported files into ImportTasks, and enqueues import_media_file per task. import_media_file moves the task through its state machine (pending → queued → processing → complete/skipped/failed), updates ImportBatch counters atomically (UPDATE ... SET col = col + 1), enqueues a thumbnail task on success, and marks the batch complete when the last task drains. generate_thumbnail runs on its own queue (thumbnail) so big imports don't starve thumbnail throughput; failure here is logged and does not fail the import. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
52 lines
1.7 KiB
Python
52 lines
1.7 KiB
Python
"""Celery configuration with separate queue lanes.
|
|
|
|
Queues:
|
|
import — filesystem import path (FC-2)
|
|
ml — WD14 + SigLIP inference (FC-2; runs in the ml-worker image)
|
|
thumbnail — image and video thumbnail generation (FC-2)
|
|
download — gallery-dl tasks (FC-3)
|
|
scan — periodic source checks (FC-3) — kept separate so long imports
|
|
don't starve the scheduler
|
|
maintenance — pHash recomputation, centroid rebuild, etc. (FC-2/FC-3)
|
|
default — anything not explicitly routed
|
|
"""
|
|
|
|
from celery import Celery
|
|
|
|
from .config import get_config
|
|
|
|
|
|
def make_celery() -> Celery:
|
|
cfg = get_config()
|
|
app = Celery(
|
|
"fabledcurator",
|
|
broker=cfg.celery_broker_url,
|
|
backend=cfg.celery_result_backend,
|
|
include=[
|
|
"backend.app.tasks.smoke",
|
|
"backend.app.tasks.scan",
|
|
"backend.app.tasks.import_file",
|
|
"backend.app.tasks.thumbnail",
|
|
"backend.app.tasks.maintenance",
|
|
],
|
|
)
|
|
app.conf.update(
|
|
task_default_queue="default",
|
|
task_routes={
|
|
"backend.app.tasks.import_file.*": {"queue": "import"},
|
|
"backend.app.tasks.ml.*": {"queue": "ml"},
|
|
"backend.app.tasks.thumbnail.*": {"queue": "thumbnail"},
|
|
"backend.app.tasks.download.*": {"queue": "download"},
|
|
"backend.app.tasks.scan.*": {"queue": "scan"},
|
|
"backend.app.tasks.maintenance.*": {"queue": "maintenance"},
|
|
},
|
|
# Heavy ML tasks need fair dispatch — see ImageRepo's precedent.
|
|
task_acks_late=True,
|
|
worker_prefetch_multiplier=1,
|
|
broker_connection_retry_on_startup=True,
|
|
)
|
|
return app
|
|
|
|
|
|
celery = make_celery()
|