"""Celery configuration with separate queue lanes. Queues: import — filesystem import path (FC-2) ml — WD14 + SigLIP inference (FC-2; runs in the ml-worker image) thumbnail — image and video thumbnail generation (FC-2) download — gallery-dl tasks (FC-3) scan — periodic source checks (FC-3) — kept separate so long imports don't starve the scheduler maintenance — pHash recomputation, centroid rebuild, etc. (FC-2/FC-3) default — anything not explicitly routed """ from celery import Celery from .config import get_config def make_celery() -> Celery: cfg = get_config() app = Celery( "fabledcurator", broker=cfg.celery_broker_url, backend=cfg.celery_result_backend, include=[ "backend.app.tasks.smoke", "backend.app.tasks.scan", "backend.app.tasks.import_file", "backend.app.tasks.thumbnail", "backend.app.tasks.maintenance", "backend.app.tasks.ml", "backend.app.tasks.download", "backend.app.tasks.external", "backend.app.tasks.backup", "backend.app.tasks.admin", "backend.app.tasks.library_audit", ], ) app.conf.update( task_default_queue="default", task_routes={ "backend.app.tasks.import_file.*": {"queue": "import"}, "backend.app.tasks.ml.*": {"queue": "ml"}, "backend.app.tasks.thumbnail.*": {"queue": "thumbnail"}, "backend.app.tasks.download.*": {"queue": "download"}, # External file-host fetches are downloads — same lane (they can run # long, but the download worker already tolerates long backfills). "backend.app.tasks.external.*": {"queue": "download"}, "backend.app.tasks.scan.*": {"queue": "scan"}, # `maintenance` is the QUICK lane — recovery sweeps, vacuum, cleanup # (concurrency-1 on the scheduler). The long one-shots (DB backups, # library audits, admin maintenance: normalize/re-extract/cascade- # delete) run on a SEPARATE `maintenance_long` lane + worker so they # can never starve the quick self-healing sweeps (operator-flagged # 2026-06-07: a 2h audit blocked vacuum/backup/normalize for hours). "backend.app.tasks.maintenance.*": {"queue": "maintenance"}, "backend.app.tasks.backup.*": {"queue": "maintenance_long"}, "backend.app.tasks.admin.*": {"queue": "maintenance_long"}, "backend.app.tasks.library_audit.*": {"queue": "maintenance_long"}, }, # Heavy ML tasks need fair dispatch — see ImageRepo's precedent. task_acks_late=True, worker_prefetch_multiplier=1, # Broker resilience (2026-06-24): a swarm overlay-network blip after a # redeploy left Redis healthy but transiently unreachable, and a worker # starting in that window crash-looped on the initial broker connect # (kombu OperationalError) instead of waiting it out — needing a manual # Redis reset to recover. Retry the broker FOREVER (None) on startup and # at runtime so a transient outage self-heals when routing returns, # rather than the worker exiting. broker_connection_retry_on_startup=True, broker_connection_retry=True, broker_connection_max_retries=None, # Redis-transport socket options (apply to the BROKER connection): a # short connect timeout + TCP keepalive so a dead/blocked socket is # noticed and retried, and a periodic health check that proactively # reconnects a live worker through a network hiccup. broker_transport_options={ "socket_connect_timeout": 5, "socket_timeout": 30, "socket_keepalive": True, "retry_on_timeout": True, "health_check_interval": 30, }, # Same hardening for the Redis RESULT backend (separate connection pool). redis_socket_connect_timeout=5, redis_socket_timeout=30, redis_socket_keepalive=True, redis_retry_on_timeout=True, redis_backend_health_check_interval=30, beat_schedule={ "recover-interrupted-tasks": { "task": "backend.app.tasks.maintenance.recover_interrupted_tasks", "schedule": 300.0, # every 5 minutes }, "cleanup-old-tasks": { "task": "backend.app.tasks.maintenance.cleanup_old_tasks", "schedule": 86400.0, # daily }, "ml-backfill-daily": { "task": "backend.app.tasks.ml.backfill", "schedule": 86400.0, }, "recompute-centroids-daily": { "task": "backend.app.tasks.ml.recompute_centroids", "schedule": 86400.0, }, "apply-allowlist-sweep-daily": { "task": "backend.app.tasks.ml.apply_allowlist_tags", "schedule": 86400.0, }, "integrity-verify-weekly": { "task": "backend.app.tasks.maintenance.verify_integrity", "schedule": 604800.0, # weekly }, "fc3d-tick-due-sources": { "task": "backend.app.tasks.scan.tick_due_sources", "schedule": 60.0, # every minute }, "fc3d-cleanup-download-events": { "task": "backend.app.tasks.maintenance.cleanup_old_download_events", "schedule": 86400.0, # daily }, "recover-stalled-download-events": { "task": "backend.app.tasks.maintenance.recover_stalled_download_events", "schedule": 300.0, # every 5 min, matches recover-interrupted-tasks }, "recover-stalled-task-runs": { "task": "backend.app.tasks.maintenance.recover_stalled_task_runs", "schedule": 300.0, # every 5 min, matches recover-interrupted-tasks }, "prune-task-runs": { "task": "backend.app.tasks.maintenance.prune_task_runs", "schedule": 86400.0, # daily }, "vacuum-analyze": { "task": "backend.app.tasks.maintenance.vacuum_analyze", "schedule": 604800.0, # weekly — reclaim dead-tuple bloat + refresh stats }, "fc3h-backup-db-nightly": { "task": "backend.app.tasks.backup.backup_db_nightly", "schedule": 3600.0, # hourly tick; task self-gates on configured UTC hour }, "fc3h-prune-backups": { "task": "backend.app.tasks.backup.prune_backups", "schedule": 86400.0, # daily }, # Audit 2026-06-02 — three new per-entity recovery sweeps. # Each runs every 5 min like the other recover_stalled_* # sweeps; each is a no-op when nothing is stuck. "recover-stalled-backup-runs": { "task": "backend.app.tasks.maintenance.recover_stalled_backup_runs", "schedule": 300.0, }, "recover-stalled-library-audit-runs": { "task": "backend.app.tasks.maintenance.recover_stalled_library_audit_runs", "schedule": 300.0, }, "recover-stalled-tag-eval-runs": { "task": "backend.app.tasks.maintenance.recover_stalled_tag_eval_runs", "schedule": 300.0, }, "recover-stalled-import-batches": { "task": "backend.app.tasks.maintenance.recover_stalled_import_batches", "schedule": 300.0, }, # Audit 2026-06-02 — daily retention for two entities # whose terminal rows otherwise accumulate forever. "prune-library-audit-runs": { "task": "backend.app.tasks.maintenance.prune_library_audit_runs", "schedule": 86400.0, }, "prune-import-batches": { "task": "backend.app.tasks.maintenance.prune_import_batches", "schedule": 86400.0, }, # Audit 2026-06-02 — backfill_thumbnails's docstring claimed # "periodic Beat" but the entry was never registered, so the # library got no self-healing thumbnail repair; only the # manual admin-UI button fired it. Daily cadence is gentle # (the task is idempotent and only enqueues regen for rows # whose stored thumbnails are missing or corrupt). "backfill-thumbnails-daily": { "task": "backend.app.tasks.thumbnail.backfill_thumbnails", "schedule": 86400.0, }, # External file-host downloads (#830): a steady sweep catches links # the post-download hook missed (worker down, etc.); recovery re-tries # dead links daily; retention prunes long-dead rows. "extdl-sweep": { "task": "backend.app.tasks.external.sweep_external_links", "schedule": 600.0, # every 10 min }, "extdl-recover-daily": { "task": "backend.app.tasks.external.recover_external_links", "schedule": 86400.0, }, "extdl-prune-daily": { "task": "backend.app.tasks.external.prune_external_links", "schedule": 86400.0, }, }, timezone="UTC", ) # FC-3i: register task_run signal handlers (side-effect import). from . import celery_signals # noqa: F401 return app celery = make_celery()