FabledCurator/backend/app/celery_app.py

"""Celery configuration with separate queue lanes.

Queues:
  import      — filesystem import path (FC-2)
  ml          — WD14 + SigLIP inference (FC-2; runs in the ml-worker image)
  thumbnail   — image and video thumbnail generation (FC-2)
  download    — gallery-dl tasks (FC-3)
  scan        — periodic source checks (FC-3) — kept separate so long imports
                don't starve the scheduler
  maintenance — pHash recomputation, centroid rebuild, etc. (FC-2/FC-3)
  default     — anything not explicitly routed
"""

from celery import Celery

from .config import get_config


def make_celery() -> Celery:
    cfg = get_config()
    app = Celery(
        "fabledcurator",
        broker=cfg.celery_broker_url,
        backend=cfg.celery_result_backend,
        include=[
            "backend.app.tasks.smoke",
            "backend.app.tasks.scan",
            "backend.app.tasks.import_file",
            "backend.app.tasks.thumbnail",
            "backend.app.tasks.maintenance",
            "backend.app.tasks.ml",
            "backend.app.tasks.download",
            "backend.app.tasks.backup",
            "backend.app.tasks.admin",
            "backend.app.tasks.library_audit",
        ],
    )
    app.conf.update(
        task_default_queue="default",
        task_routes={
            "backend.app.tasks.import_file.*": {"queue": "import"},
            "backend.app.tasks.ml.*": {"queue": "ml"},
            "backend.app.tasks.thumbnail.*": {"queue": "thumbnail"},
            "backend.app.tasks.download.*": {"queue": "download"},
            "backend.app.tasks.scan.*": {"queue": "scan"},
            # `maintenance` is the QUICK lane — recovery sweeps, vacuum, cleanup
            # (concurrency-1 on the scheduler). The long one-shots (DB backups,
            # library audits, admin maintenance: normalize/re-extract/cascade-
            # delete) run on a SEPARATE `maintenance_long` lane + worker so they
            # can never starve the quick self-healing sweeps (operator-flagged
            # 2026-06-07: a 2h audit blocked vacuum/backup/normalize for hours).
            "backend.app.tasks.maintenance.*": {"queue": "maintenance"},
            "backend.app.tasks.backup.*": {"queue": "maintenance_long"},
            "backend.app.tasks.admin.*": {"queue": "maintenance_long"},
            "backend.app.tasks.library_audit.*": {"queue": "maintenance_long"},
        },
        # Heavy ML tasks need fair dispatch — see ImageRepo's precedent.
        task_acks_late=True,
        worker_prefetch_multiplier=1,
        broker_connection_retry_on_startup=True,
        beat_schedule={
            "recover-interrupted-tasks": {
                "task": "backend.app.tasks.maintenance.recover_interrupted_tasks",
                "schedule": 300.0,  # every 5 minutes
            },
            "cleanup-old-tasks": {
                "task": "backend.app.tasks.maintenance.cleanup_old_tasks",
                "schedule": 86400.0,  # daily
            },
            "ml-backfill-daily": {
                "task": "backend.app.tasks.ml.backfill",
                "schedule": 86400.0,
            },
            "recompute-centroids-daily": {
                "task": "backend.app.tasks.ml.recompute_centroids",
                "schedule": 86400.0,
            },
            "apply-allowlist-sweep-daily": {
                "task": "backend.app.tasks.ml.apply_allowlist_tags",
                "schedule": 86400.0,
            },
            "integrity-verify-weekly": {
                "task": "backend.app.tasks.maintenance.verify_integrity",
                "schedule": 604800.0,  # weekly
            },
            "fc3d-tick-due-sources": {
                "task": "backend.app.tasks.scan.tick_due_sources",
                "schedule": 60.0,  # every minute
            },
            "fc3d-cleanup-download-events": {
                "task": "backend.app.tasks.maintenance.cleanup_old_download_events",
                "schedule": 86400.0,  # daily
            },
            "recover-stalled-download-events": {
                "task": "backend.app.tasks.maintenance.recover_stalled_download_events",
                "schedule": 300.0,  # every 5 min, matches recover-interrupted-tasks
            },
            "recover-stalled-task-runs": {
                "task": "backend.app.tasks.maintenance.recover_stalled_task_runs",
                "schedule": 300.0,  # every 5 min, matches recover-interrupted-tasks
            },
            "prune-task-runs": {
                "task": "backend.app.tasks.maintenance.prune_task_runs",
                "schedule": 86400.0,  # daily
            },
            "vacuum-analyze": {
                "task": "backend.app.tasks.maintenance.vacuum_analyze",
                "schedule": 604800.0,  # weekly — reclaim dead-tuple bloat + refresh stats
            },
            "fc3h-backup-db-nightly": {
                "task": "backend.app.tasks.backup.backup_db_nightly",
                "schedule": 3600.0,  # hourly tick; task self-gates on configured UTC hour
            },
            "fc3h-prune-backups": {
                "task": "backend.app.tasks.backup.prune_backups",
                "schedule": 86400.0,  # daily
            },
            # Audit 2026-06-02 — three new per-entity recovery sweeps.
            # Each runs every 5 min like the other recover_stalled_*
            # sweeps; each is a no-op when nothing is stuck.
            "recover-stalled-backup-runs": {
                "task": "backend.app.tasks.maintenance.recover_stalled_backup_runs",
                "schedule": 300.0,
            },
            "recover-stalled-library-audit-runs": {
                "task": "backend.app.tasks.maintenance.recover_stalled_library_audit_runs",
                "schedule": 300.0,
            },
            "recover-stalled-import-batches": {
                "task": "backend.app.tasks.maintenance.recover_stalled_import_batches",
                "schedule": 300.0,
            },
            # Audit 2026-06-02 — daily retention for two entities
            # whose terminal rows otherwise accumulate forever.
            "prune-library-audit-runs": {
                "task": "backend.app.tasks.maintenance.prune_library_audit_runs",
                "schedule": 86400.0,
            },
            "prune-import-batches": {
                "task": "backend.app.tasks.maintenance.prune_import_batches",
                "schedule": 86400.0,
            },
            # Audit 2026-06-02 — backfill_thumbnails's docstring claimed
            # "periodic Beat" but the entry was never registered, so the
            # library got no self-healing thumbnail repair; only the
            # manual admin-UI button fired it. Daily cadence is gentle
            # (the task is idempotent and only enqueues regen for rows
            # whose stored thumbnails are missing or corrupt).
            "backfill-thumbnails-daily": {
                "task": "backend.app.tasks.thumbnail.backfill_thumbnails",
                "schedule": 86400.0,
            },
        },
        timezone="UTC",
    )
    # FC-3i: register task_run signal handlers (side-effect import).
    from . import celery_signals  # noqa: F401

    return app


celery = make_celery()