bvandeusen
8649a13118
The GS/IR migration cutover is complete, so the runbook tooling is dead weight. Removed: - services/migrators/ (gs_ingest, ir_ingest, tag_apply, ml_queue, verify, cleanup), tasks/migration.py, api/migrate.py (+ blueprint registration) - MigrationRun model; alembic 0027 drops the migration_run table - frontend LegacyMigrationCard + migration store (+ MaintenancePanel ref) - celery include + task route + celery_signals queue mapping for migration.* - the 1 GB MAX_CONTENT_LENGTH / MAX_FORM_MEMORY override (added solely for the ir_ingest upload) - migration-surface tests (test_api_migrate, test_migration_verify, test_ir_ingest, test_gs_ingest, test_tag_apply) Kept: the alembic schema-migration tests (test_migration_00XX — unrelated) and cleanup_service.py (the permanent artist-cascade/unlink home). Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
114 lines
4.4 KiB
Python
114 lines
4.4 KiB
Python
"""Celery configuration with separate queue lanes.
|
|
|
|
Queues:
|
|
import — filesystem import path (FC-2)
|
|
ml — WD14 + SigLIP inference (FC-2; runs in the ml-worker image)
|
|
thumbnail — image and video thumbnail generation (FC-2)
|
|
download — gallery-dl tasks (FC-3)
|
|
scan — periodic source checks (FC-3) — kept separate so long imports
|
|
don't starve the scheduler
|
|
maintenance — pHash recomputation, centroid rebuild, etc. (FC-2/FC-3)
|
|
default — anything not explicitly routed
|
|
"""
|
|
|
|
from celery import Celery
|
|
|
|
from .config import get_config
|
|
|
|
|
|
def make_celery() -> Celery:
|
|
cfg = get_config()
|
|
app = Celery(
|
|
"fabledcurator",
|
|
broker=cfg.celery_broker_url,
|
|
backend=cfg.celery_result_backend,
|
|
include=[
|
|
"backend.app.tasks.smoke",
|
|
"backend.app.tasks.scan",
|
|
"backend.app.tasks.import_file",
|
|
"backend.app.tasks.thumbnail",
|
|
"backend.app.tasks.maintenance",
|
|
"backend.app.tasks.ml",
|
|
"backend.app.tasks.download",
|
|
"backend.app.tasks.backup",
|
|
"backend.app.tasks.admin",
|
|
"backend.app.tasks.library_audit",
|
|
],
|
|
)
|
|
app.conf.update(
|
|
task_default_queue="default",
|
|
task_routes={
|
|
"backend.app.tasks.import_file.*": {"queue": "import"},
|
|
"backend.app.tasks.ml.*": {"queue": "ml"},
|
|
"backend.app.tasks.thumbnail.*": {"queue": "thumbnail"},
|
|
"backend.app.tasks.download.*": {"queue": "download"},
|
|
"backend.app.tasks.scan.*": {"queue": "scan"},
|
|
"backend.app.tasks.maintenance.*": {"queue": "maintenance"},
|
|
"backend.app.tasks.backup.*": {"queue": "maintenance"},
|
|
"backend.app.tasks.admin.*": {"queue": "maintenance"},
|
|
"backend.app.tasks.library_audit.*": {"queue": "maintenance"},
|
|
},
|
|
# Heavy ML tasks need fair dispatch — see ImageRepo's precedent.
|
|
task_acks_late=True,
|
|
worker_prefetch_multiplier=1,
|
|
broker_connection_retry_on_startup=True,
|
|
beat_schedule={
|
|
"recover-interrupted-tasks": {
|
|
"task": "backend.app.tasks.maintenance.recover_interrupted_tasks",
|
|
"schedule": 300.0, # every 5 minutes
|
|
},
|
|
"cleanup-old-tasks": {
|
|
"task": "backend.app.tasks.maintenance.cleanup_old_tasks",
|
|
"schedule": 86400.0, # daily
|
|
},
|
|
"ml-backfill-daily": {
|
|
"task": "backend.app.tasks.ml.backfill",
|
|
"schedule": 86400.0,
|
|
},
|
|
"recompute-centroids-daily": {
|
|
"task": "backend.app.tasks.ml.recompute_centroids",
|
|
"schedule": 86400.0,
|
|
},
|
|
"apply-allowlist-sweep-daily": {
|
|
"task": "backend.app.tasks.ml.apply_allowlist_tags",
|
|
"schedule": 86400.0,
|
|
},
|
|
"integrity-verify-weekly": {
|
|
"task": "backend.app.tasks.maintenance.verify_integrity",
|
|
"schedule": 604800.0, # weekly
|
|
},
|
|
"fc3d-tick-due-sources": {
|
|
"task": "backend.app.tasks.scan.tick_due_sources",
|
|
"schedule": 60.0, # every minute
|
|
},
|
|
"fc3d-cleanup-download-events": {
|
|
"task": "backend.app.tasks.maintenance.cleanup_old_download_events",
|
|
"schedule": 86400.0, # daily
|
|
},
|
|
"recover-stalled-task-runs": {
|
|
"task": "backend.app.tasks.maintenance.recover_stalled_task_runs",
|
|
"schedule": 300.0, # every 5 min, matches recover-interrupted-tasks
|
|
},
|
|
"prune-task-runs": {
|
|
"task": "backend.app.tasks.maintenance.prune_task_runs",
|
|
"schedule": 86400.0, # daily
|
|
},
|
|
"fc3h-backup-db-nightly": {
|
|
"task": "backend.app.tasks.backup.backup_db_nightly",
|
|
"schedule": 3600.0, # hourly tick; task self-gates on configured UTC hour
|
|
},
|
|
"fc3h-prune-backups": {
|
|
"task": "backend.app.tasks.backup.prune_backups",
|
|
"schedule": 86400.0, # daily
|
|
},
|
|
},
|
|
timezone="UTC",
|
|
)
|
|
# FC-3i: register task_run signal handlers (side-effect import).
|
|
from . import celery_signals # noqa: F401
|
|
|
|
return app
|
|
|
|
|
|
celery = make_celery()
|