"""Queue routing — the long one-shot maintenance tasks run on a dedicated `maintenance_long` lane so a 30-min backup or a multi-chunk audit can't starve the quick recovery sweeps / vacuum on the concurrency-1 `maintenance` lane (operator-flagged 2026-06-07).""" import pytest from backend.app.celery_app import celery def test_long_one_shots_route_to_maintenance_long(): routes = celery.conf.task_routes for prefix in ( "backend.app.tasks.backup.*", "backend.app.tasks.admin.*", "backend.app.tasks.library_audit.*", ): assert routes[prefix]["queue"] == "maintenance_long" def test_quick_maintenance_stays_on_maintenance(): routes = celery.conf.task_routes assert routes["backend.app.tasks.maintenance.*"]["queue"] == "maintenance" @pytest.mark.parametrize(("name", "queue"), [ ("backend.app.tasks.external.fetch_external_link", "download"), # The rows #4432 found recorded on the wrong lane: ("backend.app.tasks.translation.translate_posts", "maintenance_long"), ("backend.app.tasks.gpu_queue.enqueue_gpu_backfill", "maintenance"), ("backend.app.tasks.maintenance.backfill_phash", "maintenance_long"), ("backend.app.tasks.maintenance.date_posts_from_records", "maintenance_long"), ("backend.app.tasks.admin.normalize_tags_task", "maintenance_long"), ("backend.app.tasks.backup.backup_db_task", "maintenance_long"), ("backend.app.tasks.maintenance.vacuum_analyze", "maintenance"), ("backend.app.tasks.not_routed.anything", "default"), ]) def test_task_run_records_the_queue_the_router_sends_to(name, queue): """TaskRun.queue comes from the router, not a copy of task_routes (#4432): the stall sweep's per-queue thresholds and the System activity filters both key off it.""" from backend.app.celery_signals import _queue_for class _T: pass t = _T() t.name = name t.app = celery assert _queue_for(t) == queue def test_no_task_outlives_its_stall_threshold(): """A task whose hard time limit is longer than the stall sweep's threshold for it gets failed 'RecoverySweep' while it is still healthy — the class #4432 found on the ml, import and long-maintenance lanes. The threshold is resolved the way recover_stalled_task_runs resolves it: task-name override, then queue, then the default.""" from backend.app.celery_signals import _queue_for from backend.app.tasks.maintenance import ( QUEUE_STUCK_THRESHOLD_MINUTES, STUCK_THRESHOLD_MINUTES, TASK_STUCK_THRESHOLD_MINUTES, ) celery.loader.import_default_modules() checked = 0 too_short = [] for name, task in sorted(celery.tasks.items()): if name.startswith("celery."): continue limit = getattr(task, "time_limit", None) if not limit: continue threshold = TASK_STUCK_THRESHOLD_MINUTES.get( name, QUEUE_STUCK_THRESHOLD_MINUTES.get(_queue_for(task), STUCK_THRESHOLD_MINUTES), ) checked += 1 if limit / 60 > threshold: too_short.append(f"{name}: limit {limit / 60:.0f} min > sweep {threshold} min") assert checked > 20, "the task registry did not load — this guard checked nothing" assert not too_short, "\n".join(too_short) def test_backfill_phash_runs_on_the_long_lane(): """It lives in maintenance.py, so the quick-lane glob matches it too — the router must pick the exact name. A 35-minute rehash on the scheduler lane blocked the minute ticks behind it (2026-09-24).""" route = celery.amqp.router.route( {}, "backend.app.tasks.maintenance.backfill_phash", ) assert route["queue"].name == "maintenance_long" quick = celery.amqp.router.route( {}, "backend.app.tasks.maintenance.recover_stalled_task_runs", ) assert quick["queue"].name == "maintenance"