From 058fa8560674e39e25cf7dd461753877e433c2df Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Thu, 24 Sep 2026 19:26:48 -0400 Subject: [PATCH] fix: backfill_phash runs on the long maintenance lane, not the scheduler's quick one (4411) A whole-library rehash with a 35-minute limit was matched by the maintenance.* glob. It held a scheduler process for its whole run, and the minute ticks queued behind it. An exact-name route now sends it to maintenance_long. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01LVjrnpQjRgHdvq95rASoiR --- backend/app/celery_app.py | 7 +++++++ tests/test_celery_routing.py | 14 ++++++++++++++ 2 files changed, 21 insertions(+) diff --git a/backend/app/celery_app.py b/backend/app/celery_app.py index 5f45567..bd0ad2c 100644 --- a/backend/app/celery_app.py +++ b/backend/app/celery_app.py @@ -62,6 +62,13 @@ def make_celery() -> Celery: # can never starve the quick self-healing sweeps (operator-flagged # 2026-06-07: a 2h audit blocked vacuum/backup/normalize for hours). "backend.app.tasks.maintenance.*": {"queue": "maintenance"}, + # The one long job in maintenance.py: a whole-library phash + # recompute (35 min hard limit; the library was cleared for + # re-hashing by migration 0098). On the quick lane it held a + # scheduler process for its whole run, and the minute ticks queued + # up behind it (2026-09-24: 7 waiting, "all workers busy for 18 + # minutes"). An exact name wins over the glob above. + "backend.app.tasks.maintenance.backfill_phash": {"queue": "maintenance_long"}, "backend.app.tasks.backup.*": {"queue": "maintenance_long"}, "backend.app.tasks.admin.*": {"queue": "maintenance_long"}, "backend.app.tasks.library_audit.*": {"queue": "maintenance_long"}, diff --git a/tests/test_celery_routing.py b/tests/test_celery_routing.py index 8a7cb67..e7a24e6 100644 --- a/tests/test_celery_routing.py +++ b/tests/test_celery_routing.py @@ -36,3 +36,17 @@ def test_queue_for_mirrors_external_to_download(): celery.conf.task_routes["backend.app.tasks.external.*"]["queue"] == "download" ) + + +def test_backfill_phash_runs_on_the_long_lane(): + """It lives in maintenance.py, so the quick-lane glob matches it too — + the router must pick the exact name. A 35-minute rehash on the scheduler + lane blocked the minute ticks behind it (2026-09-24).""" + route = celery.amqp.router.route( + {}, "backend.app.tasks.maintenance.backfill_phash", + ) + assert route["queue"].name == "maintenance_long" + quick = celery.amqp.router.route( + {}, "backend.app.tasks.maintenance.recover_stalled_task_runs", + ) + assert quick["queue"].name == "maintenance"