feat(maintenance): dedicated maintenance_long lane for long one-shot tasks

Even chunked, a single concurrency-1 maintenance lane is fragile — a 30-min DB
backup or a multi-chunk library audit holds the slot and delays the quick
self-healing recovery sweeps / vacuum (operator-flagged 2026-06-07: long runs
must never block quick maintenance).

Route the long one-shots — backup.*, admin.* (normalize/re-extract/cascade-
delete), library_audit.* — to a new `maintenance_long` queue served by a
dedicated worker (concurrency 1), added to docker-compose (+ dev override). The
scheduler keeps the quick `maintenance` lane (sweeps, vacuum, cleanup) for
itself, so a backup can no longer starve a 5-min vacuum. UI queue list +
routing tests updated.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

backend/app/api/system_activity.py

@@ -31,7 +31,7 @@ system_activity_bp = Blueprint(
 # absent.
 _QUEUE_NAMES = (
     "default", "import", "thumbnail", "ml",
-    "download", "scan", "maintenance",
+    "download", "scan", "maintenance", "maintenance_long",
 )
 
 # Cache module-level so all requests share the cache between polls.

backend/app/celery_app.py

@@ -43,10 +43,16 @@ def make_celery() -> Celery:
             "backend.app.tasks.thumbnail.*": {"queue": "thumbnail"},
             "backend.app.tasks.download.*": {"queue": "download"},
             "backend.app.tasks.scan.*": {"queue": "scan"},
+            # `maintenance` is the QUICK lane — recovery sweeps, vacuum, cleanup
+            # (concurrency-1 on the scheduler). The long one-shots (DB backups,
+            # library audits, admin maintenance: normalize/re-extract/cascade-
+            # delete) run on a SEPARATE `maintenance_long` lane + worker so they
+            # can never starve the quick self-healing sweeps (operator-flagged
+            # 2026-06-07: a 2h audit blocked vacuum/backup/normalize for hours).
             "backend.app.tasks.maintenance.*": {"queue": "maintenance"},
-            "backend.app.tasks.backup.*": {"queue": "maintenance"},
+            "backend.app.tasks.backup.*": {"queue": "maintenance_long"},
-            "backend.app.tasks.admin.*": {"queue": "maintenance"},
+            "backend.app.tasks.admin.*": {"queue": "maintenance_long"},
-            "backend.app.tasks.library_audit.*": {"queue": "maintenance"},
+            "backend.app.tasks.library_audit.*": {"queue": "maintenance_long"},
         },
         # Heavy ML tasks need fair dispatch — see ImageRepo's precedent.
         task_acks_late=True,

docker-compose.override.yml

@@ -35,6 +35,15 @@ services:
     volumes:
       - ./backend:/app/backend
 
+  maintenance-long:
+    build:
+      context: .
+      dockerfile: Dockerfile
+    environment:
+      LOG_LEVEL: DEBUG
+    volumes:
+      - ./backend:/app/backend
+
   ml-worker:
     build:
       context: .

docker-compose.yml

@@ -98,6 +98,25 @@ services:
       postgres: { condition: service_healthy }
       redis:    { condition: service_healthy }
 
+  # Dedicated lane for long one-shot maintenance (DB backups, library audits,
+  # admin maintenance). Kept off the scheduler's quick `maintenance` lane so a
+  # 30-min backup or a multi-chunk audit can never starve the 5-min recovery
+  # sweeps / vacuum (operator-flagged 2026-06-07). One slot — these are heavy.
+  maintenance-long:
+    image: git.fabledsword.com/bvandeusen/fabledcurator:dev
+    command: ["worker"]
+    environment:
+      <<: *app_env
+      CELERY_QUEUES: maintenance_long
+      CELERY_CONCURRENCY: "1"
+    volumes:
+      - ./images:/images
+      - ./import:/import
+      - ./downloads:/downloads
+    depends_on:
+      postgres: { condition: service_healthy }
+      redis:    { condition: service_healthy }
+
   ml-worker:
     image: git.fabledsword.com/bvandeusen/fabledcurator-ml:dev
     command: ["ml-worker"]

tests/test_celery_routing.py

@@ -0,0 +1,20 @@
+"""Queue routing — the long one-shot maintenance tasks run on a dedicated
+`maintenance_long` lane so a 30-min backup or a multi-chunk audit can't starve
+the quick recovery sweeps / vacuum on the concurrency-1 `maintenance` lane
+(operator-flagged 2026-06-07)."""
+from backend.app.celery_app import celery
+
+
+def test_long_one_shots_route_to_maintenance_long():
+    routes = celery.conf.task_routes
+    for prefix in (
+        "backend.app.tasks.backup.*",
+        "backend.app.tasks.admin.*",
+        "backend.app.tasks.library_audit.*",
+    ):
+        assert routes[prefix]["queue"] == "maintenance_long"
+
+
+def test_quick_maintenance_stays_on_maintenance():
+    routes = celery.conf.task_routes
+    assert routes["backend.app.tasks.maintenance.*"]["queue"] == "maintenance"