From c2170094255dd9fb7ff5f14d94a4c5daeba96a8a Mon Sep 17 00:00:00 2001
From: Bryan Van Deusen <bryan.vandeusen@fabledsword.com>
Date: Sun, 7 Jun 2026 09:00:03 -0400
Subject: [PATCH] feat(maintenance): dedicated maintenance_long lane for long
 one-shot tasks
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Even chunked, a single concurrency-1 maintenance lane is fragile — a 30-min DB
backup or a multi-chunk library audit holds the slot and delays the quick
self-healing recovery sweeps / vacuum (operator-flagged 2026-06-07: long runs
must never block quick maintenance).

Route the long one-shots — backup.*, admin.* (normalize/re-extract/cascade-
delete), library_audit.* — to a new `maintenance_long` queue served by a
dedicated worker (concurrency 1), added to docker-compose (+ dev override). The
scheduler keeps the quick `maintenance` lane (sweeps, vacuum, cleanup) for
itself, so a backup can no longer starve a 5-min vacuum. UI queue list +
routing tests updated.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
---
 backend/app/api/system_activity.py |  2 +-
 backend/app/celery_app.py          | 12 +++++++++---
 docker-compose.override.yml        |  9 +++++++++
 docker-compose.yml                 | 19 +++++++++++++++++++
 tests/test_celery_routing.py       | 20 ++++++++++++++++++++
 5 files changed, 58 insertions(+), 4 deletions(-)
 create mode 100644 tests/test_celery_routing.py

diff --git a/backend/app/api/system_activity.py b/backend/app/api/system_activity.py
index 0d0600c..0371415 100644
--- a/backend/app/api/system_activity.py
+++ b/backend/app/api/system_activity.py
@@ -31,7 +31,7 @@ system_activity_bp = Blueprint(
 # absent.
 _QUEUE_NAMES = (
     "default", "import", "thumbnail", "ml",
-    "download", "scan", "maintenance",
+    "download", "scan", "maintenance", "maintenance_long",
 )
 
 # Cache module-level so all requests share the cache between polls.
diff --git a/backend/app/celery_app.py b/backend/app/celery_app.py
index 6620a6f..2d39f8f 100644
--- a/backend/app/celery_app.py
+++ b/backend/app/celery_app.py
@@ -43,10 +43,16 @@ def make_celery() -> Celery:
             "backend.app.tasks.thumbnail.*": {"queue": "thumbnail"},
             "backend.app.tasks.download.*": {"queue": "download"},
             "backend.app.tasks.scan.*": {"queue": "scan"},
+            # `maintenance` is the QUICK lane — recovery sweeps, vacuum, cleanup
+            # (concurrency-1 on the scheduler). The long one-shots (DB backups,
+            # library audits, admin maintenance: normalize/re-extract/cascade-
+            # delete) run on a SEPARATE `maintenance_long` lane + worker so they
+            # can never starve the quick self-healing sweeps (operator-flagged
+            # 2026-06-07: a 2h audit blocked vacuum/backup/normalize for hours).
             "backend.app.tasks.maintenance.*": {"queue": "maintenance"},
-            "backend.app.tasks.backup.*": {"queue": "maintenance"},
-            "backend.app.tasks.admin.*": {"queue": "maintenance"},
-            "backend.app.tasks.library_audit.*": {"queue": "maintenance"},
+            "backend.app.tasks.backup.*": {"queue": "maintenance_long"},
+            "backend.app.tasks.admin.*": {"queue": "maintenance_long"},
+            "backend.app.tasks.library_audit.*": {"queue": "maintenance_long"},
         },
         # Heavy ML tasks need fair dispatch — see ImageRepo's precedent.
         task_acks_late=True,
diff --git a/docker-compose.override.yml b/docker-compose.override.yml
index 5baecb7..aefd81f 100644
--- a/docker-compose.override.yml
+++ b/docker-compose.override.yml
@@ -35,6 +35,15 @@ services:
     volumes:
       - ./backend:/app/backend
 
+  maintenance-long:
+    build:
+      context: .
+      dockerfile: Dockerfile
+    environment:
+      LOG_LEVEL: DEBUG
+    volumes:
+      - ./backend:/app/backend
+
   ml-worker:
     build:
       context: .
diff --git a/docker-compose.yml b/docker-compose.yml
index f292640..9dea10a 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -98,6 +98,25 @@ services:
       postgres: { condition: service_healthy }
       redis:    { condition: service_healthy }
 
+  # Dedicated lane for long one-shot maintenance (DB backups, library audits,
+  # admin maintenance). Kept off the scheduler's quick `maintenance` lane so a
+  # 30-min backup or a multi-chunk audit can never starve the 5-min recovery
+  # sweeps / vacuum (operator-flagged 2026-06-07). One slot — these are heavy.
+  maintenance-long:
+    image: git.fabledsword.com/bvandeusen/fabledcurator:dev
+    command: ["worker"]
+    environment:
+      <<: *app_env
+      CELERY_QUEUES: maintenance_long
+      CELERY_CONCURRENCY: "1"
+    volumes:
+      - ./images:/images
+      - ./import:/import
+      - ./downloads:/downloads
+    depends_on:
+      postgres: { condition: service_healthy }
+      redis:    { condition: service_healthy }
+
   ml-worker:
     image: git.fabledsword.com/bvandeusen/fabledcurator-ml:dev
     command: ["ml-worker"]
diff --git a/tests/test_celery_routing.py b/tests/test_celery_routing.py
new file mode 100644
index 0000000..cbba0b7
--- /dev/null
+++ b/tests/test_celery_routing.py
@@ -0,0 +1,20 @@
+"""Queue routing — the long one-shot maintenance tasks run on a dedicated
+`maintenance_long` lane so a 30-min backup or a multi-chunk audit can't starve
+the quick recovery sweeps / vacuum on the concurrency-1 `maintenance` lane
+(operator-flagged 2026-06-07)."""
+from backend.app.celery_app import celery
+
+
+def test_long_one_shots_route_to_maintenance_long():
+    routes = celery.conf.task_routes
+    for prefix in (
+        "backend.app.tasks.backup.*",
+        "backend.app.tasks.admin.*",
+        "backend.app.tasks.library_audit.*",
+    ):
+        assert routes[prefix]["queue"] == "maintenance_long"
+
+
+def test_quick_maintenance_stays_on_maintenance():
+    routes = celery.conf.task_routes
+    assert routes["backend.app.tasks.maintenance.*"]["queue"] == "maintenance"