bvandeusenandClaude Opus 5.5
a360d69ee8
CI and images / lint (push) Successful in 3s
CI and images / extension-version (push) Successful in 4s
CI and images / frontend-build (push) Successful in 25s
CI and images / extension-test (push) Successful in 28s
CI and images / backend-lint-and-test (push) Successful in 34s
CI and images / integration (push) Failing after 2m24s
CI and images / sign-extension (push) Skipped
CI and images / build-web (push) Skipped
CI and images / smoke-web (push) Skipped
CI and images / promote (push) Skipped
CI and images / build-agent (push) Skipped
celery_signals._queue_for was a hand-kept copy of task_routes and had drifted: - backup, admin and library_audit jobs, and backfill_phash, run on maintenance_long but were recorded as `maintenance`; - translation and gpu_queue jobs were recorded as `default`, where the 5-minute stall sweep failed healthy 35-minute translation runs. It now asks the router, cached per task name. A new guard test checks every registered task's hard time limit against the stall threshold the sweep would use for it. It also caught these sweeps, which failed healthy runs mid-flight and are fixed here: - ml's scheduled sweeps (35 min, previously swept at 25); - train_heads and apply_head_tags (65 min); - import_media_file (6 min, previously swept at 5); - the long lane, which now has its own 45-minute threshold. UI changes: - the admin job poller follows a job by celery_task_id, via a new filter on /runs, instead of by lane; - the archive re-extract and missing-file repair cards show the long lane's backlog, where their jobs actually wait; - the queue table lists maintenance_long. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01LVjrnpQjRgHdvq95rASoiR
96 lines
3.7 KiB
Python
96 lines
3.7 KiB
Python
"""Queue routing — the long one-shot maintenance tasks run on a dedicated
|
|
`maintenance_long` lane so a 30-min backup or a multi-chunk audit can't starve
|
|
the quick recovery sweeps / vacuum on the concurrency-1 `maintenance` lane
|
|
(operator-flagged 2026-06-07)."""
|
|
import pytest
|
|
|
|
from backend.app.celery_app import celery
|
|
|
|
|
|
def test_long_one_shots_route_to_maintenance_long():
|
|
routes = celery.conf.task_routes
|
|
for prefix in (
|
|
"backend.app.tasks.backup.*",
|
|
"backend.app.tasks.admin.*",
|
|
"backend.app.tasks.library_audit.*",
|
|
):
|
|
assert routes[prefix]["queue"] == "maintenance_long"
|
|
|
|
|
|
def test_quick_maintenance_stays_on_maintenance():
|
|
routes = celery.conf.task_routes
|
|
assert routes["backend.app.tasks.maintenance.*"]["queue"] == "maintenance"
|
|
|
|
|
|
@pytest.mark.parametrize(("name", "queue"), [
|
|
("backend.app.tasks.external.fetch_external_link", "download"),
|
|
# The rows #4432 found recorded on the wrong lane:
|
|
("backend.app.tasks.translation.translate_posts", "maintenance_long"),
|
|
("backend.app.tasks.gpu_queue.enqueue_gpu_backfill", "maintenance"),
|
|
("backend.app.tasks.maintenance.backfill_phash", "maintenance_long"),
|
|
("backend.app.tasks.admin.normalize_tags_task", "maintenance_long"),
|
|
("backend.app.tasks.backup.backup_db_task", "maintenance_long"),
|
|
("backend.app.tasks.maintenance.vacuum_analyze", "maintenance"),
|
|
("backend.app.tasks.not_routed.anything", "default"),
|
|
])
|
|
def test_task_run_records_the_queue_the_router_sends_to(name, queue):
|
|
"""TaskRun.queue comes from the router, not a copy of task_routes (#4432):
|
|
the stall sweep's per-queue thresholds and the System activity filters both
|
|
key off it."""
|
|
from backend.app.celery_signals import _queue_for
|
|
|
|
class _T:
|
|
pass
|
|
|
|
t = _T()
|
|
t.name = name
|
|
t.app = celery
|
|
assert _queue_for(t) == queue
|
|
|
|
|
|
def test_no_task_outlives_its_stall_threshold():
|
|
"""A task whose hard time limit is longer than the stall sweep's threshold
|
|
for it gets failed 'RecoverySweep' while it is still healthy — the class
|
|
#4432 found on the ml, import and long-maintenance lanes. The threshold is
|
|
resolved the way recover_stalled_task_runs resolves it: task-name override,
|
|
then queue, then the default."""
|
|
from backend.app.celery_signals import _queue_for
|
|
from backend.app.tasks.maintenance import (
|
|
QUEUE_STUCK_THRESHOLD_MINUTES,
|
|
STUCK_THRESHOLD_MINUTES,
|
|
TASK_STUCK_THRESHOLD_MINUTES,
|
|
)
|
|
|
|
celery.loader.import_default_modules()
|
|
checked = 0
|
|
too_short = []
|
|
for name, task in sorted(celery.tasks.items()):
|
|
if name.startswith("celery."):
|
|
continue
|
|
limit = getattr(task, "time_limit", None)
|
|
if not limit:
|
|
continue
|
|
threshold = TASK_STUCK_THRESHOLD_MINUTES.get(
|
|
name,
|
|
QUEUE_STUCK_THRESHOLD_MINUTES.get(_queue_for(task), STUCK_THRESHOLD_MINUTES),
|
|
)
|
|
checked += 1
|
|
if limit / 60 > threshold:
|
|
too_short.append(f"{name}: limit {limit / 60:.0f} min > sweep {threshold} min")
|
|
assert checked > 20, "the task registry did not load — this guard checked nothing"
|
|
assert not too_short, "\n".join(too_short)
|
|
|
|
|
|
def test_backfill_phash_runs_on_the_long_lane():
|
|
"""It lives in maintenance.py, so the quick-lane glob matches it too —
|
|
the router must pick the exact name. A 35-minute rehash on the scheduler
|
|
lane blocked the minute ticks behind it (2026-09-24)."""
|
|
route = celery.amqp.router.route(
|
|
{}, "backend.app.tasks.maintenance.backfill_phash",
|
|
)
|
|
assert route["queue"].name == "maintenance_long"
|
|
quick = celery.amqp.router.route(
|
|
{}, "backend.app.tasks.maintenance.recover_stalled_task_runs",
|
|
)
|
|
assert quick["queue"].name == "maintenance"
|