fix(maintenance): recover_interrupted_tasks also sweeps pending/queued orphans (>30 min) to failed

scan_directory creates ImportTask rows with status='pending' (commit) then
in a second pass transitions to 'queued' + .delay() (commit). Crashes in
that window leave rows orphaned with no recovery path. Operator hit 5490
such rows 2026-05-25; the existing sweep only handled 'processing'.
Flipping to 'failed' (not re-enqueue) lets the operator drain via the
existing /api/import/retry-failed endpoint at their own pace.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
2026-05-25 02:00:29 -04:00
parent 832345a245
commit 9d5abb09f6
2 changed files with 154 additions and 20 deletions
+58 -20
View File
@@ -16,6 +16,7 @@ from ._sync_engine import sync_session_factory as _sync_session_factory
log = logging.getLogger(__name__)
STUCK_THRESHOLD_MINUTES = 5
ORPHAN_PENDING_THRESHOLD_MINUTES = 30
OLD_TASK_DAYS = 7
PHASH_PAGE = 500
VERIFY_PAGE = 200
@@ -26,40 +27,77 @@ TASK_RUN_KEEP_FAILURE_SECONDS = 7 * 24 * 3600 # 7 days
@celery.task(name="backend.app.tasks.maintenance.recover_interrupted_tasks")
def recover_interrupted_tasks() -> int:
"""Find ImportTask rows stuck in 'processing' for >5 min and re-queue them.
"""Recover stuck ImportTask rows. Two distinct stuck states:
Why 5 min: import_media_file is sub-second for the vast majority of
files; even a large-video transcode caps at the per-task soft_time_limit
(5 min) defined on the task itself. Anything still 'processing' after
that window is a confirmed crash (worker died, DB disconnect mid-flush,
OOM) and must be recycled. Was 30 min historically; tightened
2026-05-24 after operator hit a 2224-row zombie pile during the IR
migration scan.
1. 'processing' > 5 min — worker crash mid-import. Re-queue via
.delay() and let the import retry. Was 30 min historically;
tightened 2026-05-24 after operator hit a 2224-row zombie pile.
import_media_file is sub-second for the vast majority of files and
capped at the per-task soft_time_limit (5 min), so anything still
'processing' after that window is a confirmed crash.
2. 'pending' or 'queued' > 30 min — enqueue-phase crash. scan_directory
creates rows with status='pending' (commit), then in a second pass
transitions to 'queued' and calls .delay() (commit). If the scanner
crashes between those two commits, rows are orphaned in 'pending'
(never enqueued) with no recovery path — invisible to the
'processing' sweep above. Flagged 2026-05-25 by operator hitting a
5490-row orphan pile. Flip these to 'failed' (not re-enqueue) so
the operator drains them via /api/import/retry-failed at their own
pace; bulk-re-enqueueing 5000+ rows would thundering-herd the
import worker.
Returns total rows touched (recovered + marked failed).
"""
SessionLocal = _sync_session_factory()
cutoff = datetime.now(UTC) - timedelta(minutes=STUCK_THRESHOLD_MINUTES)
now = datetime.now(UTC)
processing_cutoff = now - timedelta(minutes=STUCK_THRESHOLD_MINUTES)
orphan_cutoff = now - timedelta(minutes=ORPHAN_PENDING_THRESHOLD_MINUTES)
with SessionLocal() as session:
stuck_ids = session.execute(
select(ImportTask.id)
.where(ImportTask.status == "processing")
.where(ImportTask.started_at < cutoff)
.where(ImportTask.started_at < processing_cutoff)
).scalars().all()
if not stuck_ids:
orphan_ids = session.execute(
select(ImportTask.id)
.where(ImportTask.status.in_(["pending", "queued"]))
.where(ImportTask.created_at < orphan_cutoff)
).scalars().all()
if not stuck_ids and not orphan_ids:
return 0
session.execute(
update(ImportTask)
.where(ImportTask.id.in_(stuck_ids))
.values(status="queued", started_at=None, error="recovered from stuck state")
)
if stuck_ids:
session.execute(
update(ImportTask)
.where(ImportTask.id.in_(stuck_ids))
.values(status="queued", started_at=None, error="recovered from stuck state")
)
if orphan_ids:
session.execute(
update(ImportTask)
.where(ImportTask.id.in_(orphan_ids))
.values(
status="failed",
error=(
"orphan pending/queued swept by recover_interrupted_tasks "
"(scanner likely crashed mid-enqueue); retry via "
"/api/import/retry-failed"
),
)
)
session.commit()
from .import_file import import_media_file
for tid in stuck_ids:
import_media_file.delay(tid)
if stuck_ids:
from .import_file import import_media_file
for tid in stuck_ids:
import_media_file.delay(tid)
return len(stuck_ids)
return len(stuck_ids) + len(orphan_ids)
@celery.task(name="backend.app.tasks.maintenance.cleanup_old_tasks")