fix(maintenance): recover_interrupted_tasks also sweeps pending/queued orphans (>30 min) to failed
scan_directory creates ImportTask rows with status='pending' (commit) then in a second pass transitions to 'queued' + .delay() (commit). Crashes in that window leave rows orphaned with no recovery path. Operator hit 5490 such rows 2026-05-25; the existing sweep only handled 'processing'. Flipping to 'failed' (not re-enqueue) lets the operator drain via the existing /api/import/retry-failed endpoint at their own pace. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -16,6 +16,7 @@ from ._sync_engine import sync_session_factory as _sync_session_factory
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
STUCK_THRESHOLD_MINUTES = 5
|
||||
ORPHAN_PENDING_THRESHOLD_MINUTES = 30
|
||||
OLD_TASK_DAYS = 7
|
||||
PHASH_PAGE = 500
|
||||
VERIFY_PAGE = 200
|
||||
@@ -26,40 +27,77 @@ TASK_RUN_KEEP_FAILURE_SECONDS = 7 * 24 * 3600 # 7 days
|
||||
|
||||
@celery.task(name="backend.app.tasks.maintenance.recover_interrupted_tasks")
|
||||
def recover_interrupted_tasks() -> int:
|
||||
"""Find ImportTask rows stuck in 'processing' for >5 min and re-queue them.
|
||||
"""Recover stuck ImportTask rows. Two distinct stuck states:
|
||||
|
||||
Why 5 min: import_media_file is sub-second for the vast majority of
|
||||
files; even a large-video transcode caps at the per-task soft_time_limit
|
||||
(5 min) defined on the task itself. Anything still 'processing' after
|
||||
that window is a confirmed crash (worker died, DB disconnect mid-flush,
|
||||
OOM) and must be recycled. Was 30 min historically; tightened
|
||||
2026-05-24 after operator hit a 2224-row zombie pile during the IR
|
||||
migration scan.
|
||||
1. 'processing' > 5 min — worker crash mid-import. Re-queue via
|
||||
.delay() and let the import retry. Was 30 min historically;
|
||||
tightened 2026-05-24 after operator hit a 2224-row zombie pile.
|
||||
import_media_file is sub-second for the vast majority of files and
|
||||
capped at the per-task soft_time_limit (5 min), so anything still
|
||||
'processing' after that window is a confirmed crash.
|
||||
|
||||
2. 'pending' or 'queued' > 30 min — enqueue-phase crash. scan_directory
|
||||
creates rows with status='pending' (commit), then in a second pass
|
||||
transitions to 'queued' and calls .delay() (commit). If the scanner
|
||||
crashes between those two commits, rows are orphaned in 'pending'
|
||||
(never enqueued) with no recovery path — invisible to the
|
||||
'processing' sweep above. Flagged 2026-05-25 by operator hitting a
|
||||
5490-row orphan pile. Flip these to 'failed' (not re-enqueue) so
|
||||
the operator drains them via /api/import/retry-failed at their own
|
||||
pace; bulk-re-enqueueing 5000+ rows would thundering-herd the
|
||||
import worker.
|
||||
|
||||
Returns total rows touched (recovered + marked failed).
|
||||
"""
|
||||
SessionLocal = _sync_session_factory()
|
||||
cutoff = datetime.now(UTC) - timedelta(minutes=STUCK_THRESHOLD_MINUTES)
|
||||
now = datetime.now(UTC)
|
||||
processing_cutoff = now - timedelta(minutes=STUCK_THRESHOLD_MINUTES)
|
||||
orphan_cutoff = now - timedelta(minutes=ORPHAN_PENDING_THRESHOLD_MINUTES)
|
||||
with SessionLocal() as session:
|
||||
stuck_ids = session.execute(
|
||||
select(ImportTask.id)
|
||||
.where(ImportTask.status == "processing")
|
||||
.where(ImportTask.started_at < cutoff)
|
||||
.where(ImportTask.started_at < processing_cutoff)
|
||||
).scalars().all()
|
||||
|
||||
if not stuck_ids:
|
||||
orphan_ids = session.execute(
|
||||
select(ImportTask.id)
|
||||
.where(ImportTask.status.in_(["pending", "queued"]))
|
||||
.where(ImportTask.created_at < orphan_cutoff)
|
||||
).scalars().all()
|
||||
|
||||
if not stuck_ids and not orphan_ids:
|
||||
return 0
|
||||
|
||||
session.execute(
|
||||
update(ImportTask)
|
||||
.where(ImportTask.id.in_(stuck_ids))
|
||||
.values(status="queued", started_at=None, error="recovered from stuck state")
|
||||
)
|
||||
if stuck_ids:
|
||||
session.execute(
|
||||
update(ImportTask)
|
||||
.where(ImportTask.id.in_(stuck_ids))
|
||||
.values(status="queued", started_at=None, error="recovered from stuck state")
|
||||
)
|
||||
|
||||
if orphan_ids:
|
||||
session.execute(
|
||||
update(ImportTask)
|
||||
.where(ImportTask.id.in_(orphan_ids))
|
||||
.values(
|
||||
status="failed",
|
||||
error=(
|
||||
"orphan pending/queued swept by recover_interrupted_tasks "
|
||||
"(scanner likely crashed mid-enqueue); retry via "
|
||||
"/api/import/retry-failed"
|
||||
),
|
||||
)
|
||||
)
|
||||
|
||||
session.commit()
|
||||
|
||||
from .import_file import import_media_file
|
||||
for tid in stuck_ids:
|
||||
import_media_file.delay(tid)
|
||||
if stuck_ids:
|
||||
from .import_file import import_media_file
|
||||
for tid in stuck_ids:
|
||||
import_media_file.delay(tid)
|
||||
|
||||
return len(stuck_ids)
|
||||
return len(stuck_ids) + len(orphan_ids)
|
||||
|
||||
|
||||
@celery.task(name="backend.app.tasks.maintenance.cleanup_old_tasks")
|
||||
|
||||
Reference in New Issue
Block a user