fix(workers): worker-level recovery — autoretry on transient errors + tightened sweep (5min) + import_media_file body-wrap so no path leaves rows stuck in 'processing'
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -15,7 +15,7 @@ from ._sync_engine import sync_session_factory as _sync_session_factory
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
STUCK_THRESHOLD_MINUTES = 30
|
||||
STUCK_THRESHOLD_MINUTES = 5
|
||||
OLD_TASK_DAYS = 7
|
||||
PHASH_PAGE = 500
|
||||
VERIFY_PAGE = 200
|
||||
@@ -24,11 +24,15 @@ FFPROBE_TIMEOUT_SECONDS = 10
|
||||
|
||||
@celery.task(name="backend.app.tasks.maintenance.recover_interrupted_tasks")
|
||||
def recover_interrupted_tasks() -> int:
|
||||
"""Find ImportTask rows stuck in 'processing' for >30 min and re-queue them.
|
||||
"""Find ImportTask rows stuck in 'processing' for >5 min and re-queue them.
|
||||
|
||||
Why 30 min: large videos can legitimately take many minutes to import;
|
||||
30 is a safe gate that catches actual crashes (which leave the row stuck
|
||||
forever) without resetting slow-but-still-running jobs.
|
||||
Why 5 min: import_media_file is sub-second for the vast majority of
|
||||
files; even a large-video transcode caps at the per-task soft_time_limit
|
||||
(5 min) defined on the task itself. Anything still 'processing' after
|
||||
that window is a confirmed crash (worker died, DB disconnect mid-flush,
|
||||
OOM) and must be recycled. Was 30 min historically; tightened
|
||||
2026-05-24 after operator hit a 2224-row zombie pile during the IR
|
||||
migration scan.
|
||||
"""
|
||||
SessionLocal = _sync_session_factory()
|
||||
cutoff = datetime.now(UTC) - timedelta(minutes=STUCK_THRESHOLD_MINUTES)
|
||||
|
||||
Reference in New Issue
Block a user