fix(maintenance): recover_interrupted_tasks — fold SELECT into UPDATE…WHERE…RETURNING so the IN-list no longer blows past psycopg's 65535-parameter ceiling (operator-hit 2026-05-26 after deep scan orphan pile)
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -54,41 +54,40 @@ def recover_interrupted_tasks() -> int:
|
|||||||
processing_cutoff = now - timedelta(minutes=STUCK_THRESHOLD_MINUTES)
|
processing_cutoff = now - timedelta(minutes=STUCK_THRESHOLD_MINUTES)
|
||||||
orphan_cutoff = now - timedelta(minutes=ORPHAN_PENDING_THRESHOLD_MINUTES)
|
orphan_cutoff = now - timedelta(minutes=ORPHAN_PENDING_THRESHOLD_MINUTES)
|
||||||
with SessionLocal() as session:
|
with SessionLocal() as session:
|
||||||
stuck_ids = session.execute(
|
# Both sweeps used to be SELECT ids → UPDATE WHERE id IN (...) which
|
||||||
select(ImportTask.id)
|
# blew past psycopg's 65535-parameter ceiling once a sweep covered
|
||||||
|
# tens of thousands of rows (operator hit it 2026-05-26 after the
|
||||||
|
# /import deep scan piled up orphans). Folding the SELECT into the
|
||||||
|
# UPDATE eliminates the IN-list entirely. RETURNING gives us back
|
||||||
|
# exactly the ids that flipped so the stuck sweep can still
|
||||||
|
# .delay() each one.
|
||||||
|
stuck_result = session.execute(
|
||||||
|
update(ImportTask)
|
||||||
.where(ImportTask.status == "processing")
|
.where(ImportTask.status == "processing")
|
||||||
.where(ImportTask.started_at < processing_cutoff)
|
.where(ImportTask.started_at < processing_cutoff)
|
||||||
).scalars().all()
|
.values(
|
||||||
|
status="queued",
|
||||||
|
started_at=None,
|
||||||
|
error="recovered from stuck state",
|
||||||
|
)
|
||||||
|
.returning(ImportTask.id)
|
||||||
|
)
|
||||||
|
stuck_ids = [row[0] for row in stuck_result.all()]
|
||||||
|
|
||||||
orphan_ids = session.execute(
|
orphan_result = session.execute(
|
||||||
select(ImportTask.id)
|
update(ImportTask)
|
||||||
.where(ImportTask.status.in_(["pending", "queued"]))
|
.where(ImportTask.status.in_(["pending", "queued"]))
|
||||||
.where(ImportTask.created_at < orphan_cutoff)
|
.where(ImportTask.created_at < orphan_cutoff)
|
||||||
).scalars().all()
|
.values(
|
||||||
|
status="failed",
|
||||||
if not stuck_ids and not orphan_ids:
|
error=(
|
||||||
return 0
|
"orphan pending/queued swept by recover_interrupted_tasks "
|
||||||
|
"(scanner likely crashed mid-enqueue); retry via "
|
||||||
if stuck_ids:
|
"/api/import/retry-failed"
|
||||||
session.execute(
|
),
|
||||||
update(ImportTask)
|
|
||||||
.where(ImportTask.id.in_(stuck_ids))
|
|
||||||
.values(status="queued", started_at=None, error="recovered from stuck state")
|
|
||||||
)
|
|
||||||
|
|
||||||
if orphan_ids:
|
|
||||||
session.execute(
|
|
||||||
update(ImportTask)
|
|
||||||
.where(ImportTask.id.in_(orphan_ids))
|
|
||||||
.values(
|
|
||||||
status="failed",
|
|
||||||
error=(
|
|
||||||
"orphan pending/queued swept by recover_interrupted_tasks "
|
|
||||||
"(scanner likely crashed mid-enqueue); retry via "
|
|
||||||
"/api/import/retry-failed"
|
|
||||||
),
|
|
||||||
)
|
|
||||||
)
|
)
|
||||||
|
)
|
||||||
|
orphan_count = orphan_result.rowcount or 0
|
||||||
|
|
||||||
session.commit()
|
session.commit()
|
||||||
|
|
||||||
@@ -97,7 +96,7 @@ def recover_interrupted_tasks() -> int:
|
|||||||
for tid in stuck_ids:
|
for tid in stuck_ids:
|
||||||
import_media_file.delay(tid)
|
import_media_file.delay(tid)
|
||||||
|
|
||||||
return len(stuck_ids) + len(orphan_ids)
|
return len(stuck_ids) + orphan_count
|
||||||
|
|
||||||
|
|
||||||
@celery.task(name="backend.app.tasks.maintenance.cleanup_old_tasks")
|
@celery.task(name="backend.app.tasks.maintenance.cleanup_old_tasks")
|
||||||
|
|||||||
Reference in New Issue
Block a user