fix(maintenance): recover_interrupted_tasks also sweeps pending/queued orphans (>30 min) to failed

scan_directory creates ImportTask rows with status='pending' (commit) then
in a second pass transitions to 'queued' + .delay() (commit). Crashes in
that window leave rows orphaned with no recovery path. Operator hit 5490
such rows 2026-05-25; the existing sweep only handled 'processing'.
Flipping to 'failed' (not re-enqueue) lets the operator drain via the
existing /api/import/retry-failed endpoint at their own pace.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
2026-05-25 02:00:29 -04:00
parent 832345a245
commit 9d5abb09f6
2 changed files with 154 additions and 20 deletions
+96
View File
@@ -68,6 +68,102 @@ def test_recover_interrupted_only_old(db_sync, monkeypatch):
assert dispatched == [stale.id]
def test_recover_interrupted_sweeps_pending_orphans_to_failed(db_sync, monkeypatch):
"""A scan that creates ImportTask rows but crashes before the second
pass (transition to 'queued' + .delay()) leaves rows orphaned at
status='pending'. The sweep flips them to 'failed' so the operator
can drain via /api/import/retry-failed without thundering-herding.
Banked 2026-05-25 after operator hit 5490 stuck pending rows.
"""
from backend.app.tasks import import_file
monkeypatch.setattr(import_file.import_media_file, "delay", lambda *_: None)
batch_id = _make_batch(db_sync)
now = datetime.now(UTC)
fresh_pending = ImportTask(
batch_id=batch_id, source_path="/import/fresh.jpg", task_type="media",
status="pending",
)
db_sync.add(fresh_pending)
db_sync.flush()
# created_at defaults to now() server-side; fresh row stays untouched.
# Two stale rows simulating the orphan pile: one 'pending', one
# 'queued' (scanner crashed AFTER transitioning some rows but
# before all). Both should sweep.
stale_pending = ImportTask(
batch_id=batch_id, source_path="/import/stale1.jpg", task_type="media",
status="pending",
)
stale_queued = ImportTask(
batch_id=batch_id, source_path="/import/stale2.jpg", task_type="media",
status="queued",
)
db_sync.add_all([stale_pending, stale_queued])
db_sync.flush()
# Backdate created_at past the orphan cutoff (30 min).
from sqlalchemy import update as _upd
db_sync.execute(
_upd(ImportTask)
.where(ImportTask.id.in_([stale_pending.id, stale_queued.id]))
.values(created_at=now - timedelta(hours=2))
)
db_sync.commit()
from backend.app.tasks.maintenance import recover_interrupted_tasks
touched = recover_interrupted_tasks.apply().get()
assert touched == 2
db_sync.refresh(fresh_pending)
db_sync.refresh(stale_pending)
db_sync.refresh(stale_queued)
assert fresh_pending.status == "pending" # fresh row untouched
assert stale_pending.status == "failed"
assert stale_queued.status == "failed"
assert "orphan" in (stale_pending.error or "")
def test_recover_interrupted_handles_both_stuck_and_orphans(db_sync, monkeypatch):
"""One sweep tick handles both 'processing' crashes AND
'pending'/'queued' orphans in a single pass."""
from backend.app.tasks import import_file
dispatched: list[int] = []
monkeypatch.setattr(
import_file.import_media_file, "delay", dispatched.append
)
batch_id = _make_batch(db_sync)
now = datetime.now(UTC)
stuck = ImportTask(
batch_id=batch_id, source_path="/import/stuck.jpg", task_type="media",
status="processing", started_at=now - timedelta(hours=2),
)
orphan = ImportTask(
batch_id=batch_id, source_path="/import/orphan.jpg", task_type="media",
status="pending",
)
db_sync.add_all([stuck, orphan])
db_sync.flush()
from sqlalchemy import update as _upd
db_sync.execute(
_upd(ImportTask).where(ImportTask.id == orphan.id)
.values(created_at=now - timedelta(hours=2))
)
db_sync.commit()
from backend.app.tasks.maintenance import recover_interrupted_tasks
touched = recover_interrupted_tasks.apply().get()
assert touched == 2
db_sync.refresh(stuck)
db_sync.refresh(orphan)
assert stuck.status == "queued"
assert orphan.status == "failed"
assert dispatched == [stuck.id] # stuck rows re-enqueue; orphans don't
def test_cleanup_old_deletes_finished_old(db_sync):
batch_id = _make_batch(db_sync)
now = datetime.now(UTC)