fix(maintenance): recover_interrupted_tasks also sweeps pending/queued orphans (>30 min) to failed
scan_directory creates ImportTask rows with status='pending' (commit) then in a second pass transitions to 'queued' + .delay() (commit). Crashes in that window leave rows orphaned with no recovery path. Operator hit 5490 such rows 2026-05-25; the existing sweep only handled 'processing'. Flipping to 'failed' (not re-enqueue) lets the operator drain via the existing /api/import/retry-failed endpoint at their own pace. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -68,6 +68,102 @@ def test_recover_interrupted_only_old(db_sync, monkeypatch):
|
||||
assert dispatched == [stale.id]
|
||||
|
||||
|
||||
def test_recover_interrupted_sweeps_pending_orphans_to_failed(db_sync, monkeypatch):
|
||||
"""A scan that creates ImportTask rows but crashes before the second
|
||||
pass (transition to 'queued' + .delay()) leaves rows orphaned at
|
||||
status='pending'. The sweep flips them to 'failed' so the operator
|
||||
can drain via /api/import/retry-failed without thundering-herding.
|
||||
Banked 2026-05-25 after operator hit 5490 stuck pending rows.
|
||||
"""
|
||||
from backend.app.tasks import import_file
|
||||
monkeypatch.setattr(import_file.import_media_file, "delay", lambda *_: None)
|
||||
|
||||
batch_id = _make_batch(db_sync)
|
||||
now = datetime.now(UTC)
|
||||
|
||||
fresh_pending = ImportTask(
|
||||
batch_id=batch_id, source_path="/import/fresh.jpg", task_type="media",
|
||||
status="pending",
|
||||
)
|
||||
db_sync.add(fresh_pending)
|
||||
db_sync.flush()
|
||||
# created_at defaults to now() server-side; fresh row stays untouched.
|
||||
|
||||
# Two stale rows simulating the orphan pile: one 'pending', one
|
||||
# 'queued' (scanner crashed AFTER transitioning some rows but
|
||||
# before all). Both should sweep.
|
||||
stale_pending = ImportTask(
|
||||
batch_id=batch_id, source_path="/import/stale1.jpg", task_type="media",
|
||||
status="pending",
|
||||
)
|
||||
stale_queued = ImportTask(
|
||||
batch_id=batch_id, source_path="/import/stale2.jpg", task_type="media",
|
||||
status="queued",
|
||||
)
|
||||
db_sync.add_all([stale_pending, stale_queued])
|
||||
db_sync.flush()
|
||||
# Backdate created_at past the orphan cutoff (30 min).
|
||||
from sqlalchemy import update as _upd
|
||||
db_sync.execute(
|
||||
_upd(ImportTask)
|
||||
.where(ImportTask.id.in_([stale_pending.id, stale_queued.id]))
|
||||
.values(created_at=now - timedelta(hours=2))
|
||||
)
|
||||
db_sync.commit()
|
||||
|
||||
from backend.app.tasks.maintenance import recover_interrupted_tasks
|
||||
touched = recover_interrupted_tasks.apply().get()
|
||||
assert touched == 2
|
||||
|
||||
db_sync.refresh(fresh_pending)
|
||||
db_sync.refresh(stale_pending)
|
||||
db_sync.refresh(stale_queued)
|
||||
assert fresh_pending.status == "pending" # fresh row untouched
|
||||
assert stale_pending.status == "failed"
|
||||
assert stale_queued.status == "failed"
|
||||
assert "orphan" in (stale_pending.error or "")
|
||||
|
||||
|
||||
def test_recover_interrupted_handles_both_stuck_and_orphans(db_sync, monkeypatch):
|
||||
"""One sweep tick handles both 'processing' crashes AND
|
||||
'pending'/'queued' orphans in a single pass."""
|
||||
from backend.app.tasks import import_file
|
||||
dispatched: list[int] = []
|
||||
monkeypatch.setattr(
|
||||
import_file.import_media_file, "delay", dispatched.append
|
||||
)
|
||||
|
||||
batch_id = _make_batch(db_sync)
|
||||
now = datetime.now(UTC)
|
||||
|
||||
stuck = ImportTask(
|
||||
batch_id=batch_id, source_path="/import/stuck.jpg", task_type="media",
|
||||
status="processing", started_at=now - timedelta(hours=2),
|
||||
)
|
||||
orphan = ImportTask(
|
||||
batch_id=batch_id, source_path="/import/orphan.jpg", task_type="media",
|
||||
status="pending",
|
||||
)
|
||||
db_sync.add_all([stuck, orphan])
|
||||
db_sync.flush()
|
||||
from sqlalchemy import update as _upd
|
||||
db_sync.execute(
|
||||
_upd(ImportTask).where(ImportTask.id == orphan.id)
|
||||
.values(created_at=now - timedelta(hours=2))
|
||||
)
|
||||
db_sync.commit()
|
||||
|
||||
from backend.app.tasks.maintenance import recover_interrupted_tasks
|
||||
touched = recover_interrupted_tasks.apply().get()
|
||||
assert touched == 2
|
||||
|
||||
db_sync.refresh(stuck)
|
||||
db_sync.refresh(orphan)
|
||||
assert stuck.status == "queued"
|
||||
assert orphan.status == "failed"
|
||||
assert dispatched == [stuck.id] # stuck rows re-enqueue; orphans don't
|
||||
|
||||
|
||||
def test_cleanup_old_deletes_finished_old(db_sync):
|
||||
batch_id = _make_batch(db_sync)
|
||||
now = datetime.now(UTC)
|
||||
|
||||
Reference in New Issue
Block a user