From b6a917ac81be24a2fb384a02cfa15e6286d90d52 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Mon, 25 May 2026 12:37:07 -0400 Subject: [PATCH] =?UTF-8?q?feat(import):=20/api/import/clear-stuck=20endpo?= =?UTF-8?q?int=20+=20Clear=20stuck=20UI=20button=20=E2=80=94=20escape=20ha?= =?UTF-8?q?tch=20for=20the=20autoretry-loop=20case=20the=20automatic=20swe?= =?UTF-8?q?ep=20can't=20break?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Operator hit 3 large PNGs stuck in 'processing' for 2 days 2026-05-25: the existing recover_interrupted_tasks flips processing > 5min back to queued + .delay(), but if the underlying file is unfixably broken (e.g., PIL OSError, also patched in 68cffce), the loop never terminates and the 'Scanning...' banner sticks at 0/0 forever blocking new scans. /api/import/clear-stuck: - Flips every task in pending/queued/processing to 'failed' with a clear marker error message - Finalizes any 'running' ImportBatch that has no remaining active children - Idempotent + non-destructive: rows survive, can be retried once the underlying cause is resolved UI button 'Clear stuck...' sits next to 'Retry failed' / 'Clear completed' with a warning-tonal alert in the confirm dialog explaining what it does and recommending Retry failed once the cause is fixed. Tests: clears mixed non-terminal states, untouches complete rows, finalizes orphan batch, no-op when nothing stuck. Co-Authored-By: Claude Opus 4.7 (1M context) --- backend/app/api/import_admin.py | 77 +++++++++++++++++++ .../components/settings/ImportTaskList.vue | 40 ++++++++++ frontend/src/stores/import.js | 9 ++- tests/test_api_import_admin.py | 55 +++++++++++++ 4 files changed, 180 insertions(+), 1 deletion(-) diff --git a/backend/app/api/import_admin.py b/backend/app/api/import_admin.py index 6a16bd3..f4ecf07 100644 --- a/backend/app/api/import_admin.py +++ b/backend/app/api/import_admin.py @@ -120,6 +120,83 @@ async def retry_failed(): return jsonify({"retried": len(failed_ids)}) +@import_admin_bp.route("/clear-stuck", methods=["POST"]) +async def clear_stuck(): + """Force any non-terminal ImportTask (status in pending/queued/ + processing) to 'failed' AND finalize any ImportBatch that ends up + with no active children. Escape hatch for the operator when the + automatic recover_interrupted_tasks sweep keeps re-queueing the + same stuck row forever (e.g., underlying file is genuinely broken + and the import keeps OSError-looping at PIL load). + + Idempotent + non-destructive: rows survive as 'failed' so the + Retry-Failed button can re-attempt them once whatever was broken + is fixed. Banked 2026-05-25 — operator hit 3 large PNGs that + autoretry-looped for 2 days after a corrupt-data PIL OSError. + """ + async with get_session() as session: + stuck_ids = ( + await session.execute( + select(ImportTask.id).where( + ImportTask.status.in_(["pending", "queued", "processing"]) + ) + ) + ).scalars().all() + if stuck_ids: + await session.execute( + update(ImportTask) + .where(ImportTask.id.in_(stuck_ids)) + .values( + status="failed", + finished_at=datetime.now(UTC), + error=( + "manually cleared via /api/import/clear-stuck " + "— stuck in non-terminal state; retry once " + "underlying cause (corrupt file, missing model, " + "etc.) is resolved" + ), + ) + ) + + # Finalize any 'running' ImportBatch that no longer has any + # active children. The "Scanning..." banner is driven by + # /api/import/status finding a running batch; left untouched, + # it would persist forever after the stuck-task clear. + running_batches = ( + await session.execute( + select(ImportBatch.id).where(ImportBatch.status == "running") + ) + ).scalars().all() + finalized_batches = 0 + for batch_id in running_batches: + still_active = ( + await session.execute( + select(ImportTask.id) + .where(ImportTask.batch_id == batch_id) + .where(ImportTask.status.in_( + ["pending", "queued", "processing"] + )) + .limit(1) + ) + ).scalar_one_or_none() + if still_active is None: + await session.execute( + update(ImportBatch) + .where(ImportBatch.id == batch_id) + .values( + status="complete", + finished_at=datetime.now(UTC), + ) + ) + finalized_batches += 1 + await session.commit() + + return jsonify({ + "tasks_failed": len(stuck_ids), + "batches_finalized": finalized_batches, + }) + + @import_admin_bp.route("/clear-completed", methods=["POST"]) async def clear_completed(): body = await request.get_json(silent=True) or {} diff --git a/frontend/src/components/settings/ImportTaskList.vue b/frontend/src/components/settings/ImportTaskList.vue index a78562d..c3edbd1 100644 --- a/frontend/src/components/settings/ImportTaskList.vue +++ b/frontend/src/components/settings/ImportTaskList.vue @@ -17,6 +17,12 @@ > Retry failed + + Clear stuck… + + + + + Clear stuck tasks + + + Force every pending / queued / processing task to + failed and finalize any active batch that + has no remaining work. Use this when the automatic recovery + sweep keeps re-queueing the same row (e.g., corrupt file in + an autoretry loop, or worker model missing). + +

+ Tasks remain in the database with status=failed; + click Retry failed once the underlying cause is + resolved to re-queue them. +

+ + + + Cancel + Clear stuck + + + @@ -80,6 +111,7 @@ const store = useImportStore() const statusFilter = ref(null) const clearDialog = ref(false) const clearAgeDays = ref(7) +const clearStuckDialog = ref(false) const statusOptions = [ { title: 'All', value: null }, @@ -100,6 +132,9 @@ const headers = [ ] const hasFailed = computed(() => store.tasks.some(t => t.status === 'failed')) +const hasStuck = computed(() => store.tasks.some( + t => t.status === 'pending' || t.status === 'queued' || t.status === 'processing' +)) function statusColor(s) { return { @@ -138,4 +173,9 @@ async function onClearConfirm() { await store.clearCompleted(clearAgeDays.value) clearDialog.value = false } +function onClearStuckOpen() { clearStuckDialog.value = true } +async function onClearStuckConfirm() { + await store.clearStuck() + clearStuckDialog.value = false +} diff --git a/frontend/src/stores/import.js b/frontend/src/stores/import.js index 861db25..9d4832e 100644 --- a/frontend/src/stores/import.js +++ b/frontend/src/stores/import.js @@ -92,6 +92,13 @@ export const useImportStore = defineStore('import', () => { await loadTasks(true) } + async function clearStuck() { + const body = await api.post('/api/import/clear-stuck') + await loadTasks(true) + await refreshStatus() + return body + } + const hasMore = computed(() => tasksNextCursor.value !== null) return { @@ -101,6 +108,6 @@ export const useImportStore = defineStore('import', () => { triggerError, loadSettings, patchSettings, refreshStatus, triggerScan, - loadTasks, setStatusFilter, retryFailed, clearCompleted + loadTasks, setStatusFilter, retryFailed, clearCompleted, clearStuck } }) diff --git a/tests/test_api_import_admin.py b/tests/test_api_import_admin.py index 7e15f4b..f5512aa 100644 --- a/tests/test_api_import_admin.py +++ b/tests/test_api_import_admin.py @@ -86,6 +86,61 @@ async def test_clear_completed(client, db): assert body["deleted"] == 1 +@pytest.mark.asyncio +async def test_clear_stuck_fails_non_terminal_and_finalizes_orphan_batch(client, db): + """Operator-flagged 2026-05-25: 3 large PNGs got stuck in 'processing' + for 2 days, the active ImportBatch never finalized, and the UI's + 'Scanning...' banner persisted with 0/0 files. /api/import/clear-stuck + is the escape hatch to break the autoretry loop manually.""" + from sqlalchemy import select as _select + + batch = ImportBatch(triggered_by="manual", source_path="/import", scan_mode="quick") + db.add(batch) + await db.flush() + # Three stuck rows in mixed non-terminal states. + db.add(ImportTask( + batch_id=batch.id, source_path="/p1", task_type="media", status="processing", + )) + db.add(ImportTask( + batch_id=batch.id, source_path="/p2", task_type="media", status="queued", + )) + db.add(ImportTask( + batch_id=batch.id, source_path="/p3", task_type="media", status="pending", + )) + # One already-complete row should be untouched. + db.add(ImportTask( + batch_id=batch.id, source_path="/done", task_type="media", + status="complete", finished_at=datetime.now(UTC), + )) + await db.commit() + + resp = await client.post("/api/import/clear-stuck") + body = await resp.get_json() + assert resp.status_code == 200 + assert body["tasks_failed"] == 3 + assert body["batches_finalized"] == 1 + + statuses = { + row.status for row in + (await db.execute(_select(ImportTask).where(ImportTask.batch_id == batch.id))) + .scalars().all() + } + assert statuses == {"failed", "complete"} + + batch_status = (await db.execute( + _select(ImportBatch.status).where(ImportBatch.id == batch.id) + )).scalar_one() + assert batch_status == "complete" + + +@pytest.mark.asyncio +async def test_clear_stuck_no_op_when_nothing_stuck(client, db): + resp = await client.post("/api/import/clear-stuck") + body = await resp.get_json() + assert resp.status_code == 200 + assert body == {"tasks_failed": 0, "batches_finalized": 0} + + @pytest.mark.asyncio async def test_trigger_accepts_deep(client, monkeypatch): # Stub the task dispatch — assert the API accepts 'deep' and forwards