feat(import): /api/import/clear-stuck endpoint + Clear stuck UI button — escape hatch for the autoretry-loop case the automatic sweep can't break
Operator hit 3 large PNGs stuck in 'processing' for 2 days 2026-05-25:
the existing recover_interrupted_tasks flips processing > 5min back to
queued + .delay(), but if the underlying file is unfixably broken (e.g.,
PIL OSError, also patched in 68cffce), the loop never terminates and the
'Scanning...' banner sticks at 0/0 forever blocking new scans.
/api/import/clear-stuck:
- Flips every task in pending/queued/processing to 'failed' with a clear
marker error message
- Finalizes any 'running' ImportBatch that has no remaining active children
- Idempotent + non-destructive: rows survive, can be retried once the
underlying cause is resolved
UI button 'Clear stuck...' sits next to 'Retry failed' / 'Clear completed'
with a warning-tonal alert in the confirm dialog explaining what it does
and recommending Retry failed once the cause is fixed.
Tests: clears mixed non-terminal states, untouches complete rows,
finalizes orphan batch, no-op when nothing stuck.
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -120,6 +120,83 @@ async def retry_failed():
|
||||
return jsonify({"retried": len(failed_ids)})
|
||||
|
||||
|
||||
@import_admin_bp.route("/clear-stuck", methods=["POST"])
|
||||
async def clear_stuck():
|
||||
"""Force any non-terminal ImportTask (status in pending/queued/
|
||||
processing) to 'failed' AND finalize any ImportBatch that ends up
|
||||
with no active children. Escape hatch for the operator when the
|
||||
automatic recover_interrupted_tasks sweep keeps re-queueing the
|
||||
same stuck row forever (e.g., underlying file is genuinely broken
|
||||
and the import keeps OSError-looping at PIL load).
|
||||
|
||||
Idempotent + non-destructive: rows survive as 'failed' so the
|
||||
Retry-Failed button can re-attempt them once whatever was broken
|
||||
is fixed. Banked 2026-05-25 — operator hit 3 large PNGs that
|
||||
autoretry-looped for 2 days after a corrupt-data PIL OSError.
|
||||
"""
|
||||
async with get_session() as session:
|
||||
stuck_ids = (
|
||||
await session.execute(
|
||||
select(ImportTask.id).where(
|
||||
ImportTask.status.in_(["pending", "queued", "processing"])
|
||||
)
|
||||
)
|
||||
).scalars().all()
|
||||
if stuck_ids:
|
||||
await session.execute(
|
||||
update(ImportTask)
|
||||
.where(ImportTask.id.in_(stuck_ids))
|
||||
.values(
|
||||
status="failed",
|
||||
finished_at=datetime.now(UTC),
|
||||
error=(
|
||||
"manually cleared via /api/import/clear-stuck "
|
||||
"— stuck in non-terminal state; retry once "
|
||||
"underlying cause (corrupt file, missing model, "
|
||||
"etc.) is resolved"
|
||||
),
|
||||
)
|
||||
)
|
||||
|
||||
# Finalize any 'running' ImportBatch that no longer has any
|
||||
# active children. The "Scanning..." banner is driven by
|
||||
# /api/import/status finding a running batch; left untouched,
|
||||
# it would persist forever after the stuck-task clear.
|
||||
running_batches = (
|
||||
await session.execute(
|
||||
select(ImportBatch.id).where(ImportBatch.status == "running")
|
||||
)
|
||||
).scalars().all()
|
||||
finalized_batches = 0
|
||||
for batch_id in running_batches:
|
||||
still_active = (
|
||||
await session.execute(
|
||||
select(ImportTask.id)
|
||||
.where(ImportTask.batch_id == batch_id)
|
||||
.where(ImportTask.status.in_(
|
||||
["pending", "queued", "processing"]
|
||||
))
|
||||
.limit(1)
|
||||
)
|
||||
).scalar_one_or_none()
|
||||
if still_active is None:
|
||||
await session.execute(
|
||||
update(ImportBatch)
|
||||
.where(ImportBatch.id == batch_id)
|
||||
.values(
|
||||
status="complete",
|
||||
finished_at=datetime.now(UTC),
|
||||
)
|
||||
)
|
||||
finalized_batches += 1
|
||||
await session.commit()
|
||||
|
||||
return jsonify({
|
||||
"tasks_failed": len(stuck_ids),
|
||||
"batches_finalized": finalized_batches,
|
||||
})
|
||||
|
||||
|
||||
@import_admin_bp.route("/clear-completed", methods=["POST"])
|
||||
async def clear_completed():
|
||||
body = await request.get_json(silent=True) or {}
|
||||
|
||||
@@ -17,6 +17,12 @@
|
||||
>
|
||||
Retry failed
|
||||
</v-btn>
|
||||
<v-btn
|
||||
variant="text" rounded="pill" size="small" color="warning"
|
||||
:disabled="!hasStuck" @click="onClearStuckOpen"
|
||||
>
|
||||
Clear stuck…
|
||||
</v-btn>
|
||||
<v-btn
|
||||
variant="text" rounded="pill" size="small" color="error"
|
||||
@click="onClearOpen"
|
||||
@@ -69,6 +75,31 @@
|
||||
</v-card-actions>
|
||||
</v-card>
|
||||
</v-dialog>
|
||||
|
||||
<v-dialog v-model="clearStuckDialog" max-width="480">
|
||||
<v-card>
|
||||
<v-card-title>Clear stuck tasks</v-card-title>
|
||||
<v-card-text>
|
||||
<v-alert type="warning" variant="tonal" density="compact" class="mb-3">
|
||||
Force every <strong>pending / queued / processing</strong> task to
|
||||
<strong>failed</strong> and finalize any active batch that
|
||||
has no remaining work. Use this when the automatic recovery
|
||||
sweep keeps re-queueing the same row (e.g., corrupt file in
|
||||
an autoretry loop, or worker model missing).
|
||||
</v-alert>
|
||||
<p class="text-body-2">
|
||||
Tasks remain in the database with status=<code>failed</code>;
|
||||
click <em>Retry failed</em> once the underlying cause is
|
||||
resolved to re-queue them.
|
||||
</p>
|
||||
</v-card-text>
|
||||
<v-card-actions>
|
||||
<v-spacer />
|
||||
<v-btn @click="clearStuckDialog = false">Cancel</v-btn>
|
||||
<v-btn color="warning" rounded="pill" @click="onClearStuckConfirm">Clear stuck</v-btn>
|
||||
</v-card-actions>
|
||||
</v-card>
|
||||
</v-dialog>
|
||||
</v-card>
|
||||
</template>
|
||||
|
||||
@@ -80,6 +111,7 @@ const store = useImportStore()
|
||||
const statusFilter = ref(null)
|
||||
const clearDialog = ref(false)
|
||||
const clearAgeDays = ref(7)
|
||||
const clearStuckDialog = ref(false)
|
||||
|
||||
const statusOptions = [
|
||||
{ title: 'All', value: null },
|
||||
@@ -100,6 +132,9 @@ const headers = [
|
||||
]
|
||||
|
||||
const hasFailed = computed(() => store.tasks.some(t => t.status === 'failed'))
|
||||
const hasStuck = computed(() => store.tasks.some(
|
||||
t => t.status === 'pending' || t.status === 'queued' || t.status === 'processing'
|
||||
))
|
||||
|
||||
function statusColor(s) {
|
||||
return {
|
||||
@@ -138,4 +173,9 @@ async function onClearConfirm() {
|
||||
await store.clearCompleted(clearAgeDays.value)
|
||||
clearDialog.value = false
|
||||
}
|
||||
function onClearStuckOpen() { clearStuckDialog.value = true }
|
||||
async function onClearStuckConfirm() {
|
||||
await store.clearStuck()
|
||||
clearStuckDialog.value = false
|
||||
}
|
||||
</script>
|
||||
|
||||
@@ -92,6 +92,13 @@ export const useImportStore = defineStore('import', () => {
|
||||
await loadTasks(true)
|
||||
}
|
||||
|
||||
async function clearStuck() {
|
||||
const body = await api.post('/api/import/clear-stuck')
|
||||
await loadTasks(true)
|
||||
await refreshStatus()
|
||||
return body
|
||||
}
|
||||
|
||||
const hasMore = computed(() => tasksNextCursor.value !== null)
|
||||
|
||||
return {
|
||||
@@ -101,6 +108,6 @@ export const useImportStore = defineStore('import', () => {
|
||||
triggerError,
|
||||
loadSettings, patchSettings,
|
||||
refreshStatus, triggerScan,
|
||||
loadTasks, setStatusFilter, retryFailed, clearCompleted
|
||||
loadTasks, setStatusFilter, retryFailed, clearCompleted, clearStuck
|
||||
}
|
||||
})
|
||||
|
||||
@@ -86,6 +86,61 @@ async def test_clear_completed(client, db):
|
||||
assert body["deleted"] == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_clear_stuck_fails_non_terminal_and_finalizes_orphan_batch(client, db):
|
||||
"""Operator-flagged 2026-05-25: 3 large PNGs got stuck in 'processing'
|
||||
for 2 days, the active ImportBatch never finalized, and the UI's
|
||||
'Scanning...' banner persisted with 0/0 files. /api/import/clear-stuck
|
||||
is the escape hatch to break the autoretry loop manually."""
|
||||
from sqlalchemy import select as _select
|
||||
|
||||
batch = ImportBatch(triggered_by="manual", source_path="/import", scan_mode="quick")
|
||||
db.add(batch)
|
||||
await db.flush()
|
||||
# Three stuck rows in mixed non-terminal states.
|
||||
db.add(ImportTask(
|
||||
batch_id=batch.id, source_path="/p1", task_type="media", status="processing",
|
||||
))
|
||||
db.add(ImportTask(
|
||||
batch_id=batch.id, source_path="/p2", task_type="media", status="queued",
|
||||
))
|
||||
db.add(ImportTask(
|
||||
batch_id=batch.id, source_path="/p3", task_type="media", status="pending",
|
||||
))
|
||||
# One already-complete row should be untouched.
|
||||
db.add(ImportTask(
|
||||
batch_id=batch.id, source_path="/done", task_type="media",
|
||||
status="complete", finished_at=datetime.now(UTC),
|
||||
))
|
||||
await db.commit()
|
||||
|
||||
resp = await client.post("/api/import/clear-stuck")
|
||||
body = await resp.get_json()
|
||||
assert resp.status_code == 200
|
||||
assert body["tasks_failed"] == 3
|
||||
assert body["batches_finalized"] == 1
|
||||
|
||||
statuses = {
|
||||
row.status for row in
|
||||
(await db.execute(_select(ImportTask).where(ImportTask.batch_id == batch.id)))
|
||||
.scalars().all()
|
||||
}
|
||||
assert statuses == {"failed", "complete"}
|
||||
|
||||
batch_status = (await db.execute(
|
||||
_select(ImportBatch.status).where(ImportBatch.id == batch.id)
|
||||
)).scalar_one()
|
||||
assert batch_status == "complete"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_clear_stuck_no_op_when_nothing_stuck(client, db):
|
||||
resp = await client.post("/api/import/clear-stuck")
|
||||
body = await resp.get_json()
|
||||
assert resp.status_code == 200
|
||||
assert body == {"tasks_failed": 0, "batches_finalized": 0}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_trigger_accepts_deep(client, monkeypatch):
|
||||
# Stub the task dispatch — assert the API accepts 'deep' and forwards
|
||||
|
||||
Reference in New Issue
Block a user