b6a917ac81
Operator hit 3 large PNGs stuck in 'processing' for 2 days 2026-05-25:
the existing recover_interrupted_tasks flips processing > 5min back to
queued + .delay(), but if the underlying file is unfixably broken (e.g.,
PIL OSError, also patched in 68cffce), the loop never terminates and the
'Scanning...' banner sticks at 0/0 forever blocking new scans.
/api/import/clear-stuck:
- Flips every task in pending/queued/processing to 'failed' with a clear
marker error message
- Finalizes any 'running' ImportBatch that has no remaining active children
- Idempotent + non-destructive: rows survive, can be retried once the
underlying cause is resolved
UI button 'Clear stuck...' sits next to 'Retry failed' / 'Clear completed'
with a warning-tonal alert in the confirm dialog explaining what it does
and recommending Retry failed once the cause is fixed.
Tests: clears mixed non-terminal states, untouches complete rows,
finalizes orphan batch, no-op when nothing stuck.
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
195 lines
6.1 KiB
Python
195 lines
6.1 KiB
Python
from datetime import UTC, datetime, timedelta
|
|
|
|
import pytest
|
|
|
|
from backend.app import create_app
|
|
from backend.app.celery_app import celery
|
|
from backend.app.models import ImportBatch, ImportTask
|
|
|
|
pytestmark = pytest.mark.integration
|
|
|
|
|
|
@pytest.fixture(autouse=True)
|
|
def eager():
|
|
celery.conf.task_always_eager = True
|
|
yield
|
|
celery.conf.task_always_eager = False
|
|
|
|
|
|
@pytest.fixture
|
|
async def app():
|
|
return create_app()
|
|
|
|
|
|
@pytest.fixture
|
|
async def client(app):
|
|
async with app.test_client() as c:
|
|
yield c
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_status_when_idle(client):
|
|
resp = await client.get("/api/import/status")
|
|
body = await resp.get_json()
|
|
assert body["active_batch"] is None
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_list_tasks_with_status_filter(client, db):
|
|
batch = ImportBatch(triggered_by="manual", source_path="/import", scan_mode="quick")
|
|
db.add(batch)
|
|
await db.flush()
|
|
for status in ("complete", "failed", "complete"):
|
|
db.add(ImportTask(
|
|
batch_id=batch.id, source_path="/x", task_type="media", status=status,
|
|
finished_at=datetime.now(UTC),
|
|
))
|
|
await db.commit()
|
|
|
|
resp = await client.get("/api/import/tasks?status=failed")
|
|
body = await resp.get_json()
|
|
assert len(body["tasks"]) == 1
|
|
assert body["tasks"][0]["status"] == "failed"
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_retry_failed(client, db):
|
|
batch = ImportBatch(triggered_by="manual", source_path="/import", scan_mode="quick")
|
|
db.add(batch)
|
|
await db.flush()
|
|
db.add(ImportTask(batch_id=batch.id, source_path="/x", task_type="media", status="failed"))
|
|
await db.commit()
|
|
|
|
resp = await client.post("/api/import/retry-failed")
|
|
body = await resp.get_json()
|
|
assert body["retried"] == 1
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_clear_completed(client, db):
|
|
batch = ImportBatch(triggered_by="manual", source_path="/import", scan_mode="quick")
|
|
db.add(batch)
|
|
await db.flush()
|
|
long_ago = datetime.now(UTC) - timedelta(days=30)
|
|
db.add(ImportTask(
|
|
batch_id=batch.id, source_path="/x", task_type="media",
|
|
status="complete", finished_at=long_ago,
|
|
))
|
|
db.add(ImportTask(
|
|
batch_id=batch.id, source_path="/y", task_type="media",
|
|
status="complete", finished_at=datetime.now(UTC),
|
|
))
|
|
await db.commit()
|
|
|
|
resp = await client.post("/api/import/clear-completed", json={"age_days": 7})
|
|
body = await resp.get_json()
|
|
assert body["deleted"] == 1
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_clear_stuck_fails_non_terminal_and_finalizes_orphan_batch(client, db):
|
|
"""Operator-flagged 2026-05-25: 3 large PNGs got stuck in 'processing'
|
|
for 2 days, the active ImportBatch never finalized, and the UI's
|
|
'Scanning...' banner persisted with 0/0 files. /api/import/clear-stuck
|
|
is the escape hatch to break the autoretry loop manually."""
|
|
from sqlalchemy import select as _select
|
|
|
|
batch = ImportBatch(triggered_by="manual", source_path="/import", scan_mode="quick")
|
|
db.add(batch)
|
|
await db.flush()
|
|
# Three stuck rows in mixed non-terminal states.
|
|
db.add(ImportTask(
|
|
batch_id=batch.id, source_path="/p1", task_type="media", status="processing",
|
|
))
|
|
db.add(ImportTask(
|
|
batch_id=batch.id, source_path="/p2", task_type="media", status="queued",
|
|
))
|
|
db.add(ImportTask(
|
|
batch_id=batch.id, source_path="/p3", task_type="media", status="pending",
|
|
))
|
|
# One already-complete row should be untouched.
|
|
db.add(ImportTask(
|
|
batch_id=batch.id, source_path="/done", task_type="media",
|
|
status="complete", finished_at=datetime.now(UTC),
|
|
))
|
|
await db.commit()
|
|
|
|
resp = await client.post("/api/import/clear-stuck")
|
|
body = await resp.get_json()
|
|
assert resp.status_code == 200
|
|
assert body["tasks_failed"] == 3
|
|
assert body["batches_finalized"] == 1
|
|
|
|
statuses = {
|
|
row.status for row in
|
|
(await db.execute(_select(ImportTask).where(ImportTask.batch_id == batch.id)))
|
|
.scalars().all()
|
|
}
|
|
assert statuses == {"failed", "complete"}
|
|
|
|
batch_status = (await db.execute(
|
|
_select(ImportBatch.status).where(ImportBatch.id == batch.id)
|
|
)).scalar_one()
|
|
assert batch_status == "complete"
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_clear_stuck_no_op_when_nothing_stuck(client, db):
|
|
resp = await client.post("/api/import/clear-stuck")
|
|
body = await resp.get_json()
|
|
assert resp.status_code == 200
|
|
assert body == {"tasks_failed": 0, "batches_finalized": 0}
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_trigger_accepts_deep(client, monkeypatch):
|
|
# Stub the task dispatch — assert the API accepts 'deep' and forwards
|
|
# mode, without running the real (eager) scan_directory.
|
|
from backend.app.tasks import scan as scan_mod
|
|
|
|
seen = {}
|
|
|
|
class _Res:
|
|
id = "celery-deep"
|
|
|
|
def _fake_delay(*, triggered_by, mode):
|
|
seen["triggered_by"] = triggered_by
|
|
seen["mode"] = mode
|
|
return _Res()
|
|
|
|
monkeypatch.setattr(scan_mod.scan_directory, "delay", _fake_delay)
|
|
resp = await client.post("/api/import/trigger", json={"mode": "deep"})
|
|
assert resp.status_code == 202
|
|
body = await resp.get_json()
|
|
assert body["mode"] == "deep"
|
|
assert seen == {"triggered_by": "manual", "mode": "deep"}
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_trigger_still_rejects_unknown_mode(client):
|
|
resp = await client.post("/api/import/trigger", json={"mode": "wat"})
|
|
assert resp.status_code == 400
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_trigger_accepts_verify(client, monkeypatch):
|
|
# Stub the verify task's dispatch so the API contract is asserted
|
|
# without running the real (eager) verify_integrity end-to-end.
|
|
from backend.app.tasks import maintenance as m
|
|
|
|
seen = {}
|
|
|
|
class _Res:
|
|
id = "celery-verify"
|
|
|
|
def _fake_delay():
|
|
seen["called"] = True
|
|
return _Res()
|
|
|
|
monkeypatch.setattr(m.verify_integrity, "delay", _fake_delay)
|
|
resp = await client.post("/api/import/trigger", json={"mode": "verify"})
|
|
assert resp.status_code == 202
|
|
body = await resp.get_json()
|
|
assert body["mode"] == "verify"
|
|
assert seen == {"called": True}
|