fix: the import recovery sweep no longer fails queued files that are waiting their turn
CI and images / lint (push) Successful in 2s
CI and images / extension-version (push) Successful in 2s
CI and images / extension-test (push) Successful in 22s
CI and images / frontend-build (push) Successful in 24s
CI and images / backend-lint-and-test (push) Successful in 41s
CI and images / integration (push) Successful in 2m40s
CI and images / sign-extension (push) Successful in 2s
CI and images / build-agent (push) Successful in 5s
CI and images / build-web (push) Successful in 2m1s
CI and images / smoke-web (push) Successful in 50s
CI and images / promote (push) Successful in 1s
CI and images / refresh-dev (push) Skipped
CI and images / unattended-failure (push) Skipped
CI and images / lint (push) Successful in 2s
CI and images / extension-version (push) Successful in 2s
CI and images / extension-test (push) Successful in 22s
CI and images / frontend-build (push) Successful in 24s
CI and images / backend-lint-and-test (push) Successful in 41s
CI and images / integration (push) Successful in 2m40s
CI and images / sign-extension (push) Successful in 2s
CI and images / build-agent (push) Successful in 5s
CI and images / build-web (push) Successful in 2m1s
CI and images / smoke-web (push) Successful in 50s
CI and images / promote (push) Successful in 1s
CI and images / refresh-dev (push) Skipped
CI and images / unattended-failure (push) Skipped
The orphan sweep in recover_interrupted_tasks failed every 'queued' row older than 30 minutes. A queued row has its broker message, so in a batch bigger than half an hour of work the sweep failed the whole tail: a library import of 8,135 files lost 3,114 that way. Never-enqueued 'pending' rows are still swept after 30 minutes; a 'queued' row is swept only when the broker's import queue is empty, and is kept when the queue can't be read. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -122,7 +122,10 @@ def test_recover_interrupted_sweeps_pending_orphans_to_failed(db_sync, monkeypat
|
||||
Banked 2026-05-25 after operator hit 5490 stuck pending rows.
|
||||
"""
|
||||
from backend.app.tasks import import_file
|
||||
from backend.app.tasks import maintenance as m
|
||||
monkeypatch.setattr(import_file.import_media_file, "delay", lambda *_: None)
|
||||
# The broker's import queue is empty, so a stale 'queued' row's message is gone.
|
||||
monkeypatch.setattr(m, "_import_queue_depth", lambda: 0)
|
||||
|
||||
batch_id = _make_batch(db_sync)
|
||||
now = datetime.now(UTC)
|
||||
@@ -170,6 +173,42 @@ def test_recover_interrupted_sweeps_pending_orphans_to_failed(db_sync, monkeypat
|
||||
assert "orphan" in (stale_pending.error or "")
|
||||
|
||||
|
||||
@pytest.mark.parametrize("depth", [7, None])
|
||||
def test_recover_interrupted_keeps_queued_rows_while_the_import_queue_holds_work(
|
||||
db_sync, monkeypatch, depth,
|
||||
):
|
||||
"""A queued row in a batch bigger than half an hour of work is waiting its
|
||||
turn, not orphaned: while the import queue holds messages (or can't be
|
||||
read), only never-enqueued 'pending' rows are swept."""
|
||||
from sqlalchemy import update as _upd
|
||||
|
||||
from backend.app.tasks import maintenance as m
|
||||
|
||||
monkeypatch.setattr(m, "_import_queue_depth", lambda: depth)
|
||||
batch_id = _make_batch(db_sync)
|
||||
waiting = ImportTask(
|
||||
batch_id=batch_id, source_path="/import/waiting.jpg", task_type="media",
|
||||
status="queued",
|
||||
)
|
||||
never_sent = ImportTask(
|
||||
batch_id=batch_id, source_path="/import/never.jpg", task_type="media",
|
||||
status="pending",
|
||||
)
|
||||
db_sync.add_all([waiting, never_sent])
|
||||
db_sync.flush()
|
||||
db_sync.execute(
|
||||
_upd(ImportTask).where(ImportTask.id.in_([waiting.id, never_sent.id]))
|
||||
.values(created_at=datetime.now(UTC) - timedelta(hours=2))
|
||||
)
|
||||
db_sync.commit()
|
||||
|
||||
assert m.recover_interrupted_tasks.apply().get() == 1
|
||||
db_sync.refresh(waiting)
|
||||
db_sync.refresh(never_sent)
|
||||
assert waiting.status == "queued"
|
||||
assert never_sent.status == "failed"
|
||||
|
||||
|
||||
def test_recover_interrupted_handles_both_stuck_and_orphans(db_sync, monkeypatch):
|
||||
"""One sweep tick handles both 'processing' crashes AND
|
||||
'pending'/'queued' orphans in a single pass."""
|
||||
|
||||
Reference in New Issue
Block a user