fix(audit): chunk + self-resume library scans (stop the 2h queue-hog timeouts)
scan_library_for_rule ran one 2-hour pass that timed out on large libraries and held the concurrency-1 maintenance queue the whole time, starving vacuum/backup/ normalize (operator-flagged — it was the dominant entry in the 24h failures). It now runs ~10-min chunks and re-enqueues itself until the library is exhausted, matching the operator's preferred pattern (reasonable timeout → retry queued → other things process between). New columns (alembic 0039): resume_after_id persists the keyset cursor so a chunk continues where the last left off; last_progress_at lets the recovery sweep tell a progressing multi- chunk audit from a dead one (it now measures staleness from last_progress_at, not started_at). Matches accumulate across chunks. soft/hard limits dropped 2h→15/16.7 min so the in-chunk budget fires first; a soft-limit backstop re-enqueues to resume instead of erroring the whole run. Tests: time-box → re-enqueue (status stays running); resume carries prior matches and appends new ones. Existing full-scan tests unchanged (small sets finish in one chunk). Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -69,6 +69,60 @@ def test_scan_library_for_rule_populates_matched_ids_for_transparency(
|
||||
assert status == "ready"
|
||||
|
||||
|
||||
def test_scan_time_boxes_and_reenqueues(db_sync, tmp_path, monkeypatch):
|
||||
"""A zero chunk budget stops before scanning and re-enqueues to continue,
|
||||
leaving the audit 'running' — so a huge library can't run the task into the
|
||||
Celery limit or hog the maintenance queue (operator-flagged 2026-06-07)."""
|
||||
_mk_image(db_sync, tmp_path, mode="RGBA", color=(0, 0, 0, 0), name="a.png")
|
||||
audit = LibraryAuditRun(
|
||||
rule="transparency", params={"threshold": 0.5},
|
||||
status="running", matched_ids=[],
|
||||
)
|
||||
db_sync.add(audit)
|
||||
db_sync.commit()
|
||||
audit_id = audit.id
|
||||
|
||||
from backend.app.tasks import library_audit as la
|
||||
delays = []
|
||||
monkeypatch.setattr(la.scan_library_for_rule, "delay", lambda aid: delays.append(aid))
|
||||
monkeypatch.setattr(la, "_CHUNK_SECONDS", 0) # time-box on the first iteration
|
||||
la.scan_library_for_rule.run(audit_id)
|
||||
|
||||
status = db_sync.execute(
|
||||
select(LibraryAuditRun.status).where(LibraryAuditRun.id == audit_id)
|
||||
).scalar_one()
|
||||
assert status == "running" # not finished — handed off to the next chunk
|
||||
assert delays == [audit_id] # re-enqueued itself
|
||||
|
||||
|
||||
def test_scan_resumes_and_accumulates_matched(db_sync, tmp_path, monkeypatch):
|
||||
"""A later chunk keeps the prior chunks' matches and appends its own."""
|
||||
trans, _ = _mk_image(
|
||||
db_sync, tmp_path, mode="RGBA", color=(0, 0, 0, 0), name="t.png",
|
||||
)
|
||||
audit = LibraryAuditRun(
|
||||
rule="transparency", params={"threshold": 0.5},
|
||||
status="running", matched_ids=[424242], resume_after_id=0,
|
||||
)
|
||||
db_sync.add(audit)
|
||||
db_sync.commit()
|
||||
audit_id = audit.id
|
||||
|
||||
from backend.app.tasks import library_audit as la
|
||||
monkeypatch.setattr(la.scan_library_for_rule, "delay", lambda aid: None)
|
||||
la.scan_library_for_rule.run(audit_id)
|
||||
|
||||
matched = db_sync.execute(
|
||||
select(LibraryAuditRun.matched_ids).where(LibraryAuditRun.id == audit_id)
|
||||
).scalar_one()
|
||||
status = db_sync.execute(
|
||||
select(LibraryAuditRun.status).where(LibraryAuditRun.id == audit_id)
|
||||
).scalar_one()
|
||||
assert status == "ready"
|
||||
assert 424242 in matched # carried over from a prior chunk
|
||||
assert trans.id in matched # found this chunk
|
||||
|
||||
|
||||
def test_scan_library_for_rule_skips_missing_files_gracefully(
|
||||
db_sync, tmp_path, monkeypatch,
|
||||
):
|
||||
|
||||
Reference in New Issue
Block a user