"""Race-safe ImageProvenance insert in Importer._apply_sidecar. Operator-flagged 2026-05-26: the prior SELECT-then-INSERT pattern lost a race when two workers ran _apply_sidecar on the same (image, post) pair (plausibly seeded when the 5-min recovery sweep re-enqueued a still-running long import). Duplicates then broke .scalar_one_or_none() on every later deep-scan rederive (MultipleResultsFound). Alembic 0021 added uq_image_provenance_image_post; the importer's new savepoint+IntegrityError recovery path now trips on collision and gracefully recovers. Tests cover: - idempotent: re-running _apply_sidecar via _deep_rederive produces exactly one ImageProvenance row. - IntegrityError recovery: pre-seed a provenance row, force the first SELECT to return None (simulating the race window where two workers both observed no row), call _apply_sidecar — the savepoint INSERT trips uq_image_provenance_image_post, gets rolled back, no exception escapes, still exactly one row. """ import json from pathlib import Path import pytest from PIL import Image from sqlalchemy import func, select from backend.app.models import ( ImageProvenance, ImageRecord, ImportSettings, ) from backend.app.services.importer import Importer from backend.app.services.thumbnailer import Thumbnailer pytestmark = pytest.mark.integration @pytest.fixture def import_layout(tmp_path): import_root = tmp_path / "import" images_root = tmp_path / "images" import_root.mkdir() images_root.mkdir() return import_root, images_root @pytest.fixture def importer(db_sync, import_layout): import_root, images_root = import_layout settings = db_sync.execute( select(ImportSettings).where(ImportSettings.id == 1) ).scalar_one() return Importer( session=db_sync, images_root=images_root, import_root=import_root, thumbnailer=Thumbnailer(images_root=images_root), settings=settings, ) @pytest.fixture def deep_importer(db_sync, import_layout): import_root, images_root = import_layout settings = db_sync.execute( select(ImportSettings).where(ImportSettings.id == 1) ).scalar_one() return Importer( session=db_sync, images_root=images_root, import_root=import_root, thumbnailer=Thumbnailer(images_root=images_root), settings=settings, deep=True, ) def _split(path: Path, orient, size=(256, 256)): path.parent.mkdir(parents=True, exist_ok=True) w, h = size im = Image.new("L", size, 0) px = im.load() for y in range(h): for x in range(w): if (x / w if orient == "v" else y / h) >= 0.5: px[x, y] = 255 im.convert("RGB").save(path, "JPEG") def _sidecar(media: Path, payload: dict): media.with_suffix(".json").write_text(json.dumps(payload)) def test_apply_sidecar_idempotent_on_deep_rederive( importer, deep_importer, import_layout, ): """Normal-flow path: deep rederive on an already-imported image finds the existing provenance row via .scalar_one_or_none() and skips the insert. Exactly one ImageProvenance row after both runs.""" import_root, _ = import_layout m = import_root / "Alice" / "a.jpg" _split(m, "v") _sidecar(m, { "category": "patreon", "id": 555, "url": "https://patreon.com/posts/555", "title": "Set 1", }) r = importer.import_one(m) assert r.status == "imported" # Re-import via deep mode — sha matches → _deep_rederive → _apply_sidecar. r2 = deep_importer.import_one(m) assert r2.status == "refreshed" count = importer.session.execute( select(func.count()).select_from(ImageProvenance) ).scalar_one() assert count == 1 def test_apply_sidecar_recovers_from_integrity_error( importer, deep_importer, import_layout, db_sync, monkeypatch, ): """Race recovery: a row already exists for (image, post). We force the importer's existence-check SELECT to return None for one call, mimicking the race window where two workers both saw no row. The savepoint INSERT then trips uq_image_provenance_image_post; the helper rolls the savepoint back, no exception escapes, and the row count stays at 1. """ import_root, _ = import_layout m = import_root / "Bob" / "b.jpg" _split(m, "v") _sidecar(m, { "category": "patreon", "id": 777, "url": "https://patreon.com/posts/777", "title": "Set 2", }) # First import lays the canonical provenance row. r = importer.import_one(m) assert r.status == "imported" rec = importer.session.get(ImageRecord, r.image_id) # alembic 0030 stopped creating synthetic Source rows for filesystem # sidecars; the Post sits null-source and the provenance row points at # it directly. The race-recovery path tested below operates on # ImageProvenance regardless of Source presence. assert rec is not None # Monkeypatch session.execute so the FIRST select inside _apply_sidecar's # existence-check returns a "no row" wrapper. Subsequent selects (e.g. # the find_or_create_source / find_or_create_post existence checks # earlier in _apply_sidecar) all run normally; we intercept only the # ImageProvenance lookup, identified by the SELECT's target columns # mentioning image_provenance. real_execute = db_sync.execute intercepted = [False] def _intercepting_execute(stmt, *args, **kwargs): text = str(stmt) if ( not intercepted[0] and "image_provenance" in text and "image_record_id" in text and "post_id" in text ): intercepted[0] = True class _ForcedMiss: def scalar_one_or_none(self): return None return _ForcedMiss() return real_execute(stmt, *args, **kwargs) monkeypatch.setattr(db_sync, "execute", _intercepting_execute) # Re-import via deep mode → _deep_rederive → _apply_sidecar. With the # provenance-SELECT forced to miss, the helper will attempt the INSERT, # trip uq_image_provenance_image_post, catch IntegrityError, and recover. r2 = deep_importer.import_one(m) assert r2.status == "refreshed" # Lift the intercept, then verify the row count. monkeypatch.undo() count = db_sync.execute( select(func.count()).select_from(ImageProvenance) ).scalar_one() assert count == 1