FabledCurator/tests/test_importer_provenance_race.py

"""Race-safe ImageProvenance insert in Importer._apply_sidecar.

Operator-flagged 2026-05-26: the prior SELECT-then-INSERT pattern lost a
race when two workers ran _apply_sidecar on the same (image, post) pair
(plausibly seeded when the 5-min recovery sweep re-enqueued a still-running
long import). Duplicates then broke .scalar_one_or_none() on every later
deep-scan rederive (MultipleResultsFound). Alembic 0021 added
uq_image_provenance_image_post; the importer's new savepoint+IntegrityError
recovery path now trips on collision and gracefully recovers.

Tests cover:
  - idempotent: re-running _apply_sidecar via _deep_rederive produces
    exactly one ImageProvenance row.
  - IntegrityError recovery: pre-seed a provenance row, force the first
    SELECT to return None (simulating the race window where two workers
    both observed no row), call _apply_sidecar — the savepoint INSERT
    trips uq_image_provenance_image_post, gets rolled back, no exception
    escapes, still exactly one row.
"""

import json
from pathlib import Path

import pytest
from PIL import Image
from sqlalchemy import func, select

from backend.app.models import (
    ImageProvenance,
    ImageRecord,
    ImportSettings,
)
from backend.app.services.importer import Importer
from backend.app.services.thumbnailer import Thumbnailer

pytestmark = pytest.mark.integration


@pytest.fixture
def import_layout(tmp_path):
    import_root = tmp_path / "import"
    images_root = tmp_path / "images"
    import_root.mkdir()
    images_root.mkdir()
    return import_root, images_root


@pytest.fixture
def importer(db_sync, import_layout):
    import_root, images_root = import_layout
    settings = db_sync.execute(
        select(ImportSettings).where(ImportSettings.id == 1)
    ).scalar_one()
    return Importer(
        session=db_sync,
        images_root=images_root,
        import_root=import_root,
        thumbnailer=Thumbnailer(images_root=images_root),
        settings=settings,
    )


@pytest.fixture
def deep_importer(db_sync, import_layout):
    import_root, images_root = import_layout
    settings = db_sync.execute(
        select(ImportSettings).where(ImportSettings.id == 1)
    ).scalar_one()
    return Importer(
        session=db_sync,
        images_root=images_root,
        import_root=import_root,
        thumbnailer=Thumbnailer(images_root=images_root),
        settings=settings,
        deep=True,
    )


def _split(path: Path, orient, size=(256, 256)):
    path.parent.mkdir(parents=True, exist_ok=True)
    w, h = size
    im = Image.new("L", size, 0)
    px = im.load()
    for y in range(h):
        for x in range(w):
            if (x / w if orient == "v" else y / h) >= 0.5:
                px[x, y] = 255
    im.convert("RGB").save(path, "JPEG")


def _sidecar(media: Path, payload: dict):
    media.with_suffix(".json").write_text(json.dumps(payload))


def test_apply_sidecar_idempotent_on_deep_rederive(
    importer, deep_importer, import_layout,
):
    """Normal-flow path: deep rederive on an already-imported image finds
    the existing provenance row via .scalar_one_or_none() and skips the
    insert. Exactly one ImageProvenance row after both runs."""
    import_root, _ = import_layout
    m = import_root / "Alice" / "a.jpg"
    _split(m, "v")
    _sidecar(m, {
        "category": "patreon", "id": 555,
        "url": "https://patreon.com/posts/555", "title": "Set 1",
    })
    r = importer.import_one(m)
    assert r.status == "imported"

    # Re-import via deep mode — sha matches → _deep_rederive → _apply_sidecar.
    r2 = deep_importer.import_one(m)
    assert r2.status == "refreshed"

    count = importer.session.execute(
        select(func.count()).select_from(ImageProvenance)
    ).scalar_one()
    assert count == 1


def test_apply_sidecar_recovers_from_integrity_error(
    importer, deep_importer, import_layout, db_sync, monkeypatch,
):
    """Race recovery: a row already exists for (image, post). We force the
    importer's existence-check SELECT to return None for one call, mimicking
    the race window where two workers both saw no row. The savepoint INSERT
    then trips uq_image_provenance_image_post; the helper rolls the
    savepoint back, no exception escapes, and the row count stays at 1.
    """
    import_root, _ = import_layout
    m = import_root / "Bob" / "b.jpg"
    _split(m, "v")
    _sidecar(m, {
        "category": "patreon", "id": 777,
        "url": "https://patreon.com/posts/777", "title": "Set 2",
    })
    # First import lays the canonical provenance row.
    r = importer.import_one(m)
    assert r.status == "imported"
    rec = importer.session.get(ImageRecord, r.image_id)
    # alembic 0030 stopped creating synthetic Source rows for filesystem
    # sidecars; the Post sits null-source and the provenance row points at
    # it directly. The race-recovery path tested below operates on
    # ImageProvenance regardless of Source presence.
    assert rec is not None

    # Monkeypatch session.execute so the FIRST select inside _apply_sidecar's
    # existence-check returns a "no row" wrapper. Subsequent selects (e.g.
    # the find_or_create_source / find_or_create_post existence checks
    # earlier in _apply_sidecar) all run normally; we intercept only the
    # ImageProvenance lookup, identified by the SELECT's target columns
    # mentioning image_provenance.
    real_execute = db_sync.execute
    intercepted = [False]

    def _intercepting_execute(stmt, *args, **kwargs):
        text = str(stmt)
        if (
            not intercepted[0]
            and "image_provenance" in text
            and "image_record_id" in text
            and "post_id" in text
        ):
            intercepted[0] = True

            class _ForcedMiss:
                def scalar_one_or_none(self):
                    return None
            return _ForcedMiss()
        return real_execute(stmt, *args, **kwargs)

    monkeypatch.setattr(db_sync, "execute", _intercepting_execute)

    # Re-import via deep mode → _deep_rederive → _apply_sidecar. With the
    # provenance-SELECT forced to miss, the helper will attempt the INSERT,
    # trip uq_image_provenance_image_post, catch IntegrityError, and recover.
    r2 = deep_importer.import_one(m)
    assert r2.status == "refreshed"

    # Lift the intercept, then verify the row count.
    monkeypatch.undo()
    count = db_sync.execute(
        select(func.count()).select_from(ImageProvenance)
    ).scalar_one()
    assert count == 1