fix(importer): ImageProvenance (image_record_id, post_id) race-safe via savepoint + alembic 0021 UNIQUE — closes the SELECT-then-INSERT window that planted duplicates and broke .scalar_one_or_none() on every later deep-scan rederive (MultipleResultsFound). Migration dedupes existing rows (min(id) per pair); model gains __table_args__; gallery-filter test that seeded duplicates dropped.
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -69,20 +69,13 @@ async def test_scroll_post_id_filter(db):
|
||||
assert {x.id for x in page.images} == {i1.id, i2.id}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_scroll_post_id_dedups_multi_rows(db):
|
||||
i1 = await _img(db, 1)
|
||||
_, s, p = await _post(db, "A", "a", "10")
|
||||
# two provenance rows, same image+post (enrich-on-duplicate shape)
|
||||
db.add(ImageProvenance(image_record_id=i1.id, post_id=p.id,
|
||||
source_id=s.id))
|
||||
await db.flush()
|
||||
db.add(ImageProvenance(image_record_id=i1.id, post_id=p.id,
|
||||
source_id=s.id))
|
||||
await db.flush()
|
||||
svc = GalleryService(db)
|
||||
page = await svc.scroll(cursor=None, limit=10, post_id=p.id)
|
||||
assert [x.id for x in page.images] == [i1.id] # appears once
|
||||
# test_scroll_post_id_dedups_multi_rows removed 2026-05-26: it deliberately
|
||||
# inserted two ImageProvenance rows with the same (image_record_id, post_id),
|
||||
# now prevented at the DB layer by uq_image_provenance_image_post (alembic
|
||||
# 0021). The EXISTS-based dedup in _provenance_clause is still useful for the
|
||||
# artist-id filter (one image legitimately joins many provenance rows via
|
||||
# different posts under the same artist), so the gallery_service logic is
|
||||
# unchanged.
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
||||
@@ -0,0 +1,186 @@
|
||||
"""Race-safe ImageProvenance insert in Importer._apply_sidecar.
|
||||
|
||||
Operator-flagged 2026-05-26: the prior SELECT-then-INSERT pattern lost a
|
||||
race when two workers ran _apply_sidecar on the same (image, post) pair
|
||||
(plausibly seeded when the 5-min recovery sweep re-enqueued a still-running
|
||||
long import). Duplicates then broke .scalar_one_or_none() on every later
|
||||
deep-scan rederive (MultipleResultsFound). Alembic 0021 added
|
||||
uq_image_provenance_image_post; the importer's new savepoint+IntegrityError
|
||||
recovery path now trips on collision and gracefully recovers.
|
||||
|
||||
Tests cover:
|
||||
- idempotent: re-running _apply_sidecar via _deep_rederive produces
|
||||
exactly one ImageProvenance row.
|
||||
- IntegrityError recovery: pre-seed a provenance row, force the first
|
||||
SELECT to return None (simulating the race window where two workers
|
||||
both observed no row), call _apply_sidecar — the savepoint INSERT
|
||||
trips uq_image_provenance_image_post, gets rolled back, no exception
|
||||
escapes, still exactly one row.
|
||||
"""
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
from PIL import Image
|
||||
from sqlalchemy import func, select
|
||||
|
||||
from backend.app.models import (
|
||||
ImageProvenance,
|
||||
ImageRecord,
|
||||
ImportSettings,
|
||||
Post,
|
||||
Source,
|
||||
)
|
||||
from backend.app.services.importer import Importer
|
||||
from backend.app.services.thumbnailer import Thumbnailer
|
||||
|
||||
pytestmark = pytest.mark.integration
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def import_layout(tmp_path):
|
||||
import_root = tmp_path / "import"
|
||||
images_root = tmp_path / "images"
|
||||
import_root.mkdir()
|
||||
images_root.mkdir()
|
||||
return import_root, images_root
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def importer(db_sync, import_layout):
|
||||
import_root, images_root = import_layout
|
||||
settings = db_sync.execute(
|
||||
select(ImportSettings).where(ImportSettings.id == 1)
|
||||
).scalar_one()
|
||||
return Importer(
|
||||
session=db_sync,
|
||||
images_root=images_root,
|
||||
import_root=import_root,
|
||||
thumbnailer=Thumbnailer(images_root=images_root),
|
||||
settings=settings,
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def deep_importer(db_sync, import_layout):
|
||||
import_root, images_root = import_layout
|
||||
settings = db_sync.execute(
|
||||
select(ImportSettings).where(ImportSettings.id == 1)
|
||||
).scalar_one()
|
||||
return Importer(
|
||||
session=db_sync,
|
||||
images_root=images_root,
|
||||
import_root=import_root,
|
||||
thumbnailer=Thumbnailer(images_root=images_root),
|
||||
settings=settings,
|
||||
deep=True,
|
||||
)
|
||||
|
||||
|
||||
def _split(path: Path, orient, size=(256, 256)):
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
w, h = size
|
||||
im = Image.new("L", size, 0)
|
||||
px = im.load()
|
||||
for y in range(h):
|
||||
for x in range(w):
|
||||
if (x / w if orient == "v" else y / h) >= 0.5:
|
||||
px[x, y] = 255
|
||||
im.convert("RGB").save(path, "JPEG")
|
||||
|
||||
|
||||
def _sidecar(media: Path, payload: dict):
|
||||
media.with_suffix(".json").write_text(json.dumps(payload))
|
||||
|
||||
|
||||
def test_apply_sidecar_idempotent_on_deep_rederive(
|
||||
importer, deep_importer, import_layout,
|
||||
):
|
||||
"""Normal-flow path: deep rederive on an already-imported image finds
|
||||
the existing provenance row via .scalar_one_or_none() and skips the
|
||||
insert. Exactly one ImageProvenance row after both runs."""
|
||||
import_root, _ = import_layout
|
||||
m = import_root / "Alice" / "a.jpg"
|
||||
_split(m, "v")
|
||||
_sidecar(m, {
|
||||
"category": "patreon", "id": 555,
|
||||
"url": "https://patreon.com/posts/555", "title": "Set 1",
|
||||
})
|
||||
r = importer.import_one(m)
|
||||
assert r.status == "imported"
|
||||
|
||||
# Re-import via deep mode — sha matches → _deep_rederive → _apply_sidecar.
|
||||
r2 = deep_importer.import_one(m)
|
||||
assert r2.status == "refreshed"
|
||||
|
||||
count = importer.session.execute(
|
||||
select(func.count()).select_from(ImageProvenance)
|
||||
).scalar_one()
|
||||
assert count == 1
|
||||
|
||||
|
||||
def test_apply_sidecar_recovers_from_integrity_error(
|
||||
importer, deep_importer, import_layout, db_sync, monkeypatch,
|
||||
):
|
||||
"""Race recovery: a row already exists for (image, post). We force the
|
||||
importer's existence-check SELECT to return None for one call, mimicking
|
||||
the race window where two workers both saw no row. The savepoint INSERT
|
||||
then trips uq_image_provenance_image_post; the helper rolls the
|
||||
savepoint back, no exception escapes, and the row count stays at 1.
|
||||
"""
|
||||
import_root, _ = import_layout
|
||||
m = import_root / "Bob" / "b.jpg"
|
||||
_split(m, "v")
|
||||
_sidecar(m, {
|
||||
"category": "patreon", "id": 777,
|
||||
"url": "https://patreon.com/posts/777", "title": "Set 2",
|
||||
})
|
||||
# First import lays the canonical provenance row.
|
||||
r = importer.import_one(m)
|
||||
assert r.status == "imported"
|
||||
rec = importer.session.get(ImageRecord, r.image_id)
|
||||
post = importer.session.execute(select(Post)).scalar_one()
|
||||
src = importer.session.execute(select(Source)).scalar_one()
|
||||
assert rec is not None
|
||||
assert src is not None
|
||||
|
||||
# Monkeypatch session.execute so the FIRST select inside _apply_sidecar's
|
||||
# existence-check returns a "no row" wrapper. Subsequent selects (e.g.
|
||||
# the find_or_create_source / find_or_create_post existence checks
|
||||
# earlier in _apply_sidecar) all run normally; we intercept only the
|
||||
# ImageProvenance lookup, identified by the SELECT's target columns
|
||||
# mentioning image_provenance.
|
||||
real_execute = db_sync.execute
|
||||
intercepted = [False]
|
||||
|
||||
def _intercepting_execute(stmt, *args, **kwargs):
|
||||
text = str(stmt)
|
||||
if (
|
||||
not intercepted[0]
|
||||
and "image_provenance" in text
|
||||
and "image_record_id" in text
|
||||
and "post_id" in text
|
||||
):
|
||||
intercepted[0] = True
|
||||
|
||||
class _ForcedMiss:
|
||||
def scalar_one_or_none(self):
|
||||
return None
|
||||
return _ForcedMiss()
|
||||
return real_execute(stmt, *args, **kwargs)
|
||||
|
||||
monkeypatch.setattr(db_sync, "execute", _intercepting_execute)
|
||||
|
||||
# Re-import via deep mode → _deep_rederive → _apply_sidecar. With the
|
||||
# provenance-SELECT forced to miss, the helper will attempt the INSERT,
|
||||
# trip uq_image_provenance_image_post, catch IntegrityError, and recover.
|
||||
r2 = deep_importer.import_one(m)
|
||||
assert r2.status == "refreshed"
|
||||
|
||||
# Lift the intercept, then verify the row count.
|
||||
monkeypatch.undo()
|
||||
count = db_sync.execute(
|
||||
select(func.count()).select_from(ImageProvenance)
|
||||
).scalar_one()
|
||||
assert count == 1
|
||||
Reference in New Issue
Block a user