fix(importer): race-safe savepoint-based find-or-create for Source + Post (uq_source_artist_platform_url UniqueViolation operator-flagged 2026-05-26) — Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -18,6 +18,7 @@ from pathlib import Path
|
||||
|
||||
from PIL import Image
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.exc import IntegrityError
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from ..models import (
|
||||
@@ -202,6 +203,80 @@ class Importer:
|
||||
(phash, width or 0, height or 0, image_id)
|
||||
)
|
||||
|
||||
def _find_or_create_source(
|
||||
self, *, artist_id: int, platform: str, url: str,
|
||||
) -> Source:
|
||||
"""Race-safe find-or-create on `source` keyed by
|
||||
(artist_id, platform, url) — the same key as the
|
||||
`uq_source_artist_platform_url` constraint.
|
||||
|
||||
Two concurrent workers processing different files in the same
|
||||
post can both find no existing Source row then both INSERT,
|
||||
which trips the unique constraint and poisons the session with
|
||||
`psycopg.errors.UniqueViolation`. Operator-flagged 2026-05-26.
|
||||
|
||||
Pattern: select; if absent, open a savepoint and INSERT.
|
||||
On IntegrityError, roll the savepoint back (NOT the outer
|
||||
transaction, which would lose the surrounding scan's progress)
|
||||
and re-select — the concurrent op just created the row we
|
||||
wanted, so the second select will find it.
|
||||
"""
|
||||
existing = self.session.execute(
|
||||
select(Source).where(
|
||||
Source.artist_id == artist_id,
|
||||
Source.platform == platform,
|
||||
Source.url == url,
|
||||
)
|
||||
).scalar_one_or_none()
|
||||
if existing is not None:
|
||||
return existing
|
||||
sp = self.session.begin_nested()
|
||||
try:
|
||||
row = Source(artist_id=artist_id, platform=platform, url=url)
|
||||
self.session.add(row)
|
||||
self.session.flush()
|
||||
sp.commit()
|
||||
return row
|
||||
except IntegrityError:
|
||||
sp.rollback()
|
||||
return self.session.execute(
|
||||
select(Source).where(
|
||||
Source.artist_id == artist_id,
|
||||
Source.platform == platform,
|
||||
Source.url == url,
|
||||
)
|
||||
).scalar_one()
|
||||
|
||||
def _find_or_create_post(
|
||||
self, *, source_id: int, external_post_id: str,
|
||||
) -> Post:
|
||||
"""Race-safe find-or-create on `post` keyed by
|
||||
(source_id, external_post_id). Mirrors `_find_or_create_source`
|
||||
— same savepoint + IntegrityError-recovery pattern."""
|
||||
existing = self.session.execute(
|
||||
select(Post).where(
|
||||
Post.source_id == source_id,
|
||||
Post.external_post_id == external_post_id,
|
||||
)
|
||||
).scalar_one_or_none()
|
||||
if existing is not None:
|
||||
return existing
|
||||
sp = self.session.begin_nested()
|
||||
try:
|
||||
row = Post(source_id=source_id, external_post_id=external_post_id)
|
||||
self.session.add(row)
|
||||
self.session.flush()
|
||||
sp.commit()
|
||||
return row
|
||||
except IntegrityError:
|
||||
sp.rollback()
|
||||
return self.session.execute(
|
||||
select(Post).where(
|
||||
Post.source_id == source_id,
|
||||
Post.external_post_id == external_post_id,
|
||||
)
|
||||
).scalar_one()
|
||||
|
||||
def import_one(self, source: Path) -> ImportResult:
|
||||
"""Dispatch by kind. Media → normal pipeline. Archive → extract
|
||||
media members (one Post via the archive-adjacent sidecar) and
|
||||
@@ -241,29 +316,13 @@ class Importer:
|
||||
sd = parse_sidecar(data)
|
||||
platform = sd.platform or "unknown"
|
||||
url = sd.post_url or f"sidecar:{platform}"
|
||||
src = self.session.execute(
|
||||
select(Source).where(
|
||||
Source.artist_id == artist.id,
|
||||
Source.platform == platform,
|
||||
Source.url == url,
|
||||
)
|
||||
).scalar_one_or_none()
|
||||
if src is None:
|
||||
src = Source(artist_id=artist.id, platform=platform, url=url)
|
||||
self.session.add(src)
|
||||
self.session.flush()
|
||||
src = self._find_or_create_source(
|
||||
artist_id=artist.id, platform=platform, url=url,
|
||||
)
|
||||
epid = sd.external_post_id or sc.stem
|
||||
post = self.session.execute(
|
||||
select(Post).where(
|
||||
Post.source_id == src.id,
|
||||
Post.external_post_id == epid,
|
||||
)
|
||||
).scalar_one_or_none()
|
||||
if post is None:
|
||||
post = Post(source_id=src.id, external_post_id=epid)
|
||||
self.session.add(post)
|
||||
self.session.flush()
|
||||
return post
|
||||
return self._find_or_create_post(
|
||||
source_id=src.id, external_post_id=epid,
|
||||
)
|
||||
|
||||
def _capture_attachment(
|
||||
self, source: Path, *, post: Post | None = None,
|
||||
@@ -705,29 +764,14 @@ class Importer:
|
||||
else:
|
||||
platform = sd.platform or "unknown"
|
||||
url = sd.post_url or f"sidecar:{platform}"
|
||||
src = self.session.execute(
|
||||
select(Source).where(
|
||||
Source.artist_id == artist.id,
|
||||
Source.platform == platform,
|
||||
Source.url == url,
|
||||
)
|
||||
).scalar_one_or_none()
|
||||
if src is None:
|
||||
src = Source(artist_id=artist.id, platform=platform, url=url)
|
||||
self.session.add(src)
|
||||
self.session.flush()
|
||||
src = self._find_or_create_source(
|
||||
artist_id=artist.id, platform=platform, url=url,
|
||||
)
|
||||
|
||||
epid = sd.external_post_id or sc.stem
|
||||
post = self.session.execute(
|
||||
select(Post).where(
|
||||
Post.source_id == src.id,
|
||||
Post.external_post_id == epid,
|
||||
)
|
||||
).scalar_one_or_none()
|
||||
if post is None:
|
||||
post = Post(source_id=src.id, external_post_id=epid)
|
||||
self.session.add(post)
|
||||
self.session.flush()
|
||||
post = self._find_or_create_post(
|
||||
source_id=src.id, external_post_id=epid,
|
||||
)
|
||||
if sd.post_url is not None:
|
||||
post.post_url = sd.post_url
|
||||
if sd.post_title is not None:
|
||||
|
||||
Reference in New Issue
Block a user