feat(sidecar): importer creates Source/Post/ImageProvenance from sidecars (idempotent, fail-soft)

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

backend/app/services/importer.py

@@ -9,6 +9,8 @@ by the task. The Quart side uses an async session via the same models.
 """
 
 import hashlib
+import json
+import logging
 import shutil
 from dataclasses import dataclass
 from enum import StrEnum
@@ -18,13 +20,25 @@ from PIL import Image
 from sqlalchemy import select
 from sqlalchemy.orm import Session
 
-from ..models import Artist, ImageRecord, ImportSettings, Tag, TagKind
+from ..models import (
+    Artist,
+    ImageProvenance,
+    ImageRecord,
+    ImportSettings,
+    Post,
+    Source,
+    Tag,
+    TagKind,
+)
 from ..models.tag import image_tag
 from ..utils.paths import derive_subdir, derive_top_level_artist, hash_suffixed_name
 from ..utils.phash import compute_phash, find_similar
+from ..utils.sidecar import find_sidecar, parse_sidecar
 from ..utils.slug import slugify
 from .thumbnailer import Thumbnailer
 
+log = logging.getLogger(__name__)
+
 
 class SkipReason(StrEnum):
     too_small = "too_small"
@@ -227,9 +241,13 @@ class Importer:
         self.session.flush()
 
         # Folder→artist auto-tag.
+        artist = None
         artist_name = derive_top_level_artist(source, self.import_root)
         if artist_name:
-            self._attach_artist_tag(record.id, artist_name)
+            artist = self._attach_artist_tag(record.id, artist_name)
+
+        # Sidecar provenance (best-effort; never fails the import).
+        self._apply_sidecar(record, source, artist)
 
         # Thumbnail is queued separately by the calling task; the importer
         # does not generate thumbnails inline so the import queue stays moving.
@@ -237,16 +255,21 @@ class Importer:
         self.session.commit()
         return ImportResult(status="imported", image_id=record.id)
 
-    def _attach_artist_tag(self, image_id: int, artist_name: str) -> None:
+    def _upsert_artist(self, name: str) -> Artist:
-        """Idempotently creates Artist + artist tag, attaches to image."""
+        slug = slugify(name)
-        slug = slugify(artist_name)
         artist = self.session.execute(
             select(Artist).where(Artist.slug == slug)
         ).scalar_one_or_none()
         if artist is None:
-            artist = Artist(name=artist_name, slug=slug, is_subscription=False)
+            artist = Artist(name=name, slug=slug, is_subscription=False)
             self.session.add(artist)
             self.session.flush()
+        return artist
+
+    def _attach_artist_tag(self, image_id: int, artist_name: str) -> Artist:
+        """Idempotently creates Artist + artist tag, attaches to image.
+        Returns the Artist (sidecar provenance attaches its Source to it)."""
+        artist = self._upsert_artist(artist_name)
 
         tag = self.session.execute(
             select(Tag).where(Tag.name == artist_name).where(Tag.kind == TagKind.artist)
@@ -270,6 +293,107 @@ class Importer:
                     image_record_id=image_id, tag_id=tag.id, source="auto",
                 )
             )
+        return artist
+
+    @staticmethod
+    def _sidecar_artist_name(data: dict) -> str | None:
+        for k in ("artist", "author_name"):
+            v = data.get(k)
+            if isinstance(v, str) and v.strip():
+                return v.strip()
+        for k in ("user", "creator"):
+            v = data.get(k)
+            if isinstance(v, str) and v.strip():
+                return v.strip()
+            if isinstance(v, dict):
+                for sub in ("name", "full_name", "vanity", "account"):
+                    sv = v.get(sub)
+                    if isinstance(sv, str) and sv.strip():
+                        return sv.strip()
+        return None
+
+    def _apply_sidecar(
+        self, record: ImageRecord, source: Path, artist: Artist | None
+    ) -> None:
+        sc = find_sidecar(source)
+        if sc is None:
+            return
+        try:
+            data = json.loads(sc.read_text("utf-8"))
+            if not isinstance(data, dict):
+                raise ValueError("sidecar JSON is not an object")
+        except Exception as exc:
+            log.warning("sidecar parse failed for %s: %s", sc, exc)
+            return
+
+        sd = parse_sidecar(data)
+
+        if artist is None:
+            name = self._sidecar_artist_name(data)
+            if name:
+                artist = self._upsert_artist(name)
+        if artist is None:
+            log.warning(
+                "sidecar present for %s but no artist; skipping provenance",
+                source,
+            )
+            return
+
+        platform = sd.platform or "unknown"
+        url = sd.post_url or f"sidecar:{platform}"
+        src = self.session.execute(
+            select(Source).where(
+                Source.artist_id == artist.id,
+                Source.platform == platform,
+                Source.url == url,
+            )
+        ).scalar_one_or_none()
+        if src is None:
+            src = Source(artist_id=artist.id, platform=platform, url=url)
+            self.session.add(src)
+            self.session.flush()
+
+        epid = sd.external_post_id or sc.stem
+        post = self.session.execute(
+            select(Post).where(
+                Post.source_id == src.id,
+                Post.external_post_id == epid,
+            )
+        ).scalar_one_or_none()
+        if post is None:
+            post = Post(source_id=src.id, external_post_id=epid)
+            self.session.add(post)
+            self.session.flush()
+        if sd.post_url is not None:
+            post.post_url = sd.post_url
+        if sd.post_title is not None:
+            post.post_title = sd.post_title
+        if sd.post_date is not None:
+            post.post_date = sd.post_date
+        if sd.description is not None:
+            post.description = sd.description
+        if sd.attachment_count is not None:
+            post.attachment_count = sd.attachment_count
+        post.raw_metadata = sd.raw
+
+        exists = self.session.execute(
+            select(ImageProvenance.id).where(
+                ImageProvenance.image_record_id == record.id,
+                ImageProvenance.post_id == post.id,
+            )
+        ).scalar_one_or_none()
+        if exists is None:
+            self.session.add(
+                ImageProvenance(
+                    image_record_id=record.id,
+                    post_id=post.id,
+                    source_id=src.id,
+                    captured_metadata=sd.raw,
+                )
+            )
+        if record.primary_post_id is None:
+            record.primary_post_id = post.id
+        self.session.flush()
 
     def _supersede(
         self, existing: ImageRecord, source: Path, sha: str,

tests/test_sidecar_import.py

@@ -0,0 +1,164 @@
+"""FC-2d-v: sidecar → Source/Post/ImageProvenance (sync importer)."""
+
+import json
+from pathlib import Path
+
+import pytest
+from PIL import Image
+from sqlalchemy import func, select
+
+from backend.app.models import (
+    ImageProvenance,
+    ImageRecord,
+    ImportSettings,
+    Post,
+    Source,
+)
+from backend.app.services.importer import Importer
+from backend.app.services.thumbnailer import Thumbnailer
+
+pytestmark = pytest.mark.integration
+
+
+@pytest.fixture
+def import_layout(tmp_path):
+    import_root = tmp_path / "import"
+    images_root = tmp_path / "images"
+    import_root.mkdir()
+    images_root.mkdir()
+    return import_root, images_root
+
+
+@pytest.fixture
+def importer(db_sync, import_layout):
+    import_root, images_root = import_layout
+    settings = db_sync.execute(
+        select(ImportSettings).where(ImportSettings.id == 1)
+    ).scalar_one()
+    return Importer(
+        session=db_sync,
+        images_root=images_root,
+        import_root=import_root,
+        thumbnailer=Thumbnailer(images_root=images_root),
+        settings=settings,
+    )
+
+
+def _split(path: Path, orient, size=(256, 256)):
+    """Structured image (half/half) — solid colors phash-collapse, which
+    would route a 2nd image to the supersede path (no sidecar applied)."""
+    path.parent.mkdir(parents=True, exist_ok=True)
+    w, h = size
+    im = Image.new("L", size, 0)
+    px = im.load()
+    for y in range(h):
+        for x in range(w):
+            if (x / w if orient == "v" else y / h) >= 0.5:
+                px[x, y] = 255
+    im.convert("RGB").save(path, "JPEG")
+
+
+def _sidecar(media: Path, payload: dict):
+    media.with_suffix(".json").write_text(json.dumps(payload))
+
+
+def test_sidecar_creates_provenance(importer, import_layout):
+    import_root, _ = import_layout
+    m = import_root / "Alice" / "a.jpg"
+    _split(m, "v")
+    _sidecar(m, {
+        "category": "patreon", "id": 555,
+        "url": "https://patreon.com/posts/555", "title": "Set 1",
+        "content": "<p>hi</p>", "page_count": 2,
+        "published_at": "2023-08-01T00:00:00Z",
+    })
+    r = importer.import_one(m)
+    assert r.status == "imported"
+    rec = importer.session.get(ImageRecord, r.image_id)
+    src = importer.session.execute(select(Source)).scalar_one()
+    post = importer.session.execute(select(Post)).scalar_one()
+    assert src.platform == "patreon"
+    assert post.external_post_id == "555"
+    assert post.post_url == "https://patreon.com/posts/555"
+    assert post.post_title == "Set 1"
+    assert post.description == "<p>hi</p>"
+    assert post.attachment_count == 2
+    assert post.post_date is not None
+    assert post.raw_metadata["id"] == 555
+    prov = importer.session.execute(select(ImageProvenance)).scalar_one()
+    assert prov.image_record_id == rec.id and prov.post_id == post.id
+    assert rec.primary_post_id == post.id
+
+
+def test_reimport_same_post_idempotent(importer, import_layout):
+    import_root, _ = import_layout
+    payload = {"category": "patreon", "id": 777, "title": "P"}
+    m1 = import_root / "Bob" / "p1.jpg"
+    _split(m1, "v")
+    _sidecar(m1, payload)
+    importer.import_one(m1)
+    m2 = import_root / "Bob" / "p2.jpg"
+    _split(m2, "h")            # structurally distinct → not a phash dup
+    _sidecar(m2, payload)
+    r2 = importer.import_one(m2)
+    assert r2.status == "imported"
+    assert importer.session.execute(
+        select(func.count()).select_from(Source)
+    ).scalar_one() == 1
+    assert importer.session.execute(
+        select(func.count()).select_from(Post)
+    ).scalar_one() == 1
+    assert importer.session.execute(
+        select(func.count()).select_from(ImageProvenance)
+    ).scalar_one() == 2
+
+
+def test_garbage_sidecar_still_imports(importer, import_layout):
+    import_root, _ = import_layout
+    m = import_root / "Carol" / "c.jpg"
+    _split(m, "v")
+    m.with_suffix(".json").write_text("{ not json")
+    r = importer.import_one(m)
+    assert r.status == "imported"
+    assert importer.session.execute(
+        select(func.count()).select_from(Post)
+    ).scalar_one() == 0
+
+
+def test_no_sidecar_unchanged(importer, import_layout):
+    import_root, _ = import_layout
+    m = import_root / "Dave" / "d.jpg"
+    _split(m, "v")
+    r = importer.import_one(m)
+    assert r.status == "imported"
+    assert importer.session.execute(
+        select(func.count()).select_from(Post)
+    ).scalar_one() == 0
+
+
+def test_no_artist_anywhere_skips_provenance(importer, import_layout):
+    import_root, _ = import_layout
+    m = import_root / "rootfile.jpg"          # no top-level artist folder
+    _split(m, "v")
+    _sidecar(m, {"category": "x", "id": 1})   # no artist key
+    r = importer.import_one(m)
+    assert r.status == "imported"
+    assert importer.session.execute(
+        select(func.count()).select_from(Source)
+    ).scalar_one() == 0
+
+
+def test_sidecar_artist_used_when_no_folder_artist(importer, import_layout):
+    from backend.app.models import Artist
+
+    import_root, _ = import_layout
+    m = import_root / "e.jpg"                 # root → no folder artist
+    _split(m, "v")
+    _sidecar(m, {"category": "pixiv", "id": 9, "artist": "Yuki"})
+    r = importer.import_one(m)
+    assert r.status == "imported"
+    a = importer.session.execute(
+        select(Artist).where(Artist.slug == "yuki")
+    ).scalar_one()
+    src = importer.session.execute(select(Source)).scalar_one()
+    assert src.artist_id == a.id