diff --git a/backend/app/services/importer.py b/backend/app/services/importer.py index c45e971..82296ed 100644 --- a/backend/app/services/importer.py +++ b/backend/app/services/importer.py @@ -9,6 +9,8 @@ by the task. The Quart side uses an async session via the same models. """ import hashlib +import json +import logging import shutil from dataclasses import dataclass from enum import StrEnum @@ -18,13 +20,25 @@ from PIL import Image from sqlalchemy import select from sqlalchemy.orm import Session -from ..models import Artist, ImageRecord, ImportSettings, Tag, TagKind +from ..models import ( + Artist, + ImageProvenance, + ImageRecord, + ImportSettings, + Post, + Source, + Tag, + TagKind, +) from ..models.tag import image_tag from ..utils.paths import derive_subdir, derive_top_level_artist, hash_suffixed_name from ..utils.phash import compute_phash, find_similar +from ..utils.sidecar import find_sidecar, parse_sidecar from ..utils.slug import slugify from .thumbnailer import Thumbnailer +log = logging.getLogger(__name__) + class SkipReason(StrEnum): too_small = "too_small" @@ -227,9 +241,13 @@ class Importer: self.session.flush() # Folder→artist auto-tag. + artist = None artist_name = derive_top_level_artist(source, self.import_root) if artist_name: - self._attach_artist_tag(record.id, artist_name) + artist = self._attach_artist_tag(record.id, artist_name) + + # Sidecar provenance (best-effort; never fails the import). + self._apply_sidecar(record, source, artist) # Thumbnail is queued separately by the calling task; the importer # does not generate thumbnails inline so the import queue stays moving. @@ -237,16 +255,21 @@ class Importer: self.session.commit() return ImportResult(status="imported", image_id=record.id) - def _attach_artist_tag(self, image_id: int, artist_name: str) -> None: - """Idempotently creates Artist + artist tag, attaches to image.""" - slug = slugify(artist_name) + def _upsert_artist(self, name: str) -> Artist: + slug = slugify(name) artist = self.session.execute( select(Artist).where(Artist.slug == slug) ).scalar_one_or_none() if artist is None: - artist = Artist(name=artist_name, slug=slug, is_subscription=False) + artist = Artist(name=name, slug=slug, is_subscription=False) self.session.add(artist) self.session.flush() + return artist + + def _attach_artist_tag(self, image_id: int, artist_name: str) -> Artist: + """Idempotently creates Artist + artist tag, attaches to image. + Returns the Artist (sidecar provenance attaches its Source to it).""" + artist = self._upsert_artist(artist_name) tag = self.session.execute( select(Tag).where(Tag.name == artist_name).where(Tag.kind == TagKind.artist) @@ -270,6 +293,107 @@ class Importer: image_record_id=image_id, tag_id=tag.id, source="auto", ) ) + return artist + + @staticmethod + def _sidecar_artist_name(data: dict) -> str | None: + for k in ("artist", "author_name"): + v = data.get(k) + if isinstance(v, str) and v.strip(): + return v.strip() + for k in ("user", "creator"): + v = data.get(k) + if isinstance(v, str) and v.strip(): + return v.strip() + if isinstance(v, dict): + for sub in ("name", "full_name", "vanity", "account"): + sv = v.get(sub) + if isinstance(sv, str) and sv.strip(): + return sv.strip() + return None + + def _apply_sidecar( + self, record: ImageRecord, source: Path, artist: Artist | None + ) -> None: + sc = find_sidecar(source) + if sc is None: + return + try: + data = json.loads(sc.read_text("utf-8")) + if not isinstance(data, dict): + raise ValueError("sidecar JSON is not an object") + except Exception as exc: + log.warning("sidecar parse failed for %s: %s", sc, exc) + return + + sd = parse_sidecar(data) + + if artist is None: + name = self._sidecar_artist_name(data) + if name: + artist = self._upsert_artist(name) + if artist is None: + log.warning( + "sidecar present for %s but no artist; skipping provenance", + source, + ) + return + + platform = sd.platform or "unknown" + url = sd.post_url or f"sidecar:{platform}" + src = self.session.execute( + select(Source).where( + Source.artist_id == artist.id, + Source.platform == platform, + Source.url == url, + ) + ).scalar_one_or_none() + if src is None: + src = Source(artist_id=artist.id, platform=platform, url=url) + self.session.add(src) + self.session.flush() + + epid = sd.external_post_id or sc.stem + post = self.session.execute( + select(Post).where( + Post.source_id == src.id, + Post.external_post_id == epid, + ) + ).scalar_one_or_none() + if post is None: + post = Post(source_id=src.id, external_post_id=epid) + self.session.add(post) + self.session.flush() + if sd.post_url is not None: + post.post_url = sd.post_url + if sd.post_title is not None: + post.post_title = sd.post_title + if sd.post_date is not None: + post.post_date = sd.post_date + if sd.description is not None: + post.description = sd.description + if sd.attachment_count is not None: + post.attachment_count = sd.attachment_count + post.raw_metadata = sd.raw + + exists = self.session.execute( + select(ImageProvenance.id).where( + ImageProvenance.image_record_id == record.id, + ImageProvenance.post_id == post.id, + ) + ).scalar_one_or_none() + if exists is None: + self.session.add( + ImageProvenance( + image_record_id=record.id, + post_id=post.id, + source_id=src.id, + captured_metadata=sd.raw, + ) + ) + if record.primary_post_id is None: + record.primary_post_id = post.id + self.session.flush() def _supersede( self, existing: ImageRecord, source: Path, sha: str, diff --git a/tests/test_sidecar_import.py b/tests/test_sidecar_import.py new file mode 100644 index 0000000..817705b --- /dev/null +++ b/tests/test_sidecar_import.py @@ -0,0 +1,164 @@ +"""FC-2d-v: sidecar → Source/Post/ImageProvenance (sync importer).""" + +import json +from pathlib import Path + +import pytest +from PIL import Image +from sqlalchemy import func, select + +from backend.app.models import ( + ImageProvenance, + ImageRecord, + ImportSettings, + Post, + Source, +) +from backend.app.services.importer import Importer +from backend.app.services.thumbnailer import Thumbnailer + +pytestmark = pytest.mark.integration + + +@pytest.fixture +def import_layout(tmp_path): + import_root = tmp_path / "import" + images_root = tmp_path / "images" + import_root.mkdir() + images_root.mkdir() + return import_root, images_root + + +@pytest.fixture +def importer(db_sync, import_layout): + import_root, images_root = import_layout + settings = db_sync.execute( + select(ImportSettings).where(ImportSettings.id == 1) + ).scalar_one() + return Importer( + session=db_sync, + images_root=images_root, + import_root=import_root, + thumbnailer=Thumbnailer(images_root=images_root), + settings=settings, + ) + + +def _split(path: Path, orient, size=(256, 256)): + """Structured image (half/half) — solid colors phash-collapse, which + would route a 2nd image to the supersede path (no sidecar applied).""" + path.parent.mkdir(parents=True, exist_ok=True) + w, h = size + im = Image.new("L", size, 0) + px = im.load() + for y in range(h): + for x in range(w): + if (x / w if orient == "v" else y / h) >= 0.5: + px[x, y] = 255 + im.convert("RGB").save(path, "JPEG") + + +def _sidecar(media: Path, payload: dict): + media.with_suffix(".json").write_text(json.dumps(payload)) + + +def test_sidecar_creates_provenance(importer, import_layout): + import_root, _ = import_layout + m = import_root / "Alice" / "a.jpg" + _split(m, "v") + _sidecar(m, { + "category": "patreon", "id": 555, + "url": "https://patreon.com/posts/555", "title": "Set 1", + "content": "

hi

", "page_count": 2, + "published_at": "2023-08-01T00:00:00Z", + }) + r = importer.import_one(m) + assert r.status == "imported" + rec = importer.session.get(ImageRecord, r.image_id) + src = importer.session.execute(select(Source)).scalar_one() + post = importer.session.execute(select(Post)).scalar_one() + assert src.platform == "patreon" + assert post.external_post_id == "555" + assert post.post_url == "https://patreon.com/posts/555" + assert post.post_title == "Set 1" + assert post.description == "

hi

" + assert post.attachment_count == 2 + assert post.post_date is not None + assert post.raw_metadata["id"] == 555 + prov = importer.session.execute(select(ImageProvenance)).scalar_one() + assert prov.image_record_id == rec.id and prov.post_id == post.id + assert rec.primary_post_id == post.id + + +def test_reimport_same_post_idempotent(importer, import_layout): + import_root, _ = import_layout + payload = {"category": "patreon", "id": 777, "title": "P"} + m1 = import_root / "Bob" / "p1.jpg" + _split(m1, "v") + _sidecar(m1, payload) + importer.import_one(m1) + m2 = import_root / "Bob" / "p2.jpg" + _split(m2, "h") # structurally distinct → not a phash dup + _sidecar(m2, payload) + r2 = importer.import_one(m2) + assert r2.status == "imported" + assert importer.session.execute( + select(func.count()).select_from(Source) + ).scalar_one() == 1 + assert importer.session.execute( + select(func.count()).select_from(Post) + ).scalar_one() == 1 + assert importer.session.execute( + select(func.count()).select_from(ImageProvenance) + ).scalar_one() == 2 + + +def test_garbage_sidecar_still_imports(importer, import_layout): + import_root, _ = import_layout + m = import_root / "Carol" / "c.jpg" + _split(m, "v") + m.with_suffix(".json").write_text("{ not json") + r = importer.import_one(m) + assert r.status == "imported" + assert importer.session.execute( + select(func.count()).select_from(Post) + ).scalar_one() == 0 + + +def test_no_sidecar_unchanged(importer, import_layout): + import_root, _ = import_layout + m = import_root / "Dave" / "d.jpg" + _split(m, "v") + r = importer.import_one(m) + assert r.status == "imported" + assert importer.session.execute( + select(func.count()).select_from(Post) + ).scalar_one() == 0 + + +def test_no_artist_anywhere_skips_provenance(importer, import_layout): + import_root, _ = import_layout + m = import_root / "rootfile.jpg" # no top-level artist folder + _split(m, "v") + _sidecar(m, {"category": "x", "id": 1}) # no artist key + r = importer.import_one(m) + assert r.status == "imported" + assert importer.session.execute( + select(func.count()).select_from(Source) + ).scalar_one() == 0 + + +def test_sidecar_artist_used_when_no_folder_artist(importer, import_layout): + from backend.app.models import Artist + + import_root, _ = import_layout + m = import_root / "e.jpg" # root → no folder artist + _split(m, "v") + _sidecar(m, {"category": "pixiv", "id": 9, "artist": "Yuki"}) + r = importer.import_one(m) + assert r.status == "imported" + a = importer.session.execute( + select(Artist).where(Artist.slug == "yuki") + ).scalar_one() + src = importer.session.execute(select(Source)).scalar_one() + assert src.artist_id == a.id