bvandeusen/FabledCurator
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 33 Wiki Activity

feat(sidecar): importer creates Source/Post/ImageProvenance from sidecars (idempotent, fail-soft)

Browse Source 
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
Bryan Van Deusen
2026-05-18 14:46:17 -04:00
parent ef7e4d92a9
commit 9b79b66b76
2 changed files with 294 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/app/services/importer.py
+130 -6
View File
@@ -9,6 +9,8 @@ by the task. The Quart side uses an async session via the same models.
"""
import hashlib
import json
import logging
import shutil
from dataclasses import dataclass
from enum import StrEnum
@@ -18,13 +20,25 @@ from PIL import Image
from sqlalchemy import select
from sqlalchemy.orm import Session
from ..models import Artist, ImageRecord, ImportSettings, Tag, TagKind
from ..models import (
Artist,
ImageProvenance,
ImageRecord,
ImportSettings,
Post,
Source,
Tag,
TagKind,
)
from ..models.tag import image_tag
from ..utils.paths import derive_subdir, derive_top_level_artist, hash_suffixed_name
from ..utils.phash import compute_phash, find_similar
from ..utils.sidecar import find_sidecar, parse_sidecar
from ..utils.slug import slugify
from .thumbnailer import Thumbnailer
log = logging.getLogger(__name__)
class SkipReason(StrEnum):
too_small = "too_small"
@@ -227,9 +241,13 @@ class Importer:
self.session.flush()
# Folder→artist auto-tag.
artist = None
artist_name = derive_top_level_artist(source, self.import_root)
if artist_name:
self._attach_artist_tag(record.id, artist_name)
artist = self._attach_artist_tag(record.id, artist_name)
# Sidecar provenance (best-effort; never fails the import).
self._apply_sidecar(record, source, artist)
# Thumbnail is queued separately by the calling task; the importer
# does not generate thumbnails inline so the import queue stays moving.
@@ -237,16 +255,21 @@ class Importer:
self.session.commit()
return ImportResult(status="imported", image_id=record.id)
def _attach_artist_tag(self, image_id: int, artist_name: str) -> None:
"""Idempotently creates Artist + artist tag, attaches to image."""
slug = slugify(artist_name)
def _upsert_artist(self, name: str) -> Artist:
slug = slugify(name)
artist = self.session.execute(
select(Artist).where(Artist.slug == slug)
).scalar_one_or_none()
if artist is None:
artist = Artist(name=artist_name, slug=slug, is_subscription=False)
artist = Artist(name=name, slug=slug, is_subscription=False)
self.session.add(artist)
self.session.flush()
return artist
def _attach_artist_tag(self, image_id: int, artist_name: str) -> Artist:
"""Idempotently creates Artist + artist tag, attaches to image.
Returns the Artist (sidecar provenance attaches its Source to it)."""
artist = self._upsert_artist(artist_name)
tag = self.session.execute(
select(Tag).where(Tag.name == artist_name).where(Tag.kind == TagKind.artist)
@@ -270,6 +293,107 @@ class Importer:
image_record_id=image_id, tag_id=tag.id, source="auto",
)
)
return artist
@staticmethod
def _sidecar_artist_name(data: dict) -> str | None:
for k in ("artist", "author_name"):
v = data.get(k)
if isinstance(v, str) and v.strip():
return v.strip()
for k in ("user", "creator"):
v = data.get(k)
if isinstance(v, str) and v.strip():
return v.strip()
if isinstance(v, dict):
for sub in ("name", "full_name", "vanity", "account"):
sv = v.get(sub)
if isinstance(sv, str) and sv.strip():
return sv.strip()
return None
def _apply_sidecar(
self, record: ImageRecord, source: Path, artist: Artist | None
) -> None:
sc = find_sidecar(source)
if sc is None:
return
try:
data = json.loads(sc.read_text("utf-8"))
if not isinstance(data, dict):
raise ValueError("sidecar JSON is not an object")
except Exception as exc:
log.warning("sidecar parse failed for %s: %s", sc, exc)
return
sd = parse_sidecar(data)
if artist is None:
name = self._sidecar_artist_name(data)
if name:
artist = self._upsert_artist(name)
if artist is None:
log.warning(
"sidecar present for %s but no artist; skipping provenance",
source,
)
return
platform = sd.platform or "unknown"
url = sd.post_url or f"sidecar:{platform}"
src = self.session.execute(
select(Source).where(
Source.artist_id == artist.id,
Source.platform == platform,
Source.url == url,
)
).scalar_one_or_none()
if src is None:
src = Source(artist_id=artist.id, platform=platform, url=url)
self.session.add(src)
self.session.flush()
epid = sd.external_post_id or sc.stem
post = self.session.execute(
select(Post).where(
Post.source_id == src.id,
Post.external_post_id == epid,
)
).scalar_one_or_none()
if post is None:
post = Post(source_id=src.id, external_post_id=epid)
self.session.add(post)
self.session.flush()
if sd.post_url is not None:
post.post_url = sd.post_url
if sd.post_title is not None:
post.post_title = sd.post_title
if sd.post_date is not None:
post.post_date = sd.post_date
if sd.description is not None:
post.description = sd.description
if sd.attachment_count is not None:
post.attachment_count = sd.attachment_count
post.raw_metadata = sd.raw
exists = self.session.execute(
select(ImageProvenance.id).where(
ImageProvenance.image_record_id == record.id,
ImageProvenance.post_id == post.id,
)
).scalar_one_or_none()
if exists is None:
self.session.add(
ImageProvenance(
image_record_id=record.id,
post_id=post.id,
source_id=src.id,
captured_metadata=sd.raw,
)
)
if record.primary_post_id is None:
record.primary_post_id = post.id
self.session.flush()
def _supersede(
self, existing: ImageRecord, source: Path, sha: str,
tests/test_sidecar_import.py
+164
View File
@@ -0,0 +1,164 @@
"""FC-2d-v: sidecar → Source/Post/ImageProvenance (sync importer)."""
import json
from pathlib import Path
import pytest
from PIL import Image
from sqlalchemy import func, select
from backend.app.models import (
ImageProvenance,
ImageRecord,
ImportSettings,
Post,
Source,
)
from backend.app.services.importer import Importer
from backend.app.services.thumbnailer import Thumbnailer
pytestmark = pytest.mark.integration
@pytest.fixture
def import_layout(tmp_path):
import_root = tmp_path / "import"
images_root = tmp_path / "images"
import_root.mkdir()
images_root.mkdir()
return import_root, images_root
@pytest.fixture
def importer(db_sync, import_layout):
import_root, images_root = import_layout
settings = db_sync.execute(
select(ImportSettings).where(ImportSettings.id == 1)
).scalar_one()
return Importer(
session=db_sync,
images_root=images_root,
import_root=import_root,
thumbnailer=Thumbnailer(images_root=images_root),
settings=settings,
)
def _split(path: Path, orient, size=(256, 256)):
"""Structured image (half/half) — solid colors phash-collapse, which
would route a 2nd image to the supersede path (no sidecar applied)."""
path.parent.mkdir(parents=True, exist_ok=True)
w, h = size
im = Image.new("L", size, 0)
px = im.load()
for y in range(h):
for x in range(w):
if (x / w if orient == "v" else y / h) >= 0.5:
px[x, y] = 255
im.convert("RGB").save(path, "JPEG")
def _sidecar(media: Path, payload: dict):
media.with_suffix(".json").write_text(json.dumps(payload))
def test_sidecar_creates_provenance(importer, import_layout):
import_root, _ = import_layout
m = import_root / "Alice" / "a.jpg"
_split(m, "v")
_sidecar(m, {
"category": "patreon", "id": 555,
"url": "https://patreon.com/posts/555", "title": "Set 1",
"content": "<p>hi</p>", "page_count": 2,
"published_at": "2023-08-01T00:00:00Z",
})
r = importer.import_one(m)
assert r.status == "imported"
rec = importer.session.get(ImageRecord, r.image_id)
src = importer.session.execute(select(Source)).scalar_one()
post = importer.session.execute(select(Post)).scalar_one()
assert src.platform == "patreon"
assert post.external_post_id == "555"
assert post.post_url == "https://patreon.com/posts/555"
assert post.post_title == "Set 1"
assert post.description == "<p>hi</p>"
assert post.attachment_count == 2
assert post.post_date is not None
assert post.raw_metadata["id"] == 555
prov = importer.session.execute(select(ImageProvenance)).scalar_one()
assert prov.image_record_id == rec.id and prov.post_id == post.id
assert rec.primary_post_id == post.id
def test_reimport_same_post_idempotent(importer, import_layout):
import_root, _ = import_layout
payload = {"category": "patreon", "id": 777, "title": "P"}
m1 = import_root / "Bob" / "p1.jpg"
_split(m1, "v")
_sidecar(m1, payload)
importer.import_one(m1)
m2 = import_root / "Bob" / "p2.jpg"
_split(m2, "h") # structurally distinct → not a phash dup
_sidecar(m2, payload)
r2 = importer.import_one(m2)
assert r2.status == "imported"
assert importer.session.execute(
select(func.count()).select_from(Source)
).scalar_one() == 1
assert importer.session.execute(
select(func.count()).select_from(Post)
).scalar_one() == 1
assert importer.session.execute(
select(func.count()).select_from(ImageProvenance)
).scalar_one() == 2
def test_garbage_sidecar_still_imports(importer, import_layout):
import_root, _ = import_layout
m = import_root / "Carol" / "c.jpg"
_split(m, "v")
m.with_suffix(".json").write_text("{ not json")
r = importer.import_one(m)
assert r.status == "imported"
assert importer.session.execute(
select(func.count()).select_from(Post)
).scalar_one() == 0
def test_no_sidecar_unchanged(importer, import_layout):
import_root, _ = import_layout
m = import_root / "Dave" / "d.jpg"
_split(m, "v")
r = importer.import_one(m)
assert r.status == "imported"
assert importer.session.execute(
select(func.count()).select_from(Post)
).scalar_one() == 0
def test_no_artist_anywhere_skips_provenance(importer, import_layout):
import_root, _ = import_layout
m = import_root / "rootfile.jpg" # no top-level artist folder
_split(m, "v")
_sidecar(m, {"category": "x", "id": 1}) # no artist key
r = importer.import_one(m)
assert r.status == "imported"
assert importer.session.execute(
select(func.count()).select_from(Source)
).scalar_one() == 0
def test_sidecar_artist_used_when_no_folder_artist(importer, import_layout):
from backend.app.models import Artist
import_root, _ = import_layout
m = import_root / "e.jpg" # root → no folder artist
_split(m, "v")
_sidecar(m, {"category": "pixiv", "id": 9, "artist": "Yuki"})
r = importer.import_one(m)
assert r.status == "imported"
a = importer.session.execute(
select(Artist).where(Artist.slug == "yuki")
).scalar_one()
src = importer.session.execute(select(Source)).scalar_one()
assert src.artist_id == a.id