diff --git a/backend/app/services/importer.py b/backend/app/services/importer.py index 589fa2a..dab757a 100644 --- a/backend/app/services/importer.py +++ b/backend/app/services/importer.py @@ -12,7 +12,7 @@ import hashlib import json import logging import shutil -from dataclasses import dataclass +from dataclasses import dataclass, field from enum import StrEnum from pathlib import Path @@ -26,12 +26,15 @@ from ..models import ( ImageRecord, ImportSettings, Post, + PostAttachment, Source, ) from ..utils.paths import derive_subdir, derive_top_level_artist, hash_suffixed_name from ..utils.phash import compute_phash, find_similar from ..utils.sidecar import find_sidecar, parse_sidecar from ..utils.slug import slugify +from .archive_extractor import extract_archive, is_archive +from .attachment_store import AttachmentStore from .thumbnailer import Thumbnailer log = logging.getLogger(__name__) @@ -48,10 +51,11 @@ class SkipReason(StrEnum): @dataclass(frozen=True) class ImportResult: - status: str # 'imported' | 'skipped' | 'failed' | 'superseded' + status: str # 'imported'|'skipped'|'failed'|'superseded'|'attached' image_id: int | None = None skip_reason: SkipReason | None = None error: str | None = None + member_image_ids: list[int] = field(default_factory=list) IMAGE_EXTS = {".png", ".jpg", ".jpeg", ".gif", ".webp", ".bmp", ".tif", ".tiff"} @@ -119,12 +123,128 @@ class Importer: self.import_root = import_root self.thumbnailer = thumbnailer self.settings = settings + self.attachments = AttachmentStore(images_root) def import_one(self, source: Path) -> ImportResult: - """The full pipeline for a single file. + """Dispatch by kind. Media → normal pipeline. Archive → extract + media members (one Post via the archive-adjacent sidecar) and + preserve the archive itself. Other non-media → PostAttachment. + Nothing is skipped for being non-media (FC-2d-iii).""" + if source.suffix.lower() == ".json": + return ImportResult( + status="skipped", skip_reason=SkipReason.invalid_image, + error="sidecar json is metadata, not content", + ) + if is_archive(source): + return self._import_archive(source) + if not is_supported(source): + return self._capture_attachment(source) + return self._import_media(source, source) - Returns an ImportResult; callers (the Celery task) are responsible for - translating that into ImportTask.status and ImageRecord linkage. + def _resolve_artist(self, attribution_path: Path) -> Artist | None: + name = derive_top_level_artist(attribution_path, self.import_root) + return self._upsert_artist(name) if name else None + + def _post_for_sidecar( + self, source: Path, artist: Artist | None + ) -> Post | None: + """If a sidecar sits next to `source`, ensure its Source+Post + exist (idempotent) and return the Post — so attachments can link + to the same Post the per-member _apply_sidecar will reuse.""" + sc = find_sidecar(source) + if sc is None or artist is None: + return None + try: + data = json.loads(sc.read_text("utf-8")) + if not isinstance(data, dict): + raise ValueError("sidecar JSON is not an object") + except Exception as exc: + log.warning("sidecar parse failed for %s: %s", sc, exc) + return None + sd = parse_sidecar(data) + platform = sd.platform or "unknown" + url = sd.post_url or f"sidecar:{platform}" + src = self.session.execute( + select(Source).where( + Source.artist_id == artist.id, + Source.platform == platform, + Source.url == url, + ) + ).scalar_one_or_none() + if src is None: + src = Source(artist_id=artist.id, platform=platform, url=url) + self.session.add(src) + self.session.flush() + epid = sd.external_post_id or sc.stem + post = self.session.execute( + select(Post).where( + Post.source_id == src.id, + Post.external_post_id == epid, + ) + ).scalar_one_or_none() + if post is None: + post = Post(source_id=src.id, external_post_id=epid) + self.session.add(post) + self.session.flush() + return post + + def _capture_attachment( + self, source: Path, *, post: Post | None = None, + artist: Artist | None = None, resolved: bool = False, + ) -> ImportResult: + if not resolved: + artist = self._resolve_artist(source) + post = self._post_for_sidecar(source, artist) + sha = _sha256_of(source) + existing = self.session.execute( + select(PostAttachment).where(PostAttachment.sha256 == sha) + ).scalar_one_or_none() + if existing is None: + stored = self.attachments.store(source, sha) + self.session.add(PostAttachment( + post_id=post.id if post else None, + artist_id=artist.id if artist else None, + sha256=sha, + path=stored, + original_filename=source.name, + ext=source.suffix.lower(), + mime=_mime_for(source), + size_bytes=source.stat().st_size, + )) + self.session.flush() + self.session.commit() + return ImportResult(status="attached") + + def _import_archive(self, source: Path) -> ImportResult: + artist = self._resolve_artist(source) + post = self._post_for_sidecar(source, artist) + member_ids: list[int] = [] + with extract_archive(source) as members: + for _name, member_path in members: + if not is_supported(member_path): + continue # non-media preserved via the stored archive + res = self._import_media(member_path, source) + if res.status in ("imported", "superseded") and res.image_id: + member_ids.append(res.image_id) + # Preserve the archive itself (links to the same Post/Artist). + self._capture_attachment( + source, post=post, artist=artist, resolved=True + ) + if member_ids: + return ImportResult( + status="imported", image_id=member_ids[0], + member_image_ids=member_ids, + ) + return ImportResult(status="attached") + + def _import_media( + self, source: Path, attribution_path: Path + ) -> ImportResult: + """The media import pipeline (filters, dedup, copy, provenance). + + `attribution_path` anchors artist/subdir/sidecar derivation: for + a normal file it equals `source`; for an archive member it is the + archive's real path (the member lives in a tempdir). """ if not is_supported(source): return ImportResult( @@ -211,8 +331,8 @@ class Importer: status="superseded", image_id=match_id ) - # Destination path. - subdir = derive_subdir(source, self.import_root) + # Destination path (artist/subdir anchored to attribution_path). + subdir = derive_subdir(attribution_path, self.import_root) dest_dir = self.images_root / subdir if subdir else self.images_root dest_dir.mkdir(parents=True, exist_ok=True) dest_name = hash_suffixed_name(source.stem, sha, source.suffix) @@ -237,14 +357,16 @@ class Importer: self.session.add(record) self.session.flush() - # Folder→artist auto-tag. + # Folder→artist (anchored to attribution_path). artist = None - artist_name = derive_top_level_artist(source, self.import_root) + artist_name = derive_top_level_artist( + attribution_path, self.import_root + ) if artist_name: artist = self._attach_artist(record, artist_name) # Sidecar provenance (best-effort; never fails the import). - self._apply_sidecar(record, source, artist) + self._apply_sidecar(record, attribution_path, artist) # Thumbnail is queued separately by the calling task; the importer # does not generate thumbnails inline so the import queue stays moving. diff --git a/tests/test_importer.py b/tests/test_importer.py index a72f522..b809de6 100644 --- a/tests/test_importer.py +++ b/tests/test_importer.py @@ -117,13 +117,14 @@ def test_transparent_filter(importer, import_layout): def test_unsupported_extension(importer, import_layout): + # FC-2d-iii: non-media is no longer skipped — it's captured as a + # PostAttachment so nothing a post contained is lost. import_root, _ = import_layout src = import_root / "Alice" / "notes.txt" src.parent.mkdir(parents=True, exist_ok=True) src.write_text("hello") result = importer.import_one(src) - assert result.status == "skipped" - assert result.skip_reason == SkipReason.invalid_image + assert result.status == "attached" def test_root_level_file_has_no_artist(importer, import_layout): diff --git a/tests/test_importer_archive.py b/tests/test_importer_archive.py new file mode 100644 index 0000000..3b72057 --- /dev/null +++ b/tests/test_importer_archive.py @@ -0,0 +1,102 @@ +"""FC-2d-iii: archive → import media members + store the archive.""" + +import io +import json +import zipfile + +import pytest +from PIL import Image +from sqlalchemy import func, select + +from backend.app.models import ( + ImageProvenance, + ImageRecord, + ImportSettings, + Post, + PostAttachment, +) +from backend.app.services.importer import Importer +from backend.app.services.thumbnailer import Thumbnailer + +pytestmark = pytest.mark.integration + + +@pytest.fixture +def import_layout(tmp_path): + import_root = tmp_path / "import" + images_root = tmp_path / "images" + import_root.mkdir() + images_root.mkdir() + return import_root, images_root + + +@pytest.fixture +def importer(db_sync, import_layout): + import_root, images_root = import_layout + settings = db_sync.execute( + select(ImportSettings).where(ImportSettings.id == 1) + ).scalar_one() + return Importer( + session=db_sync, images_root=images_root, import_root=import_root, + thumbnailer=Thumbnailer(images_root=images_root), settings=settings, + ) + + +def _jpeg_bytes(color): + buf = io.BytesIO() + Image.new("RGB", (40, 40), color).save(buf, "JPEG") + return buf.getvalue() + + +def test_archive_imports_members_and_stores_archive(importer, import_layout): + import_root, _ = import_layout + arc = import_root / "Bob" / "set.cbz" + arc.parent.mkdir(parents=True, exist_ok=True) + with zipfile.ZipFile(arc, "w") as zf: + zf.writestr("a.jpg", _jpeg_bytes((200, 10, 10))) + zf.writestr("b.jpg", _jpeg_bytes((10, 200, 10))) + zf.writestr("readme.txt", b"hello") + arc.with_suffix(".cbz.json").write_text(json.dumps( + {"category": "patreon", "id": "777", "title": "Set"})) + + r = importer.import_one(arc) + assert r.status == "imported" + assert len(r.member_image_ids) == 2 + + imgs = importer.session.execute(select(ImageRecord)).scalars().all() + assert len(imgs) == 2 + post = importer.session.execute(select(Post)).scalar_one() + provs = importer.session.execute( + select(func.count()).select_from(ImageProvenance) + ).scalar_one() + assert provs == 2 # one Post, both members + att = importer.session.execute(select(PostAttachment)).scalar_one() + assert att.original_filename == "set.cbz" + assert att.post_id == post.id + + +def test_corrupt_archive_still_stored(importer, import_layout): + import_root, _ = import_layout + arc = import_root / "Bob" / "broken.zip" + arc.parent.mkdir(parents=True, exist_ok=True) + arc.write_bytes(b"definitely not a zip") + r = importer.import_one(arc) + assert r.status == "attached" + att = importer.session.execute(select(PostAttachment)).scalar_one() + assert att.original_filename == "broken.zip" + assert importer.session.execute( + select(func.count()).select_from(ImageRecord) + ).scalar_one() == 0 + + +def test_reimport_archive_is_idempotent(importer, import_layout): + import_root, _ = import_layout + arc = import_root / "Bob" / "set.zip" + arc.parent.mkdir(parents=True, exist_ok=True) + with zipfile.ZipFile(arc, "w") as zf: + zf.writestr("a.jpg", _jpeg_bytes((1, 2, 3))) + importer.import_one(arc) + importer.import_one(arc) + assert importer.session.execute( + select(func.count()).select_from(PostAttachment) + ).scalar_one() == 1 diff --git a/tests/test_importer_attachments.py b/tests/test_importer_attachments.py new file mode 100644 index 0000000..06b6083 --- /dev/null +++ b/tests/test_importer_attachments.py @@ -0,0 +1,61 @@ +"""FC-2d-iii: loose non-media → PostAttachment (no skip).""" + +import pytest +from sqlalchemy import func, select + +from backend.app.models import Artist, ImportSettings, PostAttachment +from backend.app.services.importer import Importer +from backend.app.services.thumbnailer import Thumbnailer + +pytestmark = pytest.mark.integration + + +@pytest.fixture +def import_layout(tmp_path): + import_root = tmp_path / "import" + images_root = tmp_path / "images" + import_root.mkdir() + images_root.mkdir() + return import_root, images_root + + +@pytest.fixture +def importer(db_sync, import_layout): + import_root, images_root = import_layout + settings = db_sync.execute( + select(ImportSettings).where(ImportSettings.id == 1) + ).scalar_one() + return Importer( + session=db_sync, images_root=images_root, import_root=import_root, + thumbnailer=Thumbnailer(images_root=images_root), settings=settings, + ) + + +def test_loose_non_media_becomes_attachment(importer, import_layout): + import_root, _ = import_layout + f = import_root / "Alice" / "manifesto.pdf" + f.parent.mkdir(parents=True, exist_ok=True) + f.write_bytes(b"%PDF-1.4 stuff") + r = importer.import_one(f) + assert r.status == "attached" + att = importer.session.execute(select(PostAttachment)).scalar_one() + artist = importer.session.execute( + select(Artist).where(Artist.slug == "alice") + ).scalar_one() + assert att.artist_id == artist.id + assert att.post_id is None + assert att.original_filename == "manifesto.pdf" + assert att.ext == ".pdf" + + +def test_json_sidecar_is_not_attached(importer, import_layout): + import_root, _ = import_layout + j = import_root / "Alice" / "x.json" + j.parent.mkdir(parents=True, exist_ok=True) + j.write_text("{}") + r = importer.import_one(j) + assert r.status == "skipped" + n = importer.session.execute( + select(func.count()).select_from(PostAttachment) + ).scalar_one() + assert n == 0