feat(attachments): importer dispatch — archive extract + non-media capture

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

backend/app/services/importer.py

@@ -12,7 +12,7 @@ import hashlib
 import json
 import logging
 import shutil
-from dataclasses import dataclass
+from dataclasses import dataclass, field
 from enum import StrEnum
 from pathlib import Path
 
@@ -26,12 +26,15 @@ from ..models import (
     ImageRecord,
     ImportSettings,
     Post,
+    PostAttachment,
     Source,
 )
 from ..utils.paths import derive_subdir, derive_top_level_artist, hash_suffixed_name
 from ..utils.phash import compute_phash, find_similar
 from ..utils.sidecar import find_sidecar, parse_sidecar
 from ..utils.slug import slugify
+from .archive_extractor import extract_archive, is_archive
+from .attachment_store import AttachmentStore
 from .thumbnailer import Thumbnailer
 
 log = logging.getLogger(__name__)
@@ -48,10 +51,11 @@ class SkipReason(StrEnum):
 
 @dataclass(frozen=True)
 class ImportResult:
-    status: str  # 'imported' | 'skipped' | 'failed' | 'superseded'
+    status: str  # 'imported'|'skipped'|'failed'|'superseded'|'attached'
     image_id: int | None = None
     skip_reason: SkipReason | None = None
     error: str | None = None
+    member_image_ids: list[int] = field(default_factory=list)
 
 
 IMAGE_EXTS = {".png", ".jpg", ".jpeg", ".gif", ".webp", ".bmp", ".tif", ".tiff"}
@@ -119,12 +123,128 @@ class Importer:
         self.import_root = import_root
         self.thumbnailer = thumbnailer
         self.settings = settings
+        self.attachments = AttachmentStore(images_root)
 
     def import_one(self, source: Path) -> ImportResult:
-        """The full pipeline for a single file.
+        """Dispatch by kind. Media → normal pipeline. Archive → extract
+        media members (one Post via the archive-adjacent sidecar) and
+        preserve the archive itself. Other non-media → PostAttachment.
+        Nothing is skipped for being non-media (FC-2d-iii)."""
+        if source.suffix.lower() == ".json":
+            return ImportResult(
+                status="skipped", skip_reason=SkipReason.invalid_image,
+                error="sidecar json is metadata, not content",
+            )
+        if is_archive(source):
+            return self._import_archive(source)
+        if not is_supported(source):
+            return self._capture_attachment(source)
+        return self._import_media(source, source)
 
-        Returns an ImportResult; callers (the Celery task) are responsible for
+    def _resolve_artist(self, attribution_path: Path) -> Artist | None:
-        translating that into ImportTask.status and ImageRecord linkage.
+        name = derive_top_level_artist(attribution_path, self.import_root)
+        return self._upsert_artist(name) if name else None
+
+    def _post_for_sidecar(
+        self, source: Path, artist: Artist | None
+    ) -> Post | None:
+        """If a sidecar sits next to `source`, ensure its Source+Post
+        exist (idempotent) and return the Post — so attachments can link
+        to the same Post the per-member _apply_sidecar will reuse."""
+        sc = find_sidecar(source)
+        if sc is None or artist is None:
+            return None
+        try:
+            data = json.loads(sc.read_text("utf-8"))
+            if not isinstance(data, dict):
+                raise ValueError("sidecar JSON is not an object")
+        except Exception as exc:
+            log.warning("sidecar parse failed for %s: %s", sc, exc)
+            return None
+        sd = parse_sidecar(data)
+        platform = sd.platform or "unknown"
+        url = sd.post_url or f"sidecar:{platform}"
+        src = self.session.execute(
+            select(Source).where(
+                Source.artist_id == artist.id,
+                Source.platform == platform,
+                Source.url == url,
+            )
+        ).scalar_one_or_none()
+        if src is None:
+            src = Source(artist_id=artist.id, platform=platform, url=url)
+            self.session.add(src)
+            self.session.flush()
+        epid = sd.external_post_id or sc.stem
+        post = self.session.execute(
+            select(Post).where(
+                Post.source_id == src.id,
+                Post.external_post_id == epid,
+            )
+        ).scalar_one_or_none()
+        if post is None:
+            post = Post(source_id=src.id, external_post_id=epid)
+            self.session.add(post)
+            self.session.flush()
+        return post
+
+    def _capture_attachment(
+        self, source: Path, *, post: Post | None = None,
+        artist: Artist | None = None, resolved: bool = False,
+    ) -> ImportResult:
+        if not resolved:
+            artist = self._resolve_artist(source)
+            post = self._post_for_sidecar(source, artist)
+        sha = _sha256_of(source)
+        existing = self.session.execute(
+            select(PostAttachment).where(PostAttachment.sha256 == sha)
+        ).scalar_one_or_none()
+        if existing is None:
+            stored = self.attachments.store(source, sha)
+            self.session.add(PostAttachment(
+                post_id=post.id if post else None,
+                artist_id=artist.id if artist else None,
+                sha256=sha,
+                path=stored,
+                original_filename=source.name,
+                ext=source.suffix.lower(),
+                mime=_mime_for(source),
+                size_bytes=source.stat().st_size,
+            ))
+            self.session.flush()
+        self.session.commit()
+        return ImportResult(status="attached")
+
+    def _import_archive(self, source: Path) -> ImportResult:
+        artist = self._resolve_artist(source)
+        post = self._post_for_sidecar(source, artist)
+        member_ids: list[int] = []
+        with extract_archive(source) as members:
+            for _name, member_path in members:
+                if not is_supported(member_path):
+                    continue  # non-media preserved via the stored archive
+                res = self._import_media(member_path, source)
+                if res.status in ("imported", "superseded") and res.image_id:
+                    member_ids.append(res.image_id)
+        # Preserve the archive itself (links to the same Post/Artist).
+        self._capture_attachment(
+            source, post=post, artist=artist, resolved=True
+        )
+        if member_ids:
+            return ImportResult(
+                status="imported", image_id=member_ids[0],
+                member_image_ids=member_ids,
+            )
+        return ImportResult(status="attached")
+
+    def _import_media(
+        self, source: Path, attribution_path: Path
+    ) -> ImportResult:
+        """The media import pipeline (filters, dedup, copy, provenance).
+
+        `attribution_path` anchors artist/subdir/sidecar derivation: for
+        a normal file it equals `source`; for an archive member it is the
+        archive's real path (the member lives in a tempdir).
         """
         if not is_supported(source):
             return ImportResult(
@@ -211,8 +331,8 @@ class Importer:
                         status="superseded", image_id=match_id
                     )
 
-        # Destination path.
+        # Destination path (artist/subdir anchored to attribution_path).
-        subdir = derive_subdir(source, self.import_root)
+        subdir = derive_subdir(attribution_path, self.import_root)
         dest_dir = self.images_root / subdir if subdir else self.images_root
         dest_dir.mkdir(parents=True, exist_ok=True)
         dest_name = hash_suffixed_name(source.stem, sha, source.suffix)
@@ -237,14 +357,16 @@ class Importer:
         self.session.add(record)
         self.session.flush()
 
-        # Folder→artist auto-tag.
+        # Folder→artist (anchored to attribution_path).
         artist = None
-        artist_name = derive_top_level_artist(source, self.import_root)
+        artist_name = derive_top_level_artist(
+            attribution_path, self.import_root
+        )
         if artist_name:
             artist = self._attach_artist(record, artist_name)
 
         # Sidecar provenance (best-effort; never fails the import).
-        self._apply_sidecar(record, source, artist)
+        self._apply_sidecar(record, attribution_path, artist)
 
         # Thumbnail is queued separately by the calling task; the importer
         # does not generate thumbnails inline so the import queue stays moving.

tests/test_importer.py

@@ -117,13 +117,14 @@ def test_transparent_filter(importer, import_layout):
 
 
 def test_unsupported_extension(importer, import_layout):
+    # FC-2d-iii: non-media is no longer skipped — it's captured as a
+    # PostAttachment so nothing a post contained is lost.
     import_root, _ = import_layout
     src = import_root / "Alice" / "notes.txt"
     src.parent.mkdir(parents=True, exist_ok=True)
     src.write_text("hello")
     result = importer.import_one(src)
-    assert result.status == "skipped"
+    assert result.status == "attached"
-    assert result.skip_reason == SkipReason.invalid_image
 
 
 def test_root_level_file_has_no_artist(importer, import_layout):

tests/test_importer_archive.py

@@ -0,0 +1,102 @@
+"""FC-2d-iii: archive → import media members + store the archive."""
+
+import io
+import json
+import zipfile
+
+import pytest
+from PIL import Image
+from sqlalchemy import func, select
+
+from backend.app.models import (
+    ImageProvenance,
+    ImageRecord,
+    ImportSettings,
+    Post,
+    PostAttachment,
+)
+from backend.app.services.importer import Importer
+from backend.app.services.thumbnailer import Thumbnailer
+
+pytestmark = pytest.mark.integration
+
+
+@pytest.fixture
+def import_layout(tmp_path):
+    import_root = tmp_path / "import"
+    images_root = tmp_path / "images"
+    import_root.mkdir()
+    images_root.mkdir()
+    return import_root, images_root
+
+
+@pytest.fixture
+def importer(db_sync, import_layout):
+    import_root, images_root = import_layout
+    settings = db_sync.execute(
+        select(ImportSettings).where(ImportSettings.id == 1)
+    ).scalar_one()
+    return Importer(
+        session=db_sync, images_root=images_root, import_root=import_root,
+        thumbnailer=Thumbnailer(images_root=images_root), settings=settings,
+    )
+
+
+def _jpeg_bytes(color):
+    buf = io.BytesIO()
+    Image.new("RGB", (40, 40), color).save(buf, "JPEG")
+    return buf.getvalue()
+
+
+def test_archive_imports_members_and_stores_archive(importer, import_layout):
+    import_root, _ = import_layout
+    arc = import_root / "Bob" / "set.cbz"
+    arc.parent.mkdir(parents=True, exist_ok=True)
+    with zipfile.ZipFile(arc, "w") as zf:
+        zf.writestr("a.jpg", _jpeg_bytes((200, 10, 10)))
+        zf.writestr("b.jpg", _jpeg_bytes((10, 200, 10)))
+        zf.writestr("readme.txt", b"hello")
+    arc.with_suffix(".cbz.json").write_text(json.dumps(
+        {"category": "patreon", "id": "777", "title": "Set"}))
+
+    r = importer.import_one(arc)
+    assert r.status == "imported"
+    assert len(r.member_image_ids) == 2
+
+    imgs = importer.session.execute(select(ImageRecord)).scalars().all()
+    assert len(imgs) == 2
+    post = importer.session.execute(select(Post)).scalar_one()
+    provs = importer.session.execute(
+        select(func.count()).select_from(ImageProvenance)
+    ).scalar_one()
+    assert provs == 2  # one Post, both members
+    att = importer.session.execute(select(PostAttachment)).scalar_one()
+    assert att.original_filename == "set.cbz"
+    assert att.post_id == post.id
+
+
+def test_corrupt_archive_still_stored(importer, import_layout):
+    import_root, _ = import_layout
+    arc = import_root / "Bob" / "broken.zip"
+    arc.parent.mkdir(parents=True, exist_ok=True)
+    arc.write_bytes(b"definitely not a zip")
+    r = importer.import_one(arc)
+    assert r.status == "attached"
+    att = importer.session.execute(select(PostAttachment)).scalar_one()
+    assert att.original_filename == "broken.zip"
+    assert importer.session.execute(
+        select(func.count()).select_from(ImageRecord)
+    ).scalar_one() == 0
+
+
+def test_reimport_archive_is_idempotent(importer, import_layout):
+    import_root, _ = import_layout
+    arc = import_root / "Bob" / "set.zip"
+    arc.parent.mkdir(parents=True, exist_ok=True)
+    with zipfile.ZipFile(arc, "w") as zf:
+        zf.writestr("a.jpg", _jpeg_bytes((1, 2, 3)))
+    importer.import_one(arc)
+    importer.import_one(arc)
+    assert importer.session.execute(
+        select(func.count()).select_from(PostAttachment)
+    ).scalar_one() == 1

tests/test_importer_attachments.py

@@ -0,0 +1,61 @@
+"""FC-2d-iii: loose non-media → PostAttachment (no skip)."""
+
+import pytest
+from sqlalchemy import func, select
+
+from backend.app.models import Artist, ImportSettings, PostAttachment
+from backend.app.services.importer import Importer
+from backend.app.services.thumbnailer import Thumbnailer
+
+pytestmark = pytest.mark.integration
+
+
+@pytest.fixture
+def import_layout(tmp_path):
+    import_root = tmp_path / "import"
+    images_root = tmp_path / "images"
+    import_root.mkdir()
+    images_root.mkdir()
+    return import_root, images_root
+
+
+@pytest.fixture
+def importer(db_sync, import_layout):
+    import_root, images_root = import_layout
+    settings = db_sync.execute(
+        select(ImportSettings).where(ImportSettings.id == 1)
+    ).scalar_one()
+    return Importer(
+        session=db_sync, images_root=images_root, import_root=import_root,
+        thumbnailer=Thumbnailer(images_root=images_root), settings=settings,
+    )
+
+
+def test_loose_non_media_becomes_attachment(importer, import_layout):
+    import_root, _ = import_layout
+    f = import_root / "Alice" / "manifesto.pdf"
+    f.parent.mkdir(parents=True, exist_ok=True)
+    f.write_bytes(b"%PDF-1.4 stuff")
+    r = importer.import_one(f)
+    assert r.status == "attached"
+    att = importer.session.execute(select(PostAttachment)).scalar_one()
+    artist = importer.session.execute(
+        select(Artist).where(Artist.slug == "alice")
+    ).scalar_one()
+    assert att.artist_id == artist.id
+    assert att.post_id is None
+    assert att.original_filename == "manifesto.pdf"
+    assert att.ext == ".pdf"
+
+
+def test_json_sidecar_is_not_attached(importer, import_layout):
+    import_root, _ = import_layout
+    j = import_root / "Alice" / "x.json"
+    j.parent.mkdir(parents=True, exist_ok=True)
+    j.write_text("{}")
+    r = importer.import_one(j)
+    assert r.status == "skipped"
+    n = importer.session.execute(
+        select(func.count()).select_from(PostAttachment)
+    ).scalar_one()
+    assert n == 0