feat: archive images reach the library: game builds stay whole, and re-extract covers every archive

Most archive attachments never had their images imported. The re-extract
action skipped archives without a post, and nothing kept game builds out
of an extraction.

- An archive holding executables, libraries, or game-engine data
  (Ren'Py, Unity, Godot, RPG Maker, Python builds) is kept as an
  attachment and not extracted. The same holds for one nested inside an
  art pack.
- Re-extract now reaches archives imported from a folder without a post.
  Their images go to the artist, in a folder named after the archive.
- Re-extract passes over archives whose members are already recorded.
- Re-extract stages each archive by symlink instead of copying it.
- The card says what the action does now.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

backend/app/services/archive_extractor.py

@@ -57,6 +57,50 @@ def is_archive(path: Path) -> bool:
     return detect_archive_format(path) is not None
 
 
+# Files that only a program ships: executables and libraries, and the packed
+# data and compiled scripts of the common game engines (Ren'Py, Unity, Godot,
+# RPG Maker, Python builds). An archive holding any of them is a game or app
+# build. Its images are sprites, UI and CGs bundled for the program, not art
+# for the library, so the archive is kept whole as an attachment.
+SOFTWARE_EXTS = frozenset({
+    ".exe", ".msi", ".dll", ".so", ".dylib", ".apk",
+    ".rpa", ".rpyc", ".rpy", ".pyc", ".pyo",
+    ".pak", ".pck", ".unity3d", ".assets", ".ress",
+    ".rgssad", ".rgss2a", ".rgss3a", ".rxdata", ".rvdata", ".rvdata2",
+})
+
+
+def member_names(path: Path) -> list[str]:
+    """Every member name in the archive, read from its directory without
+    decompressing anything. Empty when the archive can't be listed."""
+    try:
+        fmt = detect_archive_format(path)
+        if fmt == "zip":
+            with zipfile.ZipFile(path) as zf:
+                return zf.namelist()
+        if fmt == "rar":
+            import rarfile
+
+            with rarfile.RarFile(path) as rf:
+                return rf.namelist()
+        if fmt == "7z":
+            import py7zr
+
+            with py7zr.SevenZipFile(path, "r") as zf:
+                return zf.getnames()
+    except Exception as exc:
+        log.warning("archive listing failed for %s: %s", path, exc)
+    return []
+
+
+def software_marker(path: Path) -> str | None:
+    """The first member that marks the archive as a software build, or None."""
+    for name in member_names(path):
+        if Path(name).suffix.lower() in SOFTWARE_EXTS:
+            return name
+    return None
+
+
 @contextmanager
 def extract_archive(path: Path):
     """Context manager yielding a list of (member_name, extracted_path)

backend/app/services/cleanup_service.py

@@ -1044,45 +1044,75 @@ def cancel_audit_run(session: Session, *, audit_id: int) -> None:
 _ARCHIVE_EXT_FOR_FORMAT = {"zip": ".zip", "rar": ".rar", "7z": ".7z"}
 
 
+def _stage_archive(archive_path: Path, staged: Path) -> None:
+    """Put the stored archive at `staged` for the importer to read: a symlink,
+    so a multi-gigabyte archive isn't copied, or a copy where links aren't
+    possible."""
+    import shutil
+
+    staged.unlink(missing_ok=True)
+    try:
+        staged.symlink_to(archive_path)
+    except OSError:
+        shutil.copy2(archive_path, staged)
+
+
+def _archive_folder_name(original_filename: str | None, attachment_id: int) -> str:
+    """The library folder a post-less archive's members land in: the archive's
+    own name without its extension, made safe for a path."""
+    stem = Path(original_filename or "").stem.strip().strip(".")
+    stem = "".join(c if c not in '/\\:*?"<>|' else "_" for c in stem)[:120].strip()
+    return stem or f"archive-{attachment_id}"
+
+
 def _reextract_archive_to_post(
     importer, archive_path: Path, post, source_row, artist, images_root: Path,
+    *, folder: str = "",
 ) -> list[int]:
-    """Extract one stored archive and link its members to `post`.
+    """Extract one stored archive and import its members.
 
     The stored attachment has no adjacent sidecar (it lives in the sha-addressed
-    attachment store). Stage a copy + a reconstructed sidecar UNDER the artist's
-    library dir (`images_root/<slug>/<platform>/<post>/`) — the importer
-    re-derives the artist from the path AND copies members relative to it, so the
-    members land in the real library and resolve to the right artist — then re-run
-    `attach_in_place`: the archive extracts and `find_or_create_post` re-attaches
-    the members to the SAME Post (source_id + external_post_id). Removes only the
-    staged archive + sidecar afterward; the imported member files stay. Returns
-    the new member image ids.
+    attachment store), so the archive is staged under the artist's library dir
+    and `attach_in_place` re-run on it: the importer derives the artist from the
+    path and copies members relative to it, so they land in the real library.
+
+    - With a `post`, the stage is `images_root/<slug>/<platform>/<post>/` beside
+      a reconstructed sidecar, so `find_or_create_post` re-attaches the members
+      to that same Post (source_id + external_post_id).
+    - Without one (a filesystem import that had no sidecar), the stage is
+      `images_root/<slug>/<folder>/` with no sidecar, and the members belong to
+      the artist alone, as a filesystem import of the archive would have left
+      them.
+
+    Only the staged archive and sidecar are removed afterward; the imported
+    member files stay. Returns the new member image ids.
     """
     import json
-    import shutil
 
     from .archive_extractor import detect_archive_format
 
     fmt = detect_archive_format(archive_path)
     ext = _ARCHIVE_EXT_FOR_FORMAT.get(fmt or "", ".zip")
-    platform = source_row.platform if source_row is not None else "imported"
-    sidecar = {
-        "category": source_row.platform if source_row is not None
-        else (post.raw_metadata or {}).get("category"),
-        "id": post.external_post_id,
-        "title": post.post_title or "",
-        "content": post.description or "",
-        "published_at": post.post_date.isoformat() if post.post_date else None,
-        "url": post.post_url,
-    }
-    work = images_root / artist.slug / platform / str(post.external_post_id)
+    if post is not None:
+        platform = source_row.platform if source_row is not None else "imported"
+        work = images_root / artist.slug / platform / str(post.external_post_id)
+    else:
+        work = images_root / artist.slug / folder
     work.mkdir(parents=True, exist_ok=True)
     staged = work / f"archive{ext}"  # clean ext → is_archive + find_sidecar
     sidecar_path = staged.with_suffix(".json")
     try:
-        shutil.copy2(archive_path, staged)
-        sidecar_path.write_text(json.dumps(sidecar))
+        _stage_archive(archive_path, staged)
+        if post is not None:
+            sidecar_path.write_text(json.dumps({
+                "category": source_row.platform if source_row is not None
+                else (post.raw_metadata or {}).get("category"),
+                "id": post.external_post_id,
+                "title": post.post_title or "",
+                "content": post.description or "",
+                "published_at": post.post_date.isoformat() if post.post_date else None,
+                "url": post.post_url,
+            }))
         res = importer.attach_in_place(staged, artist=artist, source=source_row)
         return list(res.member_image_ids or [])
     finally:
@@ -1124,8 +1154,9 @@ def reextract_archive_attachments(
 
     summary = {
         "scanned": 0, "archives": 0, "members_imported": 0,
-        "posts_touched": 0, "skipped_no_post": 0, "skipped_no_artist": 0,
-        "errors": 0, "partial": False, "resume_after_id": after_id,
+        "posts_touched": 0, "without_post": 0, "already_extracted": 0,
+        "skipped_no_artist": 0, "errors": 0, "partial": False,
+        "resume_after_id": after_id,
     }
     settings = ImportSettings.load_sync(session)
     importer = Importer(
@@ -1150,15 +1181,17 @@ def reextract_archive_attachments(
         except OSError:
             continue
         summary["archives"] += 1
-        if att.post_id is None:
-            summary["skipped_no_post"] += 1
-            continue
-        post = session.get(Post, att.post_id)
-        if post is None:
-            summary["skipped_no_post"] += 1
+        # Members already recorded as coming from this archive: done before.
+        if session.execute(
+            select(ImageProvenance.id)
+            .where(ImageProvenance.from_attachment_id == att.id)
+            .limit(1)
+        ).first() is not None:
+            summary["already_extracted"] += 1
             continue
+        post = session.get(Post, att.post_id) if att.post_id else None
         artist = session.get(Artist, att.artist_id) if att.artist_id else None
-        if artist is None and post.artist_id:
+        if artist is None and post is not None and post.artist_id:
             artist = session.get(Artist, post.artist_id)
         if artist is None or not artist.slug:
             # The importer re-derives the artist from the staged path, so we need
@@ -1166,10 +1199,16 @@ def reextract_archive_attachments(
             # subscription posts; skip rather than orphan the members.)
             summary["skipped_no_artist"] += 1
             continue
-        source_row = session.get(Source, post.source_id) if post.source_id else None
+        source_row = (
+            session.get(Source, post.source_id)
+            if post is not None and post.source_id else None
+        )
+        if post is None:
+            summary["without_post"] += 1
         try:
             ids = _reextract_archive_to_post(
                 importer, stored, post, source_row, artist, images_root,
+                folder=_archive_folder_name(att.original_filename, att.id),
             )
             session.commit()
         except Exception as exc:  # one bad archive must not strand the rest
@@ -1179,7 +1218,8 @@ def reextract_archive_attachments(
             continue
         if ids:
             summary["members_imported"] += len(ids)
-            summary["posts_touched"] += 1
+            if post is not None:
+                summary["posts_touched"] += 1
             enqueue_ids.extend(ids)
 
         # Time-box the chunk. resume_after_id already points at this attachment,

backend/app/services/importer.py

@@ -45,7 +45,7 @@ from ..utils.phash import compute_phash, find_similar, fingerprint_path, fingerp
 from ..utils.sidecar import find_sidecar, parse_sidecar
 from ..utils.slug import slugify
 from ..utils.staging import is_temp_staging_file
-from .archive_extractor import extract_archive, is_archive
+from .archive_extractor import extract_archive, is_archive, software_marker
 from .attachment_store import AttachmentStore
 from .audits import single_color
 from .link_extract import extract_external_links
@@ -616,6 +616,14 @@ class Importer:
 
         artist_use = artist if artist is not None else self._resolve_artist(source)
         post = self._post_for_sidecar(source, artist_use, source_row=source_row)
+        if (marker := software_marker(source)) is not None:
+            # A game or app build: kept whole, its bundled images left alone.
+            self._capture_attachment(
+                source, post=post, artist=artist_use, resolved=True,
+            )
+            log.info("%s: a software build (holds %s), kept unextracted",
+                     source.name, marker)
+            return ImportResult(status="attached")
         # Committed before extraction, so a member that fails (and is rolled
         # back on its own) can never take the archive's post with it.
         self.session.commit()
@@ -631,7 +639,7 @@ class Importer:
         # or failed/corrupt members — instead of one catch-all string.
         counts = {
             "media": 0, "deduped": 0, "unsupported": 0,
-            "failed": 0, "nested": 0, "nested_rejected": 0,
+            "failed": 0, "nested": 0, "nested_rejected": 0, "nested_software": 0,
         }
         self._collect_archive_members(
             source, attribution=source, source_row=source_row,
@@ -663,20 +671,21 @@ class Importer:
         if counts["media"] == 0 and counts["nested"] == 0:
             reason = ("archive yielded no members (unsupported/corrupt, or the "
                       "extractor backend failed)")
-        elif counts["deduped"] and not (
+        elif (counts["deduped"] or counts["nested_software"]) and not (
             counts["unsupported"] or counts["failed"] or counts["nested_rejected"]
         ):
             log.info(
-                "%s: all %d image member(s) already in library (deduped + linked "
-                "to this post) — archive preserved, no new import",
-                source.name, counts["deduped"],
+                "%s: no new images — %d already in the library (linked to this "
+                "post), %d nested software build(s) skipped; archive preserved",
+                source.name, counts["deduped"], counts["nested_software"],
             )
             return ImportResult(status="attached")
         else:
             reason = (
                 f"no new images — media={counts['media']} deduped={counts['deduped']} "
                 f"unsupported/non-media={counts['unsupported']} failed={counts['failed']} "
-                f"nested={counts['nested']} nested-rejected={counts['nested_rejected']}"
+                f"nested={counts['nested']} nested-rejected={counts['nested_rejected']} "
+                f"nested-software={counts['nested_software']}"
             )
         log.warning("%s: %s", source.name, reason)
         return ImportResult(status="attached", error=reason)
@@ -717,6 +726,13 @@ class Importer:
                                 probe.reason, member_path.name,
                             )
                             continue
+                        if (marker := software_marker(member_path)) is not None:
+                            counts["nested_software"] += 1
+                            log.info(
+                                "nested archive is a software build (holds %s), "
+                                "skipped: %s", marker, member_path.name,
+                            )
+                            continue
                         self._collect_archive_members(
                             member_path, attribution=attribution,
                             source_row=source_row, depth=depth + 1,

frontend/src/components/settings/ArchiveReextractCard.vue

@@ -1,17 +1,15 @@
 <template>
-  <!-- #713: re-extract PostAttachments that are really archives but were filed
-       opaquely before the magic-byte gate, and attach their images to the post. -->
   <MaintenanceTile
     icon="mdi-folder-zip-outline"
     title="Re-extract archives"
-    blurb="Extract images from zip attachments filed opaquely."
+    blurb="Import the images inside archive attachments that were never extracted."
     :open="busy"
   >
       <p class="text-body-2 mb-3">
-        Some posts attached an archive (zip) whose images weren't extracted
-        because the downloaded file had no usable extension. This scans existing
-        attachments, extracts any that are really archives, and attaches their
-        images to the post. Idempotent — safe to run more than once.
+        Goes through every stored zip, rar and 7z, including ones imported from
+        a folder without a post, and imports the images inside. Archives already
+        extracted are passed over, and game or app builds stay whole. Safe to
+        run again.
       </p>
       <v-btn color="primary" rounded="pill" :loading="busy" @click="run">
         <v-icon start>mdi-folder-zip-outline</v-icon> Re-extract archives now

tests/test_importer_archive.py

@@ -269,3 +269,39 @@ def test_reimport_archive_is_idempotent(importer, import_layout):
     assert importer.session.execute(
         select(func.count()).select_from(PostAttachment)
     ).scalar_one() == 1
+
+
+def test_a_game_build_is_kept_whole_and_its_images_left_alone(importer, import_layout):
+    import_root, _ = import_layout
+    importer.settings.phash_threshold = 0
+    arc = import_root / "Bob" / "MyGame-1.0-pc.zip"
+    arc.parent.mkdir(parents=True, exist_ok=True)
+    with zipfile.ZipFile(arc, "w") as zf:
+        zf.writestr("MyGame/lib/py3-windows-x86_64/python.exe", b"MZ")
+        zf.writestr("MyGame/game/images/cg01.jpg", _split_bytes("v"))
+        zf.writestr("MyGame/game/script.rpyc", b"\x00")
+
+    r = importer.import_one(arc)
+    assert r.status == "attached"
+    assert importer.session.execute(select(func.count()).select_from(ImageRecord)).scalar_one() == 0
+    att = importer.session.execute(select(PostAttachment)).scalar_one()
+    assert att.original_filename == "MyGame-1.0-pc.zip"
+
+
+def test_a_game_build_inside_an_art_pack_is_skipped_and_the_art_imported(importer, import_layout):
+    import_root, _ = import_layout
+    importer.settings.phash_threshold = 0
+    game = io.BytesIO()
+    with zipfile.ZipFile(game, "w") as zf:
+        zf.writestr("Game_Data/sharedassets0.assets", b"\x00")
+        zf.writestr("Game_Data/sprite.jpg", _split_bytes("h"))
+    arc = import_root / "Bob" / "pack.zip"
+    arc.parent.mkdir(parents=True, exist_ok=True)
+    with zipfile.ZipFile(arc, "w") as zf:
+        zf.writestr("art.jpg", _split_bytes("v"))
+        zf.writestr("bonus/game.zip", game.getvalue())
+
+    r = importer.import_one(arc)
+    assert r.status == "imported"
+    assert len(r.member_image_ids) == 1
+    assert importer.session.execute(select(func.count()).select_from(ImageRecord)).scalar_one() == 1

tests/test_reextract_archives.py

@@ -171,3 +171,80 @@ def test_reextract_timebox_resumes_from_cursor(db_sync, tmp_path, monkeypatch):
     assert second["scanned"] == 1
     assert second["members_imported"] == 1
     assert len(db_sync.execute(select(ImageRecord)).scalars().all()) == 2
+
+
+def test_reextract_reaches_archives_without_a_post_and_passes_over_extracted_ones(
+    db_sync, tmp_path, monkeypatch,
+):
+    from backend.app.tasks import ml as ml_mod
+    from backend.app.tasks import thumbnail as thumb_mod
+
+    monkeypatch.setattr(thumb_mod.generate_thumbnail, "delay", lambda *a, **k: None)
+    monkeypatch.setattr(ml_mod.embed_image, "delay", lambda *a, **k: None)
+
+    images_root = tmp_path / "images"
+    store_dir = images_root / "attachments" / "def"
+    store_dir.mkdir(parents=True)
+    artist = Artist(name="Cleo", slug="cleo")
+    db_sync.add(artist)
+    db_sync.flush()
+
+    # A filesystem-imported archive: an artist, no post, no sidecar.
+    arc = store_dir / "20210630_Demons.zip"
+    with zipfile.ZipFile(arc, "w") as zf:
+        zf.writestr("1.jpg", _patterned_jpeg(0))
+        zf.writestr("2.jpg", _patterned_jpeg(1))
+    att = PostAttachment(
+        post_id=None, artist_id=artist.id,
+        sha256=hashlib.sha256(arc.read_bytes()).hexdigest(), path=str(arc),
+        original_filename=arc.name, ext=".zip", size_bytes=arc.stat().st_size,
+    )
+    db_sync.add(att)
+    db_sync.commit()
+
+    summary = cleanup_service.reextract_archive_attachments(db_sync, images_root=images_root)
+    assert summary["without_post"] == 1
+    assert summary["members_imported"] == 2
+    images = db_sync.execute(select(ImageRecord)).scalars().all()
+    assert {i.artist_id for i in images} == {artist.id}
+    # The members sit in a folder named after the archive, and the stored
+    # archive itself is untouched.
+    assert all("/cleo/20210630_Demons/" in i.path for i in images)
+    assert arc.is_file()
+    assert not list((images_root / "cleo" / "20210630_Demons").glob("archive.*"))
+
+
+def test_reextract_passes_over_an_archive_whose_members_are_recorded(
+    db_sync, tmp_path, monkeypatch,
+):
+    images_root = tmp_path / "images"
+    store_dir = images_root / "attachments" / "fed"
+    store_dir.mkdir(parents=True)
+    artist = Artist(name="Dee", slug="dee")
+    db_sync.add(artist)
+    db_sync.flush()
+    post = Post(artist_id=artist.id, external_post_id="1")
+    db_sync.add(post)
+    db_sync.flush()
+    arc = store_dir / "done.zip"
+    with zipfile.ZipFile(arc, "w") as zf:
+        zf.writestr("1.jpg", _patterned_jpeg(0))
+    att = PostAttachment(
+        post_id=post.id, artist_id=artist.id,
+        sha256=hashlib.sha256(arc.read_bytes()).hexdigest(), path=str(arc),
+        original_filename=arc.name, ext=".zip", size_bytes=arc.stat().st_size,
+    )
+    img = ImageRecord(path=str(images_root / "dee" / "x.jpg"), sha256="0" * 64,
+                      size_bytes=1, mime="image/jpeg", origin="downloaded")
+    db_sync.add_all([att, img])
+    db_sync.flush()
+    db_sync.add(ImageProvenance(image_record_id=img.id, post_id=post.id,
+                                from_attachment_id=att.id))
+    db_sync.commit()
+
+    called = []
+    monkeypatch.setattr(cleanup_service, "_reextract_archive_to_post",
+                        lambda *a, **k: called.append(a) or [])
+    summary = cleanup_service.reextract_archive_attachments(db_sync, images_root=images_root)
+    assert summary["already_extracted"] == 1
+    assert called == []