From c2af70e6aff31091bfea53dc0759064388428de6 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Sat, 7 Nov 2026 23:27:36 -0500 Subject: [PATCH] feat: archive images reach the library: game builds stay whole, and re-extract covers every archive Most archive attachments never had their images imported. The re-extract action skipped archives without a post, and nothing kept game builds out of an extraction. - An archive holding executables, libraries, or game-engine data (Ren'Py, Unity, Godot, RPG Maker, Python builds) is kept as an attachment and not extracted. The same holds for one nested inside an art pack. - Re-extract now reaches archives imported from a folder without a post. Their images go to the artist, in a folder named after the archive. - Re-extract passes over archives whose members are already recorded. - Re-extract stages each archive by symlink instead of copying it. - The card says what the action does now. Co-Authored-By: Claude Opus 5.5 --- backend/app/services/archive_extractor.py | 44 +++++++ backend/app/services/cleanup_service.py | 108 ++++++++++++------ backend/app/services/importer.py | 30 +++-- .../settings/ArchiveReextractCard.vue | 12 +- tests/test_importer_archive.py | 36 ++++++ tests/test_reextract_archives.py | 77 +++++++++++++ 6 files changed, 259 insertions(+), 48 deletions(-) diff --git a/backend/app/services/archive_extractor.py b/backend/app/services/archive_extractor.py index d6b6566..cf8bc10 100644 --- a/backend/app/services/archive_extractor.py +++ b/backend/app/services/archive_extractor.py @@ -57,6 +57,50 @@ def is_archive(path: Path) -> bool: return detect_archive_format(path) is not None +# Files that only a program ships: executables and libraries, and the packed +# data and compiled scripts of the common game engines (Ren'Py, Unity, Godot, +# RPG Maker, Python builds). An archive holding any of them is a game or app +# build. Its images are sprites, UI and CGs bundled for the program, not art +# for the library, so the archive is kept whole as an attachment. +SOFTWARE_EXTS = frozenset({ + ".exe", ".msi", ".dll", ".so", ".dylib", ".apk", + ".rpa", ".rpyc", ".rpy", ".pyc", ".pyo", + ".pak", ".pck", ".unity3d", ".assets", ".ress", + ".rgssad", ".rgss2a", ".rgss3a", ".rxdata", ".rvdata", ".rvdata2", +}) + + +def member_names(path: Path) -> list[str]: + """Every member name in the archive, read from its directory without + decompressing anything. Empty when the archive can't be listed.""" + try: + fmt = detect_archive_format(path) + if fmt == "zip": + with zipfile.ZipFile(path) as zf: + return zf.namelist() + if fmt == "rar": + import rarfile + + with rarfile.RarFile(path) as rf: + return rf.namelist() + if fmt == "7z": + import py7zr + + with py7zr.SevenZipFile(path, "r") as zf: + return zf.getnames() + except Exception as exc: + log.warning("archive listing failed for %s: %s", path, exc) + return [] + + +def software_marker(path: Path) -> str | None: + """The first member that marks the archive as a software build, or None.""" + for name in member_names(path): + if Path(name).suffix.lower() in SOFTWARE_EXTS: + return name + return None + + @contextmanager def extract_archive(path: Path): """Context manager yielding a list of (member_name, extracted_path) diff --git a/backend/app/services/cleanup_service.py b/backend/app/services/cleanup_service.py index 89afd51..bbf2087 100644 --- a/backend/app/services/cleanup_service.py +++ b/backend/app/services/cleanup_service.py @@ -1044,45 +1044,75 @@ def cancel_audit_run(session: Session, *, audit_id: int) -> None: _ARCHIVE_EXT_FOR_FORMAT = {"zip": ".zip", "rar": ".rar", "7z": ".7z"} +def _stage_archive(archive_path: Path, staged: Path) -> None: + """Put the stored archive at `staged` for the importer to read: a symlink, + so a multi-gigabyte archive isn't copied, or a copy where links aren't + possible.""" + import shutil + + staged.unlink(missing_ok=True) + try: + staged.symlink_to(archive_path) + except OSError: + shutil.copy2(archive_path, staged) + + +def _archive_folder_name(original_filename: str | None, attachment_id: int) -> str: + """The library folder a post-less archive's members land in: the archive's + own name without its extension, made safe for a path.""" + stem = Path(original_filename or "").stem.strip().strip(".") + stem = "".join(c if c not in '/\\:*?"<>|' else "_" for c in stem)[:120].strip() + return stem or f"archive-{attachment_id}" + + def _reextract_archive_to_post( importer, archive_path: Path, post, source_row, artist, images_root: Path, + *, folder: str = "", ) -> list[int]: - """Extract one stored archive and link its members to `post`. + """Extract one stored archive and import its members. The stored attachment has no adjacent sidecar (it lives in the sha-addressed - attachment store). Stage a copy + a reconstructed sidecar UNDER the artist's - library dir (`images_root////`) — the importer - re-derives the artist from the path AND copies members relative to it, so the - members land in the real library and resolve to the right artist — then re-run - `attach_in_place`: the archive extracts and `find_or_create_post` re-attaches - the members to the SAME Post (source_id + external_post_id). Removes only the - staged archive + sidecar afterward; the imported member files stay. Returns - the new member image ids. + attachment store), so the archive is staged under the artist's library dir + and `attach_in_place` re-run on it: the importer derives the artist from the + path and copies members relative to it, so they land in the real library. + + - With a `post`, the stage is `images_root////` beside + a reconstructed sidecar, so `find_or_create_post` re-attaches the members + to that same Post (source_id + external_post_id). + - Without one (a filesystem import that had no sidecar), the stage is + `images_root///` with no sidecar, and the members belong to + the artist alone, as a filesystem import of the archive would have left + them. + + Only the staged archive and sidecar are removed afterward; the imported + member files stay. Returns the new member image ids. """ import json - import shutil from .archive_extractor import detect_archive_format fmt = detect_archive_format(archive_path) ext = _ARCHIVE_EXT_FOR_FORMAT.get(fmt or "", ".zip") - platform = source_row.platform if source_row is not None else "imported" - sidecar = { - "category": source_row.platform if source_row is not None - else (post.raw_metadata or {}).get("category"), - "id": post.external_post_id, - "title": post.post_title or "", - "content": post.description or "", - "published_at": post.post_date.isoformat() if post.post_date else None, - "url": post.post_url, - } - work = images_root / artist.slug / platform / str(post.external_post_id) + if post is not None: + platform = source_row.platform if source_row is not None else "imported" + work = images_root / artist.slug / platform / str(post.external_post_id) + else: + work = images_root / artist.slug / folder work.mkdir(parents=True, exist_ok=True) staged = work / f"archive{ext}" # clean ext → is_archive + find_sidecar sidecar_path = staged.with_suffix(".json") try: - shutil.copy2(archive_path, staged) - sidecar_path.write_text(json.dumps(sidecar)) + _stage_archive(archive_path, staged) + if post is not None: + sidecar_path.write_text(json.dumps({ + "category": source_row.platform if source_row is not None + else (post.raw_metadata or {}).get("category"), + "id": post.external_post_id, + "title": post.post_title or "", + "content": post.description or "", + "published_at": post.post_date.isoformat() if post.post_date else None, + "url": post.post_url, + })) res = importer.attach_in_place(staged, artist=artist, source=source_row) return list(res.member_image_ids or []) finally: @@ -1124,8 +1154,9 @@ def reextract_archive_attachments( summary = { "scanned": 0, "archives": 0, "members_imported": 0, - "posts_touched": 0, "skipped_no_post": 0, "skipped_no_artist": 0, - "errors": 0, "partial": False, "resume_after_id": after_id, + "posts_touched": 0, "without_post": 0, "already_extracted": 0, + "skipped_no_artist": 0, "errors": 0, "partial": False, + "resume_after_id": after_id, } settings = ImportSettings.load_sync(session) importer = Importer( @@ -1150,15 +1181,17 @@ def reextract_archive_attachments( except OSError: continue summary["archives"] += 1 - if att.post_id is None: - summary["skipped_no_post"] += 1 - continue - post = session.get(Post, att.post_id) - if post is None: - summary["skipped_no_post"] += 1 + # Members already recorded as coming from this archive: done before. + if session.execute( + select(ImageProvenance.id) + .where(ImageProvenance.from_attachment_id == att.id) + .limit(1) + ).first() is not None: + summary["already_extracted"] += 1 continue + post = session.get(Post, att.post_id) if att.post_id else None artist = session.get(Artist, att.artist_id) if att.artist_id else None - if artist is None and post.artist_id: + if artist is None and post is not None and post.artist_id: artist = session.get(Artist, post.artist_id) if artist is None or not artist.slug: # The importer re-derives the artist from the staged path, so we need @@ -1166,10 +1199,16 @@ def reextract_archive_attachments( # subscription posts; skip rather than orphan the members.) summary["skipped_no_artist"] += 1 continue - source_row = session.get(Source, post.source_id) if post.source_id else None + source_row = ( + session.get(Source, post.source_id) + if post is not None and post.source_id else None + ) + if post is None: + summary["without_post"] += 1 try: ids = _reextract_archive_to_post( importer, stored, post, source_row, artist, images_root, + folder=_archive_folder_name(att.original_filename, att.id), ) session.commit() except Exception as exc: # one bad archive must not strand the rest @@ -1179,7 +1218,8 @@ def reextract_archive_attachments( continue if ids: summary["members_imported"] += len(ids) - summary["posts_touched"] += 1 + if post is not None: + summary["posts_touched"] += 1 enqueue_ids.extend(ids) # Time-box the chunk. resume_after_id already points at this attachment, diff --git a/backend/app/services/importer.py b/backend/app/services/importer.py index 4114cca..24d73bd 100644 --- a/backend/app/services/importer.py +++ b/backend/app/services/importer.py @@ -45,7 +45,7 @@ from ..utils.phash import compute_phash, find_similar, fingerprint_path, fingerp from ..utils.sidecar import find_sidecar, parse_sidecar from ..utils.slug import slugify from ..utils.staging import is_temp_staging_file -from .archive_extractor import extract_archive, is_archive +from .archive_extractor import extract_archive, is_archive, software_marker from .attachment_store import AttachmentStore from .audits import single_color from .link_extract import extract_external_links @@ -616,6 +616,14 @@ class Importer: artist_use = artist if artist is not None else self._resolve_artist(source) post = self._post_for_sidecar(source, artist_use, source_row=source_row) + if (marker := software_marker(source)) is not None: + # A game or app build: kept whole, its bundled images left alone. + self._capture_attachment( + source, post=post, artist=artist_use, resolved=True, + ) + log.info("%s: a software build (holds %s), kept unextracted", + source.name, marker) + return ImportResult(status="attached") # Committed before extraction, so a member that fails (and is rolled # back on its own) can never take the archive's post with it. self.session.commit() @@ -631,7 +639,7 @@ class Importer: # or failed/corrupt members — instead of one catch-all string. counts = { "media": 0, "deduped": 0, "unsupported": 0, - "failed": 0, "nested": 0, "nested_rejected": 0, + "failed": 0, "nested": 0, "nested_rejected": 0, "nested_software": 0, } self._collect_archive_members( source, attribution=source, source_row=source_row, @@ -663,20 +671,21 @@ class Importer: if counts["media"] == 0 and counts["nested"] == 0: reason = ("archive yielded no members (unsupported/corrupt, or the " "extractor backend failed)") - elif counts["deduped"] and not ( + elif (counts["deduped"] or counts["nested_software"]) and not ( counts["unsupported"] or counts["failed"] or counts["nested_rejected"] ): log.info( - "%s: all %d image member(s) already in library (deduped + linked " - "to this post) — archive preserved, no new import", - source.name, counts["deduped"], + "%s: no new images — %d already in the library (linked to this " + "post), %d nested software build(s) skipped; archive preserved", + source.name, counts["deduped"], counts["nested_software"], ) return ImportResult(status="attached") else: reason = ( f"no new images — media={counts['media']} deduped={counts['deduped']} " f"unsupported/non-media={counts['unsupported']} failed={counts['failed']} " - f"nested={counts['nested']} nested-rejected={counts['nested_rejected']}" + f"nested={counts['nested']} nested-rejected={counts['nested_rejected']} " + f"nested-software={counts['nested_software']}" ) log.warning("%s: %s", source.name, reason) return ImportResult(status="attached", error=reason) @@ -717,6 +726,13 @@ class Importer: probe.reason, member_path.name, ) continue + if (marker := software_marker(member_path)) is not None: + counts["nested_software"] += 1 + log.info( + "nested archive is a software build (holds %s), " + "skipped: %s", marker, member_path.name, + ) + continue self._collect_archive_members( member_path, attribution=attribution, source_row=source_row, depth=depth + 1, diff --git a/frontend/src/components/settings/ArchiveReextractCard.vue b/frontend/src/components/settings/ArchiveReextractCard.vue index a69a083..a1645ee 100644 --- a/frontend/src/components/settings/ArchiveReextractCard.vue +++ b/frontend/src/components/settings/ArchiveReextractCard.vue @@ -1,17 +1,15 @@