feat: archive images reach the library: game builds stay whole, and re-extract covers every archive
CI and images / lint (push) Successful in 2s
CI and images / extension-version (push) Successful in 2s
CI and images / extension-test (push) Successful in 21s
CI and images / frontend-build (push) Successful in 21s
CI and images / backend-lint-and-test (push) Successful in 45s
CI and images / integration (push) Successful in 2m31s
CI and images / sign-extension (push) Successful in 2s
CI and images / build-agent (push) Successful in 4s
CI and images / build-web (push) Successful in 2m1s
CI and images / smoke-web (push) Successful in 49s
CI and images / promote (push) Successful in 1s
CI and images / refresh-dev (push) Skipped
CI and images / unattended-failure (push) Skipped

Most archive attachments never had their images imported. The re-extract
action skipped archives without a post, and nothing kept game builds out
of an extraction.

- An archive holding executables, libraries, or game-engine data
  (Ren'Py, Unity, Godot, RPG Maker, Python builds) is kept as an
  attachment and not extracted. The same holds for one nested inside an
  art pack.
- Re-extract now reaches archives imported from a folder without a post.
  Their images go to the artist, in a folder named after the archive.
- Re-extract passes over archives whose members are already recorded.
- Re-extract stages each archive by symlink instead of copying it.
- The card says what the action does now.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
2026-11-07 23:27:36 -05:00
co-authored by Claude Opus 5.5
parent f22d284b74
commit c2af70e6af
6 changed files with 259 additions and 48 deletions
+36
View File
@@ -269,3 +269,39 @@ def test_reimport_archive_is_idempotent(importer, import_layout):
assert importer.session.execute(
select(func.count()).select_from(PostAttachment)
).scalar_one() == 1
def test_a_game_build_is_kept_whole_and_its_images_left_alone(importer, import_layout):
import_root, _ = import_layout
importer.settings.phash_threshold = 0
arc = import_root / "Bob" / "MyGame-1.0-pc.zip"
arc.parent.mkdir(parents=True, exist_ok=True)
with zipfile.ZipFile(arc, "w") as zf:
zf.writestr("MyGame/lib/py3-windows-x86_64/python.exe", b"MZ")
zf.writestr("MyGame/game/images/cg01.jpg", _split_bytes("v"))
zf.writestr("MyGame/game/script.rpyc", b"\x00")
r = importer.import_one(arc)
assert r.status == "attached"
assert importer.session.execute(select(func.count()).select_from(ImageRecord)).scalar_one() == 0
att = importer.session.execute(select(PostAttachment)).scalar_one()
assert att.original_filename == "MyGame-1.0-pc.zip"
def test_a_game_build_inside_an_art_pack_is_skipped_and_the_art_imported(importer, import_layout):
import_root, _ = import_layout
importer.settings.phash_threshold = 0
game = io.BytesIO()
with zipfile.ZipFile(game, "w") as zf:
zf.writestr("Game_Data/sharedassets0.assets", b"\x00")
zf.writestr("Game_Data/sprite.jpg", _split_bytes("h"))
arc = import_root / "Bob" / "pack.zip"
arc.parent.mkdir(parents=True, exist_ok=True)
with zipfile.ZipFile(arc, "w") as zf:
zf.writestr("art.jpg", _split_bytes("v"))
zf.writestr("bonus/game.zip", game.getvalue())
r = importer.import_one(arc)
assert r.status == "imported"
assert len(r.member_image_ids) == 1
assert importer.session.execute(select(func.count()).select_from(ImageRecord)).scalar_one() == 1
+77
View File
@@ -171,3 +171,80 @@ def test_reextract_timebox_resumes_from_cursor(db_sync, tmp_path, monkeypatch):
assert second["scanned"] == 1
assert second["members_imported"] == 1
assert len(db_sync.execute(select(ImageRecord)).scalars().all()) == 2
def test_reextract_reaches_archives_without_a_post_and_passes_over_extracted_ones(
db_sync, tmp_path, monkeypatch,
):
from backend.app.tasks import ml as ml_mod
from backend.app.tasks import thumbnail as thumb_mod
monkeypatch.setattr(thumb_mod.generate_thumbnail, "delay", lambda *a, **k: None)
monkeypatch.setattr(ml_mod.embed_image, "delay", lambda *a, **k: None)
images_root = tmp_path / "images"
store_dir = images_root / "attachments" / "def"
store_dir.mkdir(parents=True)
artist = Artist(name="Cleo", slug="cleo")
db_sync.add(artist)
db_sync.flush()
# A filesystem-imported archive: an artist, no post, no sidecar.
arc = store_dir / "20210630_Demons.zip"
with zipfile.ZipFile(arc, "w") as zf:
zf.writestr("1.jpg", _patterned_jpeg(0))
zf.writestr("2.jpg", _patterned_jpeg(1))
att = PostAttachment(
post_id=None, artist_id=artist.id,
sha256=hashlib.sha256(arc.read_bytes()).hexdigest(), path=str(arc),
original_filename=arc.name, ext=".zip", size_bytes=arc.stat().st_size,
)
db_sync.add(att)
db_sync.commit()
summary = cleanup_service.reextract_archive_attachments(db_sync, images_root=images_root)
assert summary["without_post"] == 1
assert summary["members_imported"] == 2
images = db_sync.execute(select(ImageRecord)).scalars().all()
assert {i.artist_id for i in images} == {artist.id}
# The members sit in a folder named after the archive, and the stored
# archive itself is untouched.
assert all("/cleo/20210630_Demons/" in i.path for i in images)
assert arc.is_file()
assert not list((images_root / "cleo" / "20210630_Demons").glob("archive.*"))
def test_reextract_passes_over_an_archive_whose_members_are_recorded(
db_sync, tmp_path, monkeypatch,
):
images_root = tmp_path / "images"
store_dir = images_root / "attachments" / "fed"
store_dir.mkdir(parents=True)
artist = Artist(name="Dee", slug="dee")
db_sync.add(artist)
db_sync.flush()
post = Post(artist_id=artist.id, external_post_id="1")
db_sync.add(post)
db_sync.flush()
arc = store_dir / "done.zip"
with zipfile.ZipFile(arc, "w") as zf:
zf.writestr("1.jpg", _patterned_jpeg(0))
att = PostAttachment(
post_id=post.id, artist_id=artist.id,
sha256=hashlib.sha256(arc.read_bytes()).hexdigest(), path=str(arc),
original_filename=arc.name, ext=".zip", size_bytes=arc.stat().st_size,
)
img = ImageRecord(path=str(images_root / "dee" / "x.jpg"), sha256="0" * 64,
size_bytes=1, mime="image/jpeg", origin="downloaded")
db_sync.add_all([att, img])
db_sync.flush()
db_sync.add(ImageProvenance(image_record_id=img.id, post_id=post.id,
from_attachment_id=att.id))
db_sync.commit()
called = []
monkeypatch.setattr(cleanup_service, "_reextract_archive_to_post",
lambda *a, **k: called.append(a) or [])
summary = cleanup_service.reextract_archive_attachments(db_sync, images_root=images_root)
assert summary["already_extracted"] == 1
assert called == []