feat: archive images reach the library: game builds stay whole, and re-extract covers every archive
CI and images / lint (push) Successful in 2s
CI and images / extension-version (push) Successful in 2s
CI and images / extension-test (push) Successful in 21s
CI and images / frontend-build (push) Successful in 21s
CI and images / backend-lint-and-test (push) Successful in 45s
CI and images / integration (push) Successful in 2m31s
CI and images / sign-extension (push) Successful in 2s
CI and images / build-agent (push) Successful in 4s
CI and images / build-web (push) Successful in 2m1s
CI and images / smoke-web (push) Successful in 49s
CI and images / promote (push) Successful in 1s
CI and images / refresh-dev (push) Skipped
CI and images / unattended-failure (push) Skipped
CI and images / lint (push) Successful in 2s
CI and images / extension-version (push) Successful in 2s
CI and images / extension-test (push) Successful in 21s
CI and images / frontend-build (push) Successful in 21s
CI and images / backend-lint-and-test (push) Successful in 45s
CI and images / integration (push) Successful in 2m31s
CI and images / sign-extension (push) Successful in 2s
CI and images / build-agent (push) Successful in 4s
CI and images / build-web (push) Successful in 2m1s
CI and images / smoke-web (push) Successful in 49s
CI and images / promote (push) Successful in 1s
CI and images / refresh-dev (push) Skipped
CI and images / unattended-failure (push) Skipped
Most archive attachments never had their images imported. The re-extract action skipped archives without a post, and nothing kept game builds out of an extraction. - An archive holding executables, libraries, or game-engine data (Ren'Py, Unity, Godot, RPG Maker, Python builds) is kept as an attachment and not extracted. The same holds for one nested inside an art pack. - Re-extract now reaches archives imported from a folder without a post. Their images go to the artist, in a folder named after the archive. - Re-extract passes over archives whose members are already recorded. - Re-extract stages each archive by symlink instead of copying it. - The card says what the action does now. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -269,3 +269,39 @@ def test_reimport_archive_is_idempotent(importer, import_layout):
|
||||
assert importer.session.execute(
|
||||
select(func.count()).select_from(PostAttachment)
|
||||
).scalar_one() == 1
|
||||
|
||||
|
||||
def test_a_game_build_is_kept_whole_and_its_images_left_alone(importer, import_layout):
|
||||
import_root, _ = import_layout
|
||||
importer.settings.phash_threshold = 0
|
||||
arc = import_root / "Bob" / "MyGame-1.0-pc.zip"
|
||||
arc.parent.mkdir(parents=True, exist_ok=True)
|
||||
with zipfile.ZipFile(arc, "w") as zf:
|
||||
zf.writestr("MyGame/lib/py3-windows-x86_64/python.exe", b"MZ")
|
||||
zf.writestr("MyGame/game/images/cg01.jpg", _split_bytes("v"))
|
||||
zf.writestr("MyGame/game/script.rpyc", b"\x00")
|
||||
|
||||
r = importer.import_one(arc)
|
||||
assert r.status == "attached"
|
||||
assert importer.session.execute(select(func.count()).select_from(ImageRecord)).scalar_one() == 0
|
||||
att = importer.session.execute(select(PostAttachment)).scalar_one()
|
||||
assert att.original_filename == "MyGame-1.0-pc.zip"
|
||||
|
||||
|
||||
def test_a_game_build_inside_an_art_pack_is_skipped_and_the_art_imported(importer, import_layout):
|
||||
import_root, _ = import_layout
|
||||
importer.settings.phash_threshold = 0
|
||||
game = io.BytesIO()
|
||||
with zipfile.ZipFile(game, "w") as zf:
|
||||
zf.writestr("Game_Data/sharedassets0.assets", b"\x00")
|
||||
zf.writestr("Game_Data/sprite.jpg", _split_bytes("h"))
|
||||
arc = import_root / "Bob" / "pack.zip"
|
||||
arc.parent.mkdir(parents=True, exist_ok=True)
|
||||
with zipfile.ZipFile(arc, "w") as zf:
|
||||
zf.writestr("art.jpg", _split_bytes("v"))
|
||||
zf.writestr("bonus/game.zip", game.getvalue())
|
||||
|
||||
r = importer.import_one(arc)
|
||||
assert r.status == "imported"
|
||||
assert len(r.member_image_ids) == 1
|
||||
assert importer.session.execute(select(func.count()).select_from(ImageRecord)).scalar_one() == 1
|
||||
|
||||
@@ -171,3 +171,80 @@ def test_reextract_timebox_resumes_from_cursor(db_sync, tmp_path, monkeypatch):
|
||||
assert second["scanned"] == 1
|
||||
assert second["members_imported"] == 1
|
||||
assert len(db_sync.execute(select(ImageRecord)).scalars().all()) == 2
|
||||
|
||||
|
||||
def test_reextract_reaches_archives_without_a_post_and_passes_over_extracted_ones(
|
||||
db_sync, tmp_path, monkeypatch,
|
||||
):
|
||||
from backend.app.tasks import ml as ml_mod
|
||||
from backend.app.tasks import thumbnail as thumb_mod
|
||||
|
||||
monkeypatch.setattr(thumb_mod.generate_thumbnail, "delay", lambda *a, **k: None)
|
||||
monkeypatch.setattr(ml_mod.embed_image, "delay", lambda *a, **k: None)
|
||||
|
||||
images_root = tmp_path / "images"
|
||||
store_dir = images_root / "attachments" / "def"
|
||||
store_dir.mkdir(parents=True)
|
||||
artist = Artist(name="Cleo", slug="cleo")
|
||||
db_sync.add(artist)
|
||||
db_sync.flush()
|
||||
|
||||
# A filesystem-imported archive: an artist, no post, no sidecar.
|
||||
arc = store_dir / "20210630_Demons.zip"
|
||||
with zipfile.ZipFile(arc, "w") as zf:
|
||||
zf.writestr("1.jpg", _patterned_jpeg(0))
|
||||
zf.writestr("2.jpg", _patterned_jpeg(1))
|
||||
att = PostAttachment(
|
||||
post_id=None, artist_id=artist.id,
|
||||
sha256=hashlib.sha256(arc.read_bytes()).hexdigest(), path=str(arc),
|
||||
original_filename=arc.name, ext=".zip", size_bytes=arc.stat().st_size,
|
||||
)
|
||||
db_sync.add(att)
|
||||
db_sync.commit()
|
||||
|
||||
summary = cleanup_service.reextract_archive_attachments(db_sync, images_root=images_root)
|
||||
assert summary["without_post"] == 1
|
||||
assert summary["members_imported"] == 2
|
||||
images = db_sync.execute(select(ImageRecord)).scalars().all()
|
||||
assert {i.artist_id for i in images} == {artist.id}
|
||||
# The members sit in a folder named after the archive, and the stored
|
||||
# archive itself is untouched.
|
||||
assert all("/cleo/20210630_Demons/" in i.path for i in images)
|
||||
assert arc.is_file()
|
||||
assert not list((images_root / "cleo" / "20210630_Demons").glob("archive.*"))
|
||||
|
||||
|
||||
def test_reextract_passes_over_an_archive_whose_members_are_recorded(
|
||||
db_sync, tmp_path, monkeypatch,
|
||||
):
|
||||
images_root = tmp_path / "images"
|
||||
store_dir = images_root / "attachments" / "fed"
|
||||
store_dir.mkdir(parents=True)
|
||||
artist = Artist(name="Dee", slug="dee")
|
||||
db_sync.add(artist)
|
||||
db_sync.flush()
|
||||
post = Post(artist_id=artist.id, external_post_id="1")
|
||||
db_sync.add(post)
|
||||
db_sync.flush()
|
||||
arc = store_dir / "done.zip"
|
||||
with zipfile.ZipFile(arc, "w") as zf:
|
||||
zf.writestr("1.jpg", _patterned_jpeg(0))
|
||||
att = PostAttachment(
|
||||
post_id=post.id, artist_id=artist.id,
|
||||
sha256=hashlib.sha256(arc.read_bytes()).hexdigest(), path=str(arc),
|
||||
original_filename=arc.name, ext=".zip", size_bytes=arc.stat().st_size,
|
||||
)
|
||||
img = ImageRecord(path=str(images_root / "dee" / "x.jpg"), sha256="0" * 64,
|
||||
size_bytes=1, mime="image/jpeg", origin="downloaded")
|
||||
db_sync.add_all([att, img])
|
||||
db_sync.flush()
|
||||
db_sync.add(ImageProvenance(image_record_id=img.id, post_id=post.id,
|
||||
from_attachment_id=att.id))
|
||||
db_sync.commit()
|
||||
|
||||
called = []
|
||||
monkeypatch.setattr(cleanup_service, "_reextract_archive_to_post",
|
||||
lambda *a, **k: called.append(a) or [])
|
||||
summary = cleanup_service.reextract_archive_attachments(db_sync, images_root=images_root)
|
||||
assert summary["already_extracted"] == 1
|
||||
assert called == []
|
||||
|
||||
Reference in New Issue
Block a user