feat(import): recurse nested archives + precise "no images" reason (#718)
Root cause (operator-confirmed via event metadata + lsar): a "High Resolution files" pack often wraps a per-chapter .rar/.zip INSIDE one outer archive (incase). _import_archive only extracted one level — a nested-archive member failed is_supported and was skipped, so the real pages were silently dropped and the post showed "archive but no images". The disk scan found this pattern recurring across the attachment store. - Recurse into nested archives via _collect_archive_members: a member that is itself an archive is bomb-probed and extracted too, depth-capped at _ARCHIVE_MAX_DEPTH=3. Nested members attribute to the OUTER archive's sidecar so they link to the right Post. Each level is wrapped so one bad nested archive can't abort the import. The shared path means external (mega/gdrive) archives recurse too. - Replace the catch-all "held no supported members" string with a per-outcome tally (media/deduped/unsupported/failed/nested/nested_rejected). The all-deduped case is now recognised as BENIGN — images already in the library, re-linked to this post via enrich-on-duplicate — and returns attached WITHOUT error, so it no longer false-flags in event metadata.unextracted_archives. Genuine failures carry the precise breakdown. Tests: nested zip-in-cbz imports both inner images + links them to the outer post; all-deduped archive returns attached with error=None and links images to both posts. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
@@ -93,6 +93,77 @@ def test_archive_imports_members_and_stores_archive(importer, import_layout):
|
||||
assert att.post_id == post.id
|
||||
|
||||
|
||||
def test_nested_archive_members_imported(importer, import_layout):
|
||||
"""#718: a 'high-res' pack that wraps a nested archive must still import the
|
||||
nested images and link them to the post — not silently drop them (incase's
|
||||
per-chapter .rar inside an outer 'High Resolution files' pack)."""
|
||||
import_root, _ = import_layout
|
||||
importer.settings.phash_threshold = 0 # v/h splits are distinct only at 0
|
||||
inner = io.BytesIO()
|
||||
with zipfile.ZipFile(inner, "w") as zf:
|
||||
zf.writestr("p1.jpg", _split_bytes("v"))
|
||||
zf.writestr("p2.jpg", _split_bytes("h"))
|
||||
arc = import_root / "Nessie" / "hr.cbz"
|
||||
arc.parent.mkdir(parents=True, exist_ok=True)
|
||||
with zipfile.ZipFile(arc, "w") as zf:
|
||||
zf.writestr("chapter.zip", inner.getvalue()) # nested archive, no direct media
|
||||
arc.with_suffix(".cbz.json").write_text(json.dumps(
|
||||
{"category": "patreon", "id": "900", "title": "HR Pack"}))
|
||||
|
||||
r = importer.import_one(arc)
|
||||
assert r.status == "imported"
|
||||
assert len(r.member_image_ids) == 2 # both nested images surfaced
|
||||
|
||||
assert importer.session.execute(
|
||||
select(func.count()).select_from(ImageRecord)
|
||||
).scalar_one() == 2
|
||||
post = importer.session.execute(select(Post)).scalar_one()
|
||||
provs = importer.session.execute(
|
||||
select(func.count()).select_from(ImageProvenance)
|
||||
).scalar_one()
|
||||
assert provs == 2 # both nested members linked to the one outer post
|
||||
# The outer pack is still preserved as an attachment.
|
||||
att = importer.session.execute(select(PostAttachment)).scalar_one()
|
||||
assert att.original_filename == "hr.cbz"
|
||||
assert att.post_id == post.id
|
||||
|
||||
|
||||
def test_archive_all_deduped_is_benign_not_flagged(importer, import_layout):
|
||||
"""#718 reason-string fix: when every image in an archive already exists
|
||||
(cross-posted), the images re-link to the new post and the archive is NOT
|
||||
flagged as an unextracted-archive problem (no error set)."""
|
||||
import_root, _ = import_layout
|
||||
importer.settings.phash_threshold = 0
|
||||
pv, ph = _split_bytes("v"), _split_bytes("h")
|
||||
first = import_root / "Dup" / "a.cbz"
|
||||
first.parent.mkdir(parents=True, exist_ok=True)
|
||||
with zipfile.ZipFile(first, "w") as zf:
|
||||
zf.writestr("a.jpg", pv)
|
||||
zf.writestr("b.jpg", ph)
|
||||
first.with_suffix(".cbz.json").write_text(json.dumps(
|
||||
{"category": "patreon", "id": "111", "title": "First"}))
|
||||
assert importer.import_one(first).status == "imported"
|
||||
|
||||
second = import_root / "Dup" / "b.cbz" # different post, same two images
|
||||
with zipfile.ZipFile(second, "w") as zf:
|
||||
zf.writestr("a.jpg", pv)
|
||||
zf.writestr("b.jpg", ph)
|
||||
second.with_suffix(".cbz.json").write_text(json.dumps(
|
||||
{"category": "patreon", "id": "222", "title": "Second"}))
|
||||
|
||||
r = importer.import_one(second)
|
||||
assert r.status == "attached"
|
||||
assert r.error is None # benign all-deduped — NOT a flagged problem
|
||||
|
||||
assert importer.session.execute(
|
||||
select(func.count()).select_from(ImageRecord)
|
||||
).scalar_one() == 2 # no new records
|
||||
provs = importer.session.execute(
|
||||
select(func.count()).select_from(ImageProvenance)
|
||||
).scalar_one()
|
||||
assert provs == 4 # 2 images × 2 posts (enrich-on-duplicate)
|
||||
|
||||
|
||||
def test_corrupt_archive_still_stored(importer, import_layout):
|
||||
import_root, _ = import_layout
|
||||
arc = import_root / "Bob" / "broken.zip"
|
||||
|
||||
Reference in New Issue
Block a user