feat(import): recurse nested archives + precise "no images" reason (#718)
CI / lint (push) Successful in 3s
CI / frontend-build (push) Successful in 19s
CI / backend-lint-and-test (push) Successful in 25s
CI / integration (push) Failing after 3m13s

Root cause (operator-confirmed via event metadata + lsar): a "High Resolution
files" pack often wraps a per-chapter .rar/.zip INSIDE one outer archive (incase).
_import_archive only extracted one level — a nested-archive member failed
is_supported and was skipped, so the real pages were silently dropped and the post
showed "archive but no images". The disk scan found this pattern recurring across
the attachment store.

- Recurse into nested archives via _collect_archive_members: a member that is
  itself an archive is bomb-probed and extracted too, depth-capped at
  _ARCHIVE_MAX_DEPTH=3. Nested members attribute to the OUTER archive's sidecar so
  they link to the right Post. Each level is wrapped so one bad nested archive
  can't abort the import. The shared path means external (mega/gdrive) archives
  recurse too.
- Replace the catch-all "held no supported members" string with a per-outcome
  tally (media/deduped/unsupported/failed/nested/nested_rejected). The all-deduped
  case is now recognised as BENIGN — images already in the library, re-linked to
  this post via enrich-on-duplicate — and returns attached WITHOUT error, so it no
  longer false-flags in event metadata.unextracted_archives. Genuine failures
  carry the precise breakdown.

Tests: nested zip-in-cbz imports both inner images + links them to the outer post;
all-deduped archive returns attached with error=None and links images to both posts.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
2026-06-15 21:18:02 -04:00
parent 41aa8fe39e
commit 8dee2f9628
2 changed files with 172 additions and 22 deletions
+71
View File
@@ -93,6 +93,77 @@ def test_archive_imports_members_and_stores_archive(importer, import_layout):
assert att.post_id == post.id
def test_nested_archive_members_imported(importer, import_layout):
"""#718: a 'high-res' pack that wraps a nested archive must still import the
nested images and link them to the post — not silently drop them (incase's
per-chapter .rar inside an outer 'High Resolution files' pack)."""
import_root, _ = import_layout
importer.settings.phash_threshold = 0 # v/h splits are distinct only at 0
inner = io.BytesIO()
with zipfile.ZipFile(inner, "w") as zf:
zf.writestr("p1.jpg", _split_bytes("v"))
zf.writestr("p2.jpg", _split_bytes("h"))
arc = import_root / "Nessie" / "hr.cbz"
arc.parent.mkdir(parents=True, exist_ok=True)
with zipfile.ZipFile(arc, "w") as zf:
zf.writestr("chapter.zip", inner.getvalue()) # nested archive, no direct media
arc.with_suffix(".cbz.json").write_text(json.dumps(
{"category": "patreon", "id": "900", "title": "HR Pack"}))
r = importer.import_one(arc)
assert r.status == "imported"
assert len(r.member_image_ids) == 2 # both nested images surfaced
assert importer.session.execute(
select(func.count()).select_from(ImageRecord)
).scalar_one() == 2
post = importer.session.execute(select(Post)).scalar_one()
provs = importer.session.execute(
select(func.count()).select_from(ImageProvenance)
).scalar_one()
assert provs == 2 # both nested members linked to the one outer post
# The outer pack is still preserved as an attachment.
att = importer.session.execute(select(PostAttachment)).scalar_one()
assert att.original_filename == "hr.cbz"
assert att.post_id == post.id
def test_archive_all_deduped_is_benign_not_flagged(importer, import_layout):
"""#718 reason-string fix: when every image in an archive already exists
(cross-posted), the images re-link to the new post and the archive is NOT
flagged as an unextracted-archive problem (no error set)."""
import_root, _ = import_layout
importer.settings.phash_threshold = 0
pv, ph = _split_bytes("v"), _split_bytes("h")
first = import_root / "Dup" / "a.cbz"
first.parent.mkdir(parents=True, exist_ok=True)
with zipfile.ZipFile(first, "w") as zf:
zf.writestr("a.jpg", pv)
zf.writestr("b.jpg", ph)
first.with_suffix(".cbz.json").write_text(json.dumps(
{"category": "patreon", "id": "111", "title": "First"}))
assert importer.import_one(first).status == "imported"
second = import_root / "Dup" / "b.cbz" # different post, same two images
with zipfile.ZipFile(second, "w") as zf:
zf.writestr("a.jpg", pv)
zf.writestr("b.jpg", ph)
second.with_suffix(".cbz.json").write_text(json.dumps(
{"category": "patreon", "id": "222", "title": "Second"}))
r = importer.import_one(second)
assert r.status == "attached"
assert r.error is None # benign all-deduped — NOT a flagged problem
assert importer.session.execute(
select(func.count()).select_from(ImageRecord)
).scalar_one() == 2 # no new records
provs = importer.session.execute(
select(func.count()).select_from(ImageProvenance)
).scalar_one()
assert provs == 4 # 2 images × 2 posts (enrich-on-duplicate)
def test_corrupt_archive_still_stored(importer, import_layout):
import_root, _ = import_layout
arc = import_root / "Bob" / "broken.zip"