bvandeusen
5269cd0709
Images pulled out of a .zip/.rar previously kept no record of WHICH archive
they came from — the member->archive link was computed during extraction and
discarded, leaving only image->post. So the provenance modal could only scope
attachments to the whole post, showing every archive a 'High Resolution Files'
bundle carried instead of the one a given file lives in.
- ImageProvenance.from_attachment_id: nullable FK -> post_attachment.id
(SET NULL), migration 0055.
- importer: _import_archive stamps from_attachment_id on every member's
provenance row for the post (new + superseded + deduped members), resolving
the archive's own PostAttachment by (post, sha). Post-pass UPDATE, NULL-only
and idempotent, so it doesn't touch the dedup/supersede branches and the
backfill is safe to re-run. Nested members link to the outer stored archive.
- provenance_service.for_image: when the originating post's provenance row
records from_attachment_id, return ONLY that archive; else fall back to the
primary-post scoping from 068def2.
- ProvenancePanel: heading pluralizes ('Attachment' for a single file).
- Backfill: re-running reextract_archive_attachments (ArchiveReextractCard)
routes through _import_archive and stamps existing rows — no new code.
Tests: capture stamps on fresh import, nested-archive attribution, per-post
archive on dedup; for_image filters to the containing archive; reextract
backfill stamps the link.
Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
272 lines
11 KiB
Python
272 lines
11 KiB
Python
"""FC-2d-iii: archive → import media members + store the archive."""
|
||
|
||
import io
|
||
import json
|
||
import zipfile
|
||
|
||
import pytest
|
||
from PIL import Image
|
||
from sqlalchemy import func, select
|
||
|
||
from backend.app.models import (
|
||
ImageProvenance,
|
||
ImageRecord,
|
||
ImportSettings,
|
||
Post,
|
||
PostAttachment,
|
||
)
|
||
from backend.app.services.importer import Importer
|
||
from backend.app.services.thumbnailer import Thumbnailer
|
||
|
||
pytestmark = pytest.mark.integration
|
||
|
||
|
||
@pytest.fixture
|
||
def import_layout(tmp_path):
|
||
import_root = tmp_path / "import"
|
||
images_root = tmp_path / "images"
|
||
import_root.mkdir()
|
||
images_root.mkdir()
|
||
return import_root, images_root
|
||
|
||
|
||
@pytest.fixture
|
||
def importer(db_sync, import_layout):
|
||
import_root, images_root = import_layout
|
||
settings = db_sync.execute(
|
||
select(ImportSettings).where(ImportSettings.id == 1)
|
||
).scalar_one()
|
||
return Importer(
|
||
session=db_sync, images_root=images_root, import_root=import_root,
|
||
thumbnailer=Thumbnailer(images_root=images_root), settings=settings,
|
||
)
|
||
|
||
|
||
def _jpeg_bytes(color):
|
||
buf = io.BytesIO()
|
||
Image.new("RGB", (40, 40), color).save(buf, "JPEG")
|
||
return buf.getvalue()
|
||
|
||
|
||
def _split_bytes(orient):
|
||
"""Structured half/half image — solid colors phash-collapse (distance
|
||
0); orthogonal splits are distinct only at phash_threshold=0 (see
|
||
reference-phash-test-images)."""
|
||
im = Image.new("L", (256, 256), 0)
|
||
px = im.load()
|
||
for y in range(256):
|
||
for x in range(256):
|
||
if (x / 256 if orient == "v" else y / 256) >= 0.5:
|
||
px[x, y] = 255
|
||
buf = io.BytesIO()
|
||
im.convert("RGB").save(buf, "JPEG")
|
||
return buf.getvalue()
|
||
|
||
|
||
def test_archive_imports_members_and_stores_archive(importer, import_layout):
|
||
import_root, _ = import_layout
|
||
# Distinct structure + threshold 0 so BOTH members import (solid
|
||
# colors would phash-collapse; reference-phash-test-images).
|
||
importer.settings.phash_threshold = 0
|
||
arc = import_root / "Bob" / "set.cbz"
|
||
arc.parent.mkdir(parents=True, exist_ok=True)
|
||
with zipfile.ZipFile(arc, "w") as zf:
|
||
zf.writestr("a.jpg", _split_bytes("v"))
|
||
zf.writestr("b.jpg", _split_bytes("h"))
|
||
zf.writestr("readme.txt", b"hello")
|
||
arc.with_suffix(".cbz.json").write_text(json.dumps(
|
||
{"category": "patreon", "id": "777", "title": "Set"}))
|
||
|
||
r = importer.import_one(arc)
|
||
assert r.status == "imported"
|
||
assert len(r.member_image_ids) == 2
|
||
|
||
imgs = importer.session.execute(select(ImageRecord)).scalars().all()
|
||
assert len(imgs) == 2
|
||
post = importer.session.execute(select(Post)).scalar_one()
|
||
provs = importer.session.execute(
|
||
select(func.count()).select_from(ImageProvenance)
|
||
).scalar_one()
|
||
assert provs == 2 # one Post, both members
|
||
att = importer.session.execute(select(PostAttachment)).scalar_one()
|
||
assert att.original_filename == "set.cbz"
|
||
assert att.post_id == post.id
|
||
|
||
|
||
def test_nested_archive_members_imported(importer, import_layout):
|
||
"""#718: a 'high-res' pack that wraps a nested archive must still import the
|
||
nested images and link them to the post — not silently drop them (incase's
|
||
per-chapter .rar inside an outer 'High Resolution files' pack)."""
|
||
import_root, _ = import_layout
|
||
importer.settings.phash_threshold = 0 # v/h splits are distinct only at 0
|
||
inner = io.BytesIO()
|
||
with zipfile.ZipFile(inner, "w") as zf:
|
||
zf.writestr("p1.jpg", _split_bytes("v"))
|
||
zf.writestr("p2.jpg", _split_bytes("h"))
|
||
arc = import_root / "Nessie" / "hr.cbz"
|
||
arc.parent.mkdir(parents=True, exist_ok=True)
|
||
with zipfile.ZipFile(arc, "w") as zf:
|
||
zf.writestr("chapter.zip", inner.getvalue()) # nested archive, no direct media
|
||
arc.with_suffix(".cbz.json").write_text(json.dumps(
|
||
{"category": "patreon", "id": "900", "title": "HR Pack"}))
|
||
|
||
r = importer.import_one(arc)
|
||
assert r.status == "imported"
|
||
assert len(r.member_image_ids) == 2 # both nested images surfaced
|
||
|
||
assert importer.session.execute(
|
||
select(func.count()).select_from(ImageRecord)
|
||
).scalar_one() == 2
|
||
post = importer.session.execute(select(Post)).scalar_one()
|
||
provs = importer.session.execute(
|
||
select(func.count()).select_from(ImageProvenance)
|
||
).scalar_one()
|
||
assert provs == 2 # both nested members linked to the one outer post
|
||
# The outer pack is still preserved as an attachment.
|
||
att = importer.session.execute(select(PostAttachment)).scalar_one()
|
||
assert att.original_filename == "hr.cbz"
|
||
assert att.post_id == post.id
|
||
|
||
|
||
def test_archive_all_deduped_is_benign_not_flagged(importer, import_layout):
|
||
"""#718 reason-string fix: when every image in an archive already exists
|
||
(cross-posted), the images re-link to the new post and the archive is NOT
|
||
flagged as an unextracted-archive problem (no error set)."""
|
||
import_root, _ = import_layout
|
||
importer.settings.phash_threshold = 0
|
||
pv, ph = _split_bytes("v"), _split_bytes("h")
|
||
first = import_root / "Dup" / "a.cbz"
|
||
first.parent.mkdir(parents=True, exist_ok=True)
|
||
with zipfile.ZipFile(first, "w") as zf:
|
||
zf.writestr("a.jpg", pv)
|
||
zf.writestr("b.jpg", ph)
|
||
first.with_suffix(".cbz.json").write_text(json.dumps(
|
||
{"category": "patreon", "id": "111", "title": "First"}))
|
||
assert importer.import_one(first).status == "imported"
|
||
|
||
second = import_root / "Dup" / "b.cbz" # different post, same two images
|
||
with zipfile.ZipFile(second, "w") as zf:
|
||
zf.writestr("a.jpg", pv)
|
||
zf.writestr("b.jpg", ph)
|
||
second.with_suffix(".cbz.json").write_text(json.dumps(
|
||
{"category": "patreon", "id": "222", "title": "Second"}))
|
||
|
||
r = importer.import_one(second)
|
||
assert r.status == "attached"
|
||
assert r.error is None # benign all-deduped — NOT a flagged problem
|
||
|
||
assert importer.session.execute(
|
||
select(func.count()).select_from(ImageRecord)
|
||
).scalar_one() == 2 # no new records
|
||
provs = importer.session.execute(
|
||
select(func.count()).select_from(ImageProvenance)
|
||
).scalar_one()
|
||
assert provs == 4 # 2 images × 2 posts (enrich-on-duplicate)
|
||
|
||
|
||
def test_archive_members_record_containing_archive(importer, import_layout):
|
||
"""Milestone #87: each extracted member's provenance row records the archive
|
||
PostAttachment it came out of, so provenance can show the one archive a file
|
||
lives inside."""
|
||
import_root, _ = import_layout
|
||
importer.settings.phash_threshold = 0
|
||
arc = import_root / "Bob" / "set.cbz"
|
||
arc.parent.mkdir(parents=True, exist_ok=True)
|
||
with zipfile.ZipFile(arc, "w") as zf:
|
||
zf.writestr("a.jpg", _split_bytes("v"))
|
||
zf.writestr("b.jpg", _split_bytes("h"))
|
||
arc.with_suffix(".cbz.json").write_text(json.dumps(
|
||
{"category": "patreon", "id": "777", "title": "Set"}))
|
||
|
||
importer.import_one(arc)
|
||
att = importer.session.execute(select(PostAttachment)).scalar_one()
|
||
provs = importer.session.execute(select(ImageProvenance)).scalars().all()
|
||
assert len(provs) == 2
|
||
assert all(p.from_attachment_id == att.id for p in provs)
|
||
|
||
|
||
def test_nested_archive_member_records_outer_archive(importer, import_layout):
|
||
"""Nested members link to the OUTER stored archive — the only one persisted
|
||
as a PostAttachment (the inner archive lives in a tempdir)."""
|
||
import_root, _ = import_layout
|
||
importer.settings.phash_threshold = 0
|
||
inner = io.BytesIO()
|
||
with zipfile.ZipFile(inner, "w") as zf:
|
||
zf.writestr("p1.jpg", _split_bytes("v"))
|
||
zf.writestr("p2.jpg", _split_bytes("h"))
|
||
arc = import_root / "Nessie" / "hr.cbz"
|
||
arc.parent.mkdir(parents=True, exist_ok=True)
|
||
with zipfile.ZipFile(arc, "w") as zf:
|
||
zf.writestr("chapter.zip", inner.getvalue())
|
||
arc.with_suffix(".cbz.json").write_text(json.dumps(
|
||
{"category": "patreon", "id": "900", "title": "HR Pack"}))
|
||
|
||
importer.import_one(arc)
|
||
att = importer.session.execute(select(PostAttachment)).scalar_one()
|
||
provs = importer.session.execute(select(ImageProvenance)).scalars().all()
|
||
assert len(provs) == 2
|
||
assert all(p.from_attachment_id == att.id for p in provs)
|
||
|
||
|
||
def test_deduped_archive_member_records_each_posts_archive(importer, import_layout):
|
||
"""A member re-shipped in a second post's archive gets its NEW provenance row
|
||
(for the second post) stamped with the SECOND archive — the UPDATE-of-existing
|
||
path the reextract backfill relies on. Each post's rows point at its own
|
||
archive."""
|
||
import_root, _ = import_layout
|
||
importer.settings.phash_threshold = 0
|
||
pv, ph = _split_bytes("v"), _split_bytes("h")
|
||
first = import_root / "Dup" / "a.cbz"
|
||
first.parent.mkdir(parents=True, exist_ok=True)
|
||
with zipfile.ZipFile(first, "w") as zf:
|
||
zf.writestr("a.jpg", pv)
|
||
zf.writestr("b.jpg", ph)
|
||
first.with_suffix(".cbz.json").write_text(json.dumps(
|
||
{"category": "patreon", "id": "111", "title": "First"}))
|
||
importer.import_one(first)
|
||
|
||
second = import_root / "Dup" / "b.cbz" # different post, same two images
|
||
with zipfile.ZipFile(second, "w") as zf:
|
||
zf.writestr("a.jpg", pv)
|
||
zf.writestr("b.jpg", ph)
|
||
second.with_suffix(".cbz.json").write_text(json.dumps(
|
||
{"category": "patreon", "id": "222", "title": "Second"}))
|
||
importer.import_one(second)
|
||
|
||
posts = {p.external_post_id: p.id for p in
|
||
importer.session.execute(select(Post)).scalars().all()}
|
||
atts = {a.post_id: a.id for a in
|
||
importer.session.execute(select(PostAttachment)).scalars().all()}
|
||
provs = importer.session.execute(select(ImageProvenance)).scalars().all()
|
||
assert len(provs) == 4 # 2 images × 2 posts
|
||
for p in provs:
|
||
assert p.from_attachment_id == atts[p.post_id]
|
||
assert atts[posts["111"]] != atts[posts["222"]]
|
||
|
||
|
||
def test_corrupt_archive_still_stored(importer, import_layout):
|
||
import_root, _ = import_layout
|
||
arc = import_root / "Bob" / "broken.zip"
|
||
arc.parent.mkdir(parents=True, exist_ok=True)
|
||
arc.write_bytes(b"definitely not a zip")
|
||
r = importer.import_one(arc)
|
||
assert r.status == "attached"
|
||
att = importer.session.execute(select(PostAttachment)).scalar_one()
|
||
assert att.original_filename == "broken.zip"
|
||
assert importer.session.execute(
|
||
select(func.count()).select_from(ImageRecord)
|
||
).scalar_one() == 0
|
||
|
||
|
||
def test_reimport_archive_is_idempotent(importer, import_layout):
|
||
import_root, _ = import_layout
|
||
arc = import_root / "Bob" / "set.zip"
|
||
arc.parent.mkdir(parents=True, exist_ok=True)
|
||
with zipfile.ZipFile(arc, "w") as zf:
|
||
zf.writestr("a.jpg", _jpeg_bytes((1, 2, 3)))
|
||
importer.import_one(arc)
|
||
importer.import_one(arc)
|
||
assert importer.session.execute(
|
||
select(func.count()).select_from(PostAttachment)
|
||
).scalar_one() == 1
|