feat(provenance): capture which archive an extracted image came from (#87)
Images pulled out of a .zip/.rar previously kept no record of WHICH archive
they came from — the member->archive link was computed during extraction and
discarded, leaving only image->post. So the provenance modal could only scope
attachments to the whole post, showing every archive a 'High Resolution Files'
bundle carried instead of the one a given file lives in.
- ImageProvenance.from_attachment_id: nullable FK -> post_attachment.id
(SET NULL), migration 0055.
- importer: _import_archive stamps from_attachment_id on every member's
provenance row for the post (new + superseded + deduped members), resolving
the archive's own PostAttachment by (post, sha). Post-pass UPDATE, NULL-only
and idempotent, so it doesn't touch the dedup/supersede branches and the
backfill is safe to re-run. Nested members link to the outer stored archive.
- provenance_service.for_image: when the originating post's provenance row
records from_attachment_id, return ONLY that archive; else fall back to the
primary-post scoping from 068def2.
- ProvenancePanel: heading pluralizes ('Attachment' for a single file).
- Backfill: re-running reextract_archive_attachments (ArchiveReextractCard)
routes through _import_archive and stamps existing rows — no new code.
Tests: capture stamps on fresh import, nested-archive attribution, per-post
archive on dedup; for_image filters to the containing archive; reextract
backfill stamps the link.
Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -164,6 +164,86 @@ def test_archive_all_deduped_is_benign_not_flagged(importer, import_layout):
|
||||
assert provs == 4 # 2 images × 2 posts (enrich-on-duplicate)
|
||||
|
||||
|
||||
def test_archive_members_record_containing_archive(importer, import_layout):
|
||||
"""Milestone #87: each extracted member's provenance row records the archive
|
||||
PostAttachment it came out of, so provenance can show the one archive a file
|
||||
lives inside."""
|
||||
import_root, _ = import_layout
|
||||
importer.settings.phash_threshold = 0
|
||||
arc = import_root / "Bob" / "set.cbz"
|
||||
arc.parent.mkdir(parents=True, exist_ok=True)
|
||||
with zipfile.ZipFile(arc, "w") as zf:
|
||||
zf.writestr("a.jpg", _split_bytes("v"))
|
||||
zf.writestr("b.jpg", _split_bytes("h"))
|
||||
arc.with_suffix(".cbz.json").write_text(json.dumps(
|
||||
{"category": "patreon", "id": "777", "title": "Set"}))
|
||||
|
||||
importer.import_one(arc)
|
||||
att = importer.session.execute(select(PostAttachment)).scalar_one()
|
||||
provs = importer.session.execute(select(ImageProvenance)).scalars().all()
|
||||
assert len(provs) == 2
|
||||
assert all(p.from_attachment_id == att.id for p in provs)
|
||||
|
||||
|
||||
def test_nested_archive_member_records_outer_archive(importer, import_layout):
|
||||
"""Nested members link to the OUTER stored archive — the only one persisted
|
||||
as a PostAttachment (the inner archive lives in a tempdir)."""
|
||||
import_root, _ = import_layout
|
||||
importer.settings.phash_threshold = 0
|
||||
inner = io.BytesIO()
|
||||
with zipfile.ZipFile(inner, "w") as zf:
|
||||
zf.writestr("p1.jpg", _split_bytes("v"))
|
||||
zf.writestr("p2.jpg", _split_bytes("h"))
|
||||
arc = import_root / "Nessie" / "hr.cbz"
|
||||
arc.parent.mkdir(parents=True, exist_ok=True)
|
||||
with zipfile.ZipFile(arc, "w") as zf:
|
||||
zf.writestr("chapter.zip", inner.getvalue())
|
||||
arc.with_suffix(".cbz.json").write_text(json.dumps(
|
||||
{"category": "patreon", "id": "900", "title": "HR Pack"}))
|
||||
|
||||
importer.import_one(arc)
|
||||
att = importer.session.execute(select(PostAttachment)).scalar_one()
|
||||
provs = importer.session.execute(select(ImageProvenance)).scalars().all()
|
||||
assert len(provs) == 2
|
||||
assert all(p.from_attachment_id == att.id for p in provs)
|
||||
|
||||
|
||||
def test_deduped_archive_member_records_each_posts_archive(importer, import_layout):
|
||||
"""A member re-shipped in a second post's archive gets its NEW provenance row
|
||||
(for the second post) stamped with the SECOND archive — the UPDATE-of-existing
|
||||
path the reextract backfill relies on. Each post's rows point at its own
|
||||
archive."""
|
||||
import_root, _ = import_layout
|
||||
importer.settings.phash_threshold = 0
|
||||
pv, ph = _split_bytes("v"), _split_bytes("h")
|
||||
first = import_root / "Dup" / "a.cbz"
|
||||
first.parent.mkdir(parents=True, exist_ok=True)
|
||||
with zipfile.ZipFile(first, "w") as zf:
|
||||
zf.writestr("a.jpg", pv)
|
||||
zf.writestr("b.jpg", ph)
|
||||
first.with_suffix(".cbz.json").write_text(json.dumps(
|
||||
{"category": "patreon", "id": "111", "title": "First"}))
|
||||
importer.import_one(first)
|
||||
|
||||
second = import_root / "Dup" / "b.cbz" # different post, same two images
|
||||
with zipfile.ZipFile(second, "w") as zf:
|
||||
zf.writestr("a.jpg", pv)
|
||||
zf.writestr("b.jpg", ph)
|
||||
second.with_suffix(".cbz.json").write_text(json.dumps(
|
||||
{"category": "patreon", "id": "222", "title": "Second"}))
|
||||
importer.import_one(second)
|
||||
|
||||
posts = {p.external_post_id: p.id for p in
|
||||
importer.session.execute(select(Post)).scalars().all()}
|
||||
atts = {a.post_id: a.id for a in
|
||||
importer.session.execute(select(PostAttachment)).scalars().all()}
|
||||
provs = importer.session.execute(select(ImageProvenance)).scalars().all()
|
||||
assert len(provs) == 4 # 2 images × 2 posts
|
||||
for p in provs:
|
||||
assert p.from_attachment_id == atts[p.post_id]
|
||||
assert atts[posts["111"]] != atts[posts["222"]]
|
||||
|
||||
|
||||
def test_corrupt_archive_still_stored(importer, import_layout):
|
||||
import_root, _ = import_layout
|
||||
arc = import_root / "Bob" / "broken.zip"
|
||||
|
||||
Reference in New Issue
Block a user