feat(provenance): capture which archive an extracted image came from (#87)
Images pulled out of a .zip/.rar previously kept no record of WHICH archive
they came from — the member->archive link was computed during extraction and
discarded, leaving only image->post. So the provenance modal could only scope
attachments to the whole post, showing every archive a 'High Resolution Files'
bundle carried instead of the one a given file lives in.
- ImageProvenance.from_attachment_id: nullable FK -> post_attachment.id
(SET NULL), migration 0055.
- importer: _import_archive stamps from_attachment_id on every member's
provenance row for the post (new + superseded + deduped members), resolving
the archive's own PostAttachment by (post, sha). Post-pass UPDATE, NULL-only
and idempotent, so it doesn't touch the dedup/supersede branches and the
backfill is safe to re-run. Nested members link to the outer stored archive.
- provenance_service.for_image: when the originating post's provenance row
records from_attachment_id, return ONLY that archive; else fall back to the
primary-post scoping from 068def2.
- ProvenancePanel: heading pluralizes ('Attachment' for a single file).
- Backfill: re-running reextract_archive_attachments (ArchiveReextractCard)
routes through _import_archive and stamps existing rows — no new code.
Tests: capture stamps on fresh import, nested-archive attribution, per-post
archive on dedup; for_image filters to the containing archive; reextract
backfill stamps the link.
Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -164,6 +164,86 @@ def test_archive_all_deduped_is_benign_not_flagged(importer, import_layout):
|
||||
assert provs == 4 # 2 images × 2 posts (enrich-on-duplicate)
|
||||
|
||||
|
||||
def test_archive_members_record_containing_archive(importer, import_layout):
|
||||
"""Milestone #87: each extracted member's provenance row records the archive
|
||||
PostAttachment it came out of, so provenance can show the one archive a file
|
||||
lives inside."""
|
||||
import_root, _ = import_layout
|
||||
importer.settings.phash_threshold = 0
|
||||
arc = import_root / "Bob" / "set.cbz"
|
||||
arc.parent.mkdir(parents=True, exist_ok=True)
|
||||
with zipfile.ZipFile(arc, "w") as zf:
|
||||
zf.writestr("a.jpg", _split_bytes("v"))
|
||||
zf.writestr("b.jpg", _split_bytes("h"))
|
||||
arc.with_suffix(".cbz.json").write_text(json.dumps(
|
||||
{"category": "patreon", "id": "777", "title": "Set"}))
|
||||
|
||||
importer.import_one(arc)
|
||||
att = importer.session.execute(select(PostAttachment)).scalar_one()
|
||||
provs = importer.session.execute(select(ImageProvenance)).scalars().all()
|
||||
assert len(provs) == 2
|
||||
assert all(p.from_attachment_id == att.id for p in provs)
|
||||
|
||||
|
||||
def test_nested_archive_member_records_outer_archive(importer, import_layout):
|
||||
"""Nested members link to the OUTER stored archive — the only one persisted
|
||||
as a PostAttachment (the inner archive lives in a tempdir)."""
|
||||
import_root, _ = import_layout
|
||||
importer.settings.phash_threshold = 0
|
||||
inner = io.BytesIO()
|
||||
with zipfile.ZipFile(inner, "w") as zf:
|
||||
zf.writestr("p1.jpg", _split_bytes("v"))
|
||||
zf.writestr("p2.jpg", _split_bytes("h"))
|
||||
arc = import_root / "Nessie" / "hr.cbz"
|
||||
arc.parent.mkdir(parents=True, exist_ok=True)
|
||||
with zipfile.ZipFile(arc, "w") as zf:
|
||||
zf.writestr("chapter.zip", inner.getvalue())
|
||||
arc.with_suffix(".cbz.json").write_text(json.dumps(
|
||||
{"category": "patreon", "id": "900", "title": "HR Pack"}))
|
||||
|
||||
importer.import_one(arc)
|
||||
att = importer.session.execute(select(PostAttachment)).scalar_one()
|
||||
provs = importer.session.execute(select(ImageProvenance)).scalars().all()
|
||||
assert len(provs) == 2
|
||||
assert all(p.from_attachment_id == att.id for p in provs)
|
||||
|
||||
|
||||
def test_deduped_archive_member_records_each_posts_archive(importer, import_layout):
|
||||
"""A member re-shipped in a second post's archive gets its NEW provenance row
|
||||
(for the second post) stamped with the SECOND archive — the UPDATE-of-existing
|
||||
path the reextract backfill relies on. Each post's rows point at its own
|
||||
archive."""
|
||||
import_root, _ = import_layout
|
||||
importer.settings.phash_threshold = 0
|
||||
pv, ph = _split_bytes("v"), _split_bytes("h")
|
||||
first = import_root / "Dup" / "a.cbz"
|
||||
first.parent.mkdir(parents=True, exist_ok=True)
|
||||
with zipfile.ZipFile(first, "w") as zf:
|
||||
zf.writestr("a.jpg", pv)
|
||||
zf.writestr("b.jpg", ph)
|
||||
first.with_suffix(".cbz.json").write_text(json.dumps(
|
||||
{"category": "patreon", "id": "111", "title": "First"}))
|
||||
importer.import_one(first)
|
||||
|
||||
second = import_root / "Dup" / "b.cbz" # different post, same two images
|
||||
with zipfile.ZipFile(second, "w") as zf:
|
||||
zf.writestr("a.jpg", pv)
|
||||
zf.writestr("b.jpg", ph)
|
||||
second.with_suffix(".cbz.json").write_text(json.dumps(
|
||||
{"category": "patreon", "id": "222", "title": "Second"}))
|
||||
importer.import_one(second)
|
||||
|
||||
posts = {p.external_post_id: p.id for p in
|
||||
importer.session.execute(select(Post)).scalars().all()}
|
||||
atts = {a.post_id: a.id for a in
|
||||
importer.session.execute(select(PostAttachment)).scalars().all()}
|
||||
provs = importer.session.execute(select(ImageProvenance)).scalars().all()
|
||||
assert len(provs) == 4 # 2 images × 2 posts
|
||||
for p in provs:
|
||||
assert p.from_attachment_id == atts[p.post_id]
|
||||
assert atts[posts["111"]] != atts[posts["222"]]
|
||||
|
||||
|
||||
def test_corrupt_archive_still_stored(importer, import_layout):
|
||||
import_root, _ = import_layout
|
||||
arc = import_root / "Bob" / "broken.zip"
|
||||
|
||||
@@ -165,6 +165,37 @@ async def test_for_image_attachments_scoped_to_primary_post(db):
|
||||
assert names == ["keep.zip"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_for_image_attachments_filtered_to_containing_archive(db):
|
||||
# Milestone #87: when the originating post's provenance row records which
|
||||
# archive the file came from, show ONLY that archive — not the post's other
|
||||
# attachments.
|
||||
rec = await _seed_image(db)
|
||||
a1, s1, post = await _seed_post(db, artist_name="Arc", slug="arc",
|
||||
platform="patreon", ext_id="500")
|
||||
rec.primary_post_id = post.id
|
||||
att_in = PostAttachment(
|
||||
post_id=post.id, artist_id=a1.id, sha256="e" + "0" * 63,
|
||||
path="/images/attachments/e00/in.cbz", original_filename="in.cbz",
|
||||
ext=".cbz", mime="application/zip", size_bytes=9,
|
||||
)
|
||||
att_other = PostAttachment(
|
||||
post_id=post.id, artist_id=a1.id, sha256="f" + "0" * 63,
|
||||
path="/images/attachments/f00/other.rar", original_filename="other.rar",
|
||||
ext=".rar", mime="application/x-rar", size_bytes=9,
|
||||
)
|
||||
db.add(att_in)
|
||||
db.add(att_other)
|
||||
await db.flush()
|
||||
db.add(ImageProvenance(image_record_id=rec.id, post_id=post.id,
|
||||
source_id=s1.id, from_attachment_id=att_in.id))
|
||||
await db.flush()
|
||||
|
||||
payload = await ProvenanceService(db).for_image(rec.id)
|
||||
names = [a["original_filename"] for a in payload["attachments"]]
|
||||
assert names == ["in.cbz"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_for_image_attachments_fallback_to_all_posts_when_no_primary(db):
|
||||
# No primary_post_id (older rows / filesystem imports) → preserve the
|
||||
|
||||
@@ -74,10 +74,13 @@ def test_reextract_links_archive_members_to_post(db_sync, tmp_path, monkeypatch)
|
||||
with zipfile.ZipFile(arc, "w") as zf:
|
||||
zf.writestr("inside.jpg", _jpeg("red"))
|
||||
sha = hashlib.sha256(arc.read_bytes()).hexdigest()
|
||||
db_sync.add(PostAttachment(
|
||||
att = PostAttachment(
|
||||
post_id=post.id, artist_id=artist.id, sha256=sha, path=str(arc),
|
||||
original_filename=arc.name, ext="", size_bytes=arc.stat().st_size,
|
||||
))
|
||||
)
|
||||
db_sync.add(att)
|
||||
db_sync.flush()
|
||||
att_id = att.id
|
||||
db_sync.commit()
|
||||
|
||||
summary = cleanup_service.reextract_archive_attachments(
|
||||
@@ -94,6 +97,9 @@ def test_reextract_links_archive_members_to_post(db_sync, tmp_path, monkeypatch)
|
||||
).scalars().all()
|
||||
assert len(prov) == 1
|
||||
assert prov[0].image_record_id == images[0].id
|
||||
# Milestone #87: re-extraction stamps which archive the member came from —
|
||||
# this is the backfill path for pre-existing opaque archives.
|
||||
assert prov[0].from_attachment_id == att_id
|
||||
|
||||
# Idempotent — a second run imports nothing new (member dedups by sha256).
|
||||
again = cleanup_service.reextract_archive_attachments(
|
||||
|
||||
Reference in New Issue
Block a user