bvandeusen
a497104661
Existing PostAttachments that are actually archives (filed opaquely before the magic-byte gate) need extracting retroactively. cleanup_service. reextract_archive_attachments scans PostAttachments, magic-detects the archives, and for each reconstructs the post's sidecar from the DB + re-runs attach_in_place in a temp dir — so the members extract and re-link to the SAME post via find_or_create_post (source_id + external_post_id). Idempotent (members dedupe by sha256). Enqueues thumbnail+ML for new members. Wired as a maintenance-queue Celery task (tasks/admin) + POST /api/admin/maintenance/reextract-archives (202) + a "Re-extract archive attachments" card in Settings → Maintenance. Test: a zip stored under a mangled extension-less name extracts + links its member to the post via ImageProvenance, and a second run is a no-op (idempotent). Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
92 lines
2.9 KiB
Python
92 lines
2.9 KiB
Python
"""#713 part 2: re-extract archive PostAttachments that were filed opaquely
|
|
(magic-byte gate missed them) and link their members to the post."""
|
|
|
|
import hashlib
|
|
import io
|
|
import zipfile
|
|
|
|
import pytest
|
|
from PIL import Image
|
|
from sqlalchemy import select
|
|
|
|
from backend.app.models import (
|
|
Artist,
|
|
ImageProvenance,
|
|
ImageRecord,
|
|
Post,
|
|
PostAttachment,
|
|
Source,
|
|
)
|
|
from backend.app.services import cleanup_service
|
|
|
|
pytestmark = pytest.mark.integration
|
|
|
|
|
|
def _jpeg(color, size=256):
|
|
buf = io.BytesIO()
|
|
Image.new("RGB", (size, size), color).save(buf, "JPEG")
|
|
return buf.getvalue()
|
|
|
|
|
|
def test_reextract_links_archive_members_to_post(db_sync, tmp_path, monkeypatch):
|
|
from backend.app.tasks import ml as ml_mod
|
|
from backend.app.tasks import thumbnail as thumb_mod
|
|
|
|
# No broker in this path — the post-import enqueue is best-effort anyway.
|
|
monkeypatch.setattr(thumb_mod.generate_thumbnail, "delay", lambda *a, **k: None)
|
|
monkeypatch.setattr(ml_mod.tag_and_embed, "delay", lambda *a, **k: None)
|
|
|
|
images_root = tmp_path / "images"
|
|
images_root.mkdir()
|
|
|
|
artist = Artist(name="Bob", slug="bob")
|
|
db_sync.add(artist)
|
|
db_sync.flush()
|
|
source = Source(
|
|
artist_id=artist.id, platform="patreon",
|
|
url="https://patreon.com/bob", enabled=True, config_overrides={},
|
|
)
|
|
db_sync.add(source)
|
|
db_sync.flush()
|
|
post = Post(
|
|
source_id=source.id, artist_id=artist.id, external_post_id="59102952",
|
|
post_url="https://www.patreon.com/posts/59102952",
|
|
)
|
|
db_sync.add(post)
|
|
db_sync.flush()
|
|
|
|
# A real zip stored under a mangled / extension-less name (the failure case).
|
|
store_dir = images_root / "attachments" / "abc"
|
|
store_dir.mkdir(parents=True)
|
|
arc = store_dir / "01_https___www.patreon.com_media-u_v3_59102952"
|
|
with zipfile.ZipFile(arc, "w") as zf:
|
|
zf.writestr("inside.jpg", _jpeg("red"))
|
|
sha = hashlib.sha256(arc.read_bytes()).hexdigest()
|
|
db_sync.add(PostAttachment(
|
|
post_id=post.id, artist_id=artist.id, sha256=sha, path=str(arc),
|
|
original_filename=arc.name, ext="", size_bytes=arc.stat().st_size,
|
|
))
|
|
db_sync.commit()
|
|
|
|
summary = cleanup_service.reextract_archive_attachments(
|
|
db_sync, images_root=images_root,
|
|
)
|
|
assert summary["archives"] == 1
|
|
assert summary["members_imported"] == 1
|
|
assert summary["posts_touched"] == 1
|
|
|
|
images = db_sync.execute(select(ImageRecord)).scalars().all()
|
|
assert len(images) == 1
|
|
prov = db_sync.execute(
|
|
select(ImageProvenance).where(ImageProvenance.post_id == post.id)
|
|
).scalars().all()
|
|
assert len(prov) == 1
|
|
assert prov[0].image_record_id == images[0].id
|
|
|
|
# Idempotent — a second run imports nothing new (member dedups by sha256).
|
|
again = cleanup_service.reextract_archive_attachments(
|
|
db_sync, images_root=images_root,
|
|
)
|
|
assert again["members_imported"] == 0
|
|
assert db_sync.execute(select(ImageRecord)).scalars().all() == images
|