"""attach_in_place — Importer seam used by FC-3c (downloader).""" from pathlib import Path import pytest from sqlalchemy import func, select from backend.app.models import Artist, ImageRecord, ImportSettings from backend.app.services.importer import Importer from backend.app.services.thumbnailer import Thumbnailer from backend.app.utils import safe_probe from backend.app.utils.safe_probe import ProbeResult pytestmark = pytest.mark.integration def _make_jpg(path: Path, color=(255, 0, 0), size=(64, 64)) -> bytes: from PIL import Image path.parent.mkdir(parents=True, exist_ok=True) Image.new("RGB", size, color).save(path, "JPEG") return path.read_bytes() @pytest.fixture def importer(db_sync, tmp_path): settings = db_sync.execute( select(ImportSettings).where(ImportSettings.id == 1) ).scalar_one() images_root = tmp_path / "images" images_root.mkdir() return Importer( session=db_sync, images_root=images_root, import_root=tmp_path / "ignored", thumbnailer=Thumbnailer(images_root=images_root), settings=settings, ) def test_attach_in_place_imports_happy(importer, db_sync): images_root = importer.images_root artist = Artist(name="Alice", slug="alice") db_sync.add(artist) db_sync.flush() target = images_root / "alice" / "patreon" / "post1" / "img.jpg" _make_jpg(target) result = importer.attach_in_place(target, artist=artist) assert result.status == "imported" assert result.image_id is not None rec_path = db_sync.execute( select(ImageRecord.path, ImageRecord.origin, ImageRecord.artist_id) .where(ImageRecord.id == result.image_id) ).one() assert rec_path.path == str(target) assert rec_path.origin == "downloaded" assert rec_path.artist_id == artist.id def test_attach_in_place_with_sidecar(importer, db_sync): images_root = importer.images_root artist = Artist(name="Bob", slug="bob") db_sync.add(artist) db_sync.flush() target = images_root / "bob" / "patreon" / "post1" / "img.jpg" _make_jpg(target) sidecar = target.with_suffix(target.suffix + ".json") sidecar.write_text( '{"category": "patreon", "id": "12345", "url": "https://patreon.com/post/12345", ' '"title": "Test Post", "published_at": "2026-05-01T00:00:00+00:00"}' ) result = importer.attach_in_place(target, artist=artist) assert result.status == "imported" primary_post_id = db_sync.execute( select(ImageRecord.primary_post_id).where(ImageRecord.id == result.image_id) ).scalar_one() assert primary_post_id is not None # post created from sidecar def test_attach_in_place_sha256_dedup_does_not_modify_disk(importer, db_sync): images_root = importer.images_root artist = Artist(name="Cara", slug="cara") db_sync.add(artist) db_sync.flush() first = images_root / "cara" / "p" / "a.jpg" bytes_ = _make_jpg(first) importer.attach_in_place(first, artist=artist) second = images_root / "cara" / "p" / "b.jpg" second.parent.mkdir(parents=True, exist_ok=True) second.write_bytes(bytes_) assert second.exists() result = importer.attach_in_place(second, artist=artist) assert result.status == "skipped" assert result.skip_reason.value == "duplicate_hash" # Caller is responsible for deleting the on-disk file; we don't touch it. assert second.exists() def test_attach_in_place_supersede_keeps_file_at_new_path(importer, db_sync): """phash near-dup where the new file is LARGER → existing record swaps to the new path; old file deleted.""" images_root = importer.images_root artist = Artist(name="Dee", slug="dee") db_sync.add(artist) db_sync.flush() smaller = images_root / "dee" / "p" / "small.jpg" _make_jpg(smaller, color=(50, 50, 200), size=(64, 64)) first = importer.attach_in_place(smaller, artist=artist) assert first.status == "imported" bigger = images_root / "dee" / "p" / "big.jpg" _make_jpg(bigger, color=(50, 50, 200), size=(256, 256)) result = importer.attach_in_place(bigger, artist=artist) assert result.status == "superseded" new_path = db_sync.execute( select(ImageRecord.path).where(ImageRecord.id == first.image_id) ).scalar_one() assert new_path == str(bigger) assert not smaller.exists() assert bigger.exists() def test_attach_in_place_phash_smaller_skips_without_modifying_disk( importer, db_sync ): images_root = importer.images_root artist = Artist(name="Eve", slug="eve") db_sync.add(artist) db_sync.flush() bigger = images_root / "eve" / "p" / "big.jpg" _make_jpg(bigger, color=(0, 200, 0), size=(256, 256)) importer.attach_in_place(bigger, artist=artist) smaller = images_root / "eve" / "p" / "small.jpg" _make_jpg(smaller, color=(0, 200, 0), size=(64, 64)) result = importer.attach_in_place(smaller, artist=artist) assert result.status == "skipped" assert result.skip_reason.value == "duplicate_phash" assert smaller.exists() def test_attach_in_place_duplicate_image_links_both_posts(importer, db_sync): """A byte-identical image appearing in a SECOND post must link to BOTH posts (enrich-on-duplicate) so it shows on every post — not be silently dropped, which left the second post a bare shell. Operator-flagged 2026-06-08.""" from backend.app.models import ImageProvenance, Post images_root = importer.images_root artist = Artist(name="Hana", slug="hana") db_sync.add(artist) db_sync.flush() first = images_root / "hana" / "patreon" / "post_a" / "img.jpg" bytes_ = _make_jpg(first, color=(120, 40, 90)) first.with_suffix(first.suffix + ".json").write_text( '{"category": "patreon", "id": "111", "title": "Post 111"}' ) r1 = importer.attach_in_place(first, artist=artist) assert r1.status == "imported" second = images_root / "hana" / "patreon" / "post_b" / "img.jpg" second.parent.mkdir(parents=True, exist_ok=True) second.write_bytes(bytes_) # identical bytes → sha256 duplicate second.with_suffix(second.suffix + ".json").write_text( '{"category": "patreon", "id": "222", "title": "Post 222"}' ) r2 = importer.attach_in_place(second, artist=artist) assert r2.status == "skipped" assert r2.skip_reason.value == "duplicate_hash" assert r2.image_id == r1.image_id linked = db_sync.execute( select(Post.external_post_id) .join(ImageProvenance, ImageProvenance.post_id == Post.id) .where(ImageProvenance.image_record_id == r1.image_id) ).scalars().all() assert set(linked) == {"111", "222"} # shows on BOTH posts primary = db_sync.execute( select(Post.external_post_id) .join(ImageRecord, ImageRecord.primary_post_id == Post.id) .where(ImageRecord.id == r1.image_id) ).scalar_one() assert primary == "111" # primary stays on the first post def test_attach_in_place_duplicate_attachment_links_both_posts(importer, db_sync): """The same non-art file attached to two posts gets a row on EACH post over a single sha-addressed blob — neither post is left bare. Operator-flagged 2026-06-08 (1589 empty Anduo shells from globally-unique attachment sha).""" from backend.app.models import Post, PostAttachment images_root = importer.images_root artist = Artist(name="Iris", slug="iris") db_sync.add(artist) db_sync.flush() payload = b"the same readme creator attaches to every post" a = images_root / "iris" / "patreon" / "post_a" / "readme.txt" a.parent.mkdir(parents=True, exist_ok=True) a.write_bytes(payload) a.with_suffix(a.suffix + ".json").write_text( '{"category": "patreon", "id": "301", "title": "Post 301"}' ) assert importer.attach_in_place(a, artist=artist).status == "attached" b = images_root / "iris" / "patreon" / "post_b" / "readme.txt" b.parent.mkdir(parents=True, exist_ok=True) b.write_bytes(payload) b.with_suffix(b.suffix + ".json").write_text( '{"category": "patreon", "id": "302", "title": "Post 302"}' ) assert importer.attach_in_place(b, artist=artist).status == "attached" linked = db_sync.execute( select(Post.external_post_id) .join(PostAttachment, PostAttachment.post_id == Post.id) .where(PostAttachment.original_filename == "readme.txt") ).scalars().all() assert set(linked) == {"301", "302"} # one row per post, both present def test_attach_in_place_post_first_skips_post_fields(importer, db_sync): """Post-first (#856, milestone #67): with post_first set (native path), the per-media import links provenance + localization (source_filehash) but does NOT write the post body/title — the post-record owns those. Guards the import side even if a body sneaks into the sidecar.""" from backend.app.models import ImageProvenance, Post importer.post_first = True images_root = importer.images_root artist = Artist(name="Pat", slug="pat") db_sync.add(artist) db_sync.flush() target = images_root / "pat" / "patreon" / "post1" / "img.jpg" _make_jpg(target) cdn = "https://cdn.patreon.com/" + "a" * 32 + "/x.jpg" target.with_suffix(target.suffix + ".json").write_text( '{"category": "patreon", "id": "900", "title": "Should NOT apply", ' '"content": "

body should NOT apply

", ' f'"source_url": "{cdn}"}}' ) result = importer.attach_in_place(target, artist=artist) assert result.status == "imported" # Image-specific work STILL happens: localization filehash + provenance link. rec = db_sync.execute( select(ImageRecord).where(ImageRecord.id == result.image_id) ).scalar_one() assert rec.source_filehash == "a" * 32 assert rec.primary_post_id is not None post = db_sync.get(Post, rec.primary_post_id) assert post.external_post_id == "900" # Post-first: body/title NOT written by the media path (post-record owns them). assert post.description is None assert post.post_title is None # Provenance link exists regardless of post_first. prov = db_sync.execute( select(ImageProvenance).where( ImageProvenance.image_record_id == result.image_id ) ).scalars().all() assert len(prov) == 1 def test_attach_in_place_post_first_false_applies_post_fields(importer, db_sync): """Default (gallery-dl path): the sidecar IS the body source, so post fields ARE applied. Guards that the post_first flag actually gates this — not a no-op.""" from backend.app.models import Post assert importer.post_first is False # default images_root = importer.images_root artist = Artist(name="Gal", slug="gal") db_sync.add(artist) db_sync.flush() target = images_root / "gal" / "patreon" / "post1" / "img.jpg" _make_jpg(target) target.with_suffix(target.suffix + ".json").write_text( '{"category": "patreon", "id": "901", "title": "Applies", ' '"content": "

body applies

"}' ) result = importer.attach_in_place(target, artist=artist) assert result.status == "imported" rec = db_sync.get(ImageRecord, result.image_id) post = db_sync.get(Post, rec.primary_post_id) assert post.description == "

body applies

" assert post.post_title == "Applies" def _fake_video_probe(monkeypatch, dims): """Patch safe_probe.probe_video to return controlled (w,h,duration) keyed by filename — so video dedup (#871) can be tested without real mp4s/ffprobe.""" def _probe(path, **_kw): w, h, d = dims[Path(path).name] return ProbeResult(ok=True, width=w, height=h, duration=d) monkeypatch.setattr(safe_probe, "probe_video", _probe) def _mkvideo(images_root, rel, content): p = images_root / rel p.parent.mkdir(parents=True, exist_ok=True) p.write_bytes(content) return p def test_attach_in_place_video_dedup_skips_smaller(importer, db_sync, monkeypatch): """#871 Tier-1: a same-artist video with matching duration+aspect but smaller resolution is a near-dup of the existing larger video → skipped + linked, not re-imported (the #859 'same video from multiple sources' case).""" images_root = importer.images_root artist = Artist(name="Vid", slug="vid") db_sync.add(artist) db_sync.flush() _fake_video_probe(monkeypatch, { "a.mp4": (1920, 1080, 300.0), "b.mp4": (1280, 720, 300.4), # within 1s, same 16:9 aspect, smaller }) big = _mkvideo(images_root, "vid/patreon/p/a.mp4", b"AAAA") r1 = importer.attach_in_place(big, artist=artist) assert r1.status == "imported" small = _mkvideo(images_root, "vid/patreon/p2/b.mp4", b"BBBB") r2 = importer.attach_in_place(small, artist=artist) assert r2.status == "skipped" assert r2.skip_reason.value == "duplicate_phash" assert r2.image_id == r1.image_id assert db_sync.execute( select(func.count()).select_from(ImageRecord) ).scalar_one() == 1 def test_attach_in_place_video_dedup_supersedes_larger(importer, db_sync, monkeypatch): """A higher-res re-download supersedes the smaller existing video, keeping the row id + adopting the new duration.""" images_root = importer.images_root artist = Artist(name="Vid2", slug="vid2") db_sync.add(artist) db_sync.flush() _fake_video_probe(monkeypatch, { "s.mp4": (640, 360, 120.0), "l.mp4": (1920, 1080, 120.2), # within 1s, same aspect, larger }) small = _mkvideo(images_root, "vid2/p/s.mp4", b"SSSS") r1 = importer.attach_in_place(small, artist=artist) assert r1.status == "imported" big = _mkvideo(images_root, "vid2/p/l.mp4", b"LLLL") r2 = importer.attach_in_place(big, artist=artist) assert r2.status == "superseded" assert r2.image_id == r1.image_id db_sync.expire_all() rec = db_sync.get(ImageRecord, r1.image_id) assert rec.width == 1920 assert rec.duration_seconds == 120.2 assert db_sync.execute( select(func.count()).select_from(ImageRecord) ).scalar_one() == 1 def test_attach_in_place_video_distinct_durations_not_merged(importer, db_sync, monkeypatch): """Guard against false-merge: two videos with clearly different durations stay separate even at the same artist + resolution.""" images_root = importer.images_root artist = Artist(name="Vid3", slug="vid3") db_sync.add(artist) db_sync.flush() _fake_video_probe(monkeypatch, { "a.mp4": (1920, 1080, 60.0), "b.mp4": (1920, 1080, 200.0), # 140s apart — different content }) a = _mkvideo(images_root, "vid3/p/a.mp4", b"AAAA") b = _mkvideo(images_root, "vid3/p/b.mp4", b"BBBB") assert importer.attach_in_place(a, artist=artist).status == "imported" assert importer.attach_in_place(b, artist=artist).status == "imported" assert db_sync.execute( select(func.count()).select_from(ImageRecord) ).scalar_one() == 2 def test_attach_in_place_invalid_image_returns_skipped(importer): images_root = importer.images_root bad = images_root / "fred" / "bad.jpg" bad.parent.mkdir(parents=True, exist_ok=True) bad.write_bytes(b"not a jpeg, just some bytes") result = importer.attach_in_place(bad) assert result.status == "skipped" assert result.skip_reason.value == "invalid_image" def test_attach_in_place_non_media_routes_to_attachment(importer, db_sync): """FC-2d-iii dispatch parity for the download path. A non-media file (.pdf, .txt etc.) downloaded by gallery-dl must become a PostAttachment, not bounce back as `skipped+invalid_image` — the latter flipped otherwise-successful runs to status="error" downstream. Operator-flagged 2026-06-02 (Lustria patreon OST zip).""" from backend.app.models import PostAttachment images_root = importer.images_root artist = Artist(name="Gus", slug="gus") db_sync.add(artist) db_sync.flush() txt = images_root / "gus" / "patreon" / "post" / "notes.txt" txt.parent.mkdir(parents=True, exist_ok=True) txt.write_bytes(b"some non-media payload that the importer should preserve") result = importer.attach_in_place(txt, artist=artist) assert result.status == "attached" row = db_sync.execute( select(PostAttachment).where(PostAttachment.original_filename == "notes.txt") ).scalar_one() assert row.ext == ".txt" assert row.artist_id == artist.id