bvandeusen
f154603811
Videos deduped on sha256 only (pHash is images-only), so a different encode/remux of the same clip imported as a distinct record — the "same video from multiple sources" clutter surfaced by #859. Tier-1 metadata fingerprint: identity = container duration (±1.0s) + matching aspect ratio, scoped to the same artist; quality axis = pixel dimensions (mirrors image pHash: larger_exists→skip+link, smaller_exists→supersede). Codec/bitrate are deliberately NOT part of identity (the point is matching across re-encodes). Tight tolerances because a wrong video merge is destructive. - image_record.duration_seconds (Float, nullable; migration 0052). NULL for images. - safe_probe.probe_video also reads format=duration (one extra ffprobe field on the call that already runs); ProbeResult.duration. - _find_similar_video(duration,w,h,artist) shared by both import pipelines. - _import_media (filesystem/archive path): captures duration, video near-dup branch, persists duration. - attach_in_place (download path — handles #859's videos, previously didn't probe video at all): best-effort probe for dims+duration (LENIENT — never newly rejects a downloaded video on probe failure), video near-dup branch, persists duration. - _supersede carries duration onto the kept row. Reuses SkipReason.duplicate_phash so the existing download/external dup-cleanup (path-safe unlink, #859) applies unchanged. Tests: skip-smaller, supersede-larger (+ duration adopted), and distinct-durations-not-merged (false-merge guard). Follow-up (Phase 2, #871): a backfill to re-probe NULL-duration existing videos so the current library participates in dedup; retroactive merge of existing dups is a separate destructive maintenance action. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
438 lines
16 KiB
Python
438 lines
16 KiB
Python
"""attach_in_place — Importer seam used by FC-3c (downloader)."""
|
|
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
from sqlalchemy import func, select
|
|
|
|
from backend.app.models import Artist, ImageRecord, ImportSettings
|
|
from backend.app.services.importer import Importer
|
|
from backend.app.services.thumbnailer import Thumbnailer
|
|
from backend.app.utils import safe_probe
|
|
from backend.app.utils.safe_probe import ProbeResult
|
|
|
|
pytestmark = pytest.mark.integration
|
|
|
|
|
|
def _make_jpg(path: Path, color=(255, 0, 0), size=(64, 64)) -> bytes:
|
|
from PIL import Image
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
Image.new("RGB", size, color).save(path, "JPEG")
|
|
return path.read_bytes()
|
|
|
|
|
|
@pytest.fixture
|
|
def importer(db_sync, tmp_path):
|
|
settings = db_sync.execute(
|
|
select(ImportSettings).where(ImportSettings.id == 1)
|
|
).scalar_one()
|
|
images_root = tmp_path / "images"
|
|
images_root.mkdir()
|
|
return Importer(
|
|
session=db_sync,
|
|
images_root=images_root,
|
|
import_root=tmp_path / "ignored",
|
|
thumbnailer=Thumbnailer(images_root=images_root),
|
|
settings=settings,
|
|
)
|
|
|
|
|
|
def test_attach_in_place_imports_happy(importer, db_sync):
|
|
images_root = importer.images_root
|
|
artist = Artist(name="Alice", slug="alice")
|
|
db_sync.add(artist)
|
|
db_sync.flush()
|
|
|
|
target = images_root / "alice" / "patreon" / "post1" / "img.jpg"
|
|
_make_jpg(target)
|
|
|
|
result = importer.attach_in_place(target, artist=artist)
|
|
assert result.status == "imported"
|
|
assert result.image_id is not None
|
|
|
|
rec_path = db_sync.execute(
|
|
select(ImageRecord.path, ImageRecord.origin, ImageRecord.artist_id)
|
|
.where(ImageRecord.id == result.image_id)
|
|
).one()
|
|
assert rec_path.path == str(target)
|
|
assert rec_path.origin == "downloaded"
|
|
assert rec_path.artist_id == artist.id
|
|
|
|
|
|
def test_attach_in_place_with_sidecar(importer, db_sync):
|
|
images_root = importer.images_root
|
|
artist = Artist(name="Bob", slug="bob")
|
|
db_sync.add(artist)
|
|
db_sync.flush()
|
|
|
|
target = images_root / "bob" / "patreon" / "post1" / "img.jpg"
|
|
_make_jpg(target)
|
|
sidecar = target.with_suffix(target.suffix + ".json")
|
|
sidecar.write_text(
|
|
'{"category": "patreon", "id": "12345", "url": "https://patreon.com/post/12345", '
|
|
'"title": "Test Post", "published_at": "2026-05-01T00:00:00+00:00"}'
|
|
)
|
|
|
|
result = importer.attach_in_place(target, artist=artist)
|
|
assert result.status == "imported"
|
|
|
|
primary_post_id = db_sync.execute(
|
|
select(ImageRecord.primary_post_id).where(ImageRecord.id == result.image_id)
|
|
).scalar_one()
|
|
assert primary_post_id is not None # post created from sidecar
|
|
|
|
|
|
def test_attach_in_place_sha256_dedup_does_not_modify_disk(importer, db_sync):
|
|
images_root = importer.images_root
|
|
artist = Artist(name="Cara", slug="cara")
|
|
db_sync.add(artist)
|
|
db_sync.flush()
|
|
|
|
first = images_root / "cara" / "p" / "a.jpg"
|
|
bytes_ = _make_jpg(first)
|
|
importer.attach_in_place(first, artist=artist)
|
|
|
|
second = images_root / "cara" / "p" / "b.jpg"
|
|
second.parent.mkdir(parents=True, exist_ok=True)
|
|
second.write_bytes(bytes_)
|
|
assert second.exists()
|
|
|
|
result = importer.attach_in_place(second, artist=artist)
|
|
assert result.status == "skipped"
|
|
assert result.skip_reason.value == "duplicate_hash"
|
|
# Caller is responsible for deleting the on-disk file; we don't touch it.
|
|
assert second.exists()
|
|
|
|
|
|
def test_attach_in_place_supersede_keeps_file_at_new_path(importer, db_sync):
|
|
"""phash near-dup where the new file is LARGER → existing record
|
|
swaps to the new path; old file deleted."""
|
|
images_root = importer.images_root
|
|
artist = Artist(name="Dee", slug="dee")
|
|
db_sync.add(artist)
|
|
db_sync.flush()
|
|
|
|
smaller = images_root / "dee" / "p" / "small.jpg"
|
|
_make_jpg(smaller, color=(50, 50, 200), size=(64, 64))
|
|
first = importer.attach_in_place(smaller, artist=artist)
|
|
assert first.status == "imported"
|
|
|
|
bigger = images_root / "dee" / "p" / "big.jpg"
|
|
_make_jpg(bigger, color=(50, 50, 200), size=(256, 256))
|
|
|
|
result = importer.attach_in_place(bigger, artist=artist)
|
|
assert result.status == "superseded"
|
|
|
|
new_path = db_sync.execute(
|
|
select(ImageRecord.path).where(ImageRecord.id == first.image_id)
|
|
).scalar_one()
|
|
assert new_path == str(bigger)
|
|
assert not smaller.exists()
|
|
assert bigger.exists()
|
|
|
|
|
|
def test_attach_in_place_phash_smaller_skips_without_modifying_disk(
|
|
importer, db_sync
|
|
):
|
|
images_root = importer.images_root
|
|
artist = Artist(name="Eve", slug="eve")
|
|
db_sync.add(artist)
|
|
db_sync.flush()
|
|
|
|
bigger = images_root / "eve" / "p" / "big.jpg"
|
|
_make_jpg(bigger, color=(0, 200, 0), size=(256, 256))
|
|
importer.attach_in_place(bigger, artist=artist)
|
|
|
|
smaller = images_root / "eve" / "p" / "small.jpg"
|
|
_make_jpg(smaller, color=(0, 200, 0), size=(64, 64))
|
|
|
|
result = importer.attach_in_place(smaller, artist=artist)
|
|
assert result.status == "skipped"
|
|
assert result.skip_reason.value == "duplicate_phash"
|
|
assert smaller.exists()
|
|
|
|
|
|
def test_attach_in_place_duplicate_image_links_both_posts(importer, db_sync):
|
|
"""A byte-identical image appearing in a SECOND post must link to BOTH posts
|
|
(enrich-on-duplicate) so it shows on every post — not be silently dropped,
|
|
which left the second post a bare shell. Operator-flagged 2026-06-08."""
|
|
from backend.app.models import ImageProvenance, Post
|
|
|
|
images_root = importer.images_root
|
|
artist = Artist(name="Hana", slug="hana")
|
|
db_sync.add(artist)
|
|
db_sync.flush()
|
|
|
|
first = images_root / "hana" / "patreon" / "post_a" / "img.jpg"
|
|
bytes_ = _make_jpg(first, color=(120, 40, 90))
|
|
first.with_suffix(first.suffix + ".json").write_text(
|
|
'{"category": "patreon", "id": "111", "title": "Post 111"}'
|
|
)
|
|
r1 = importer.attach_in_place(first, artist=artist)
|
|
assert r1.status == "imported"
|
|
|
|
second = images_root / "hana" / "patreon" / "post_b" / "img.jpg"
|
|
second.parent.mkdir(parents=True, exist_ok=True)
|
|
second.write_bytes(bytes_) # identical bytes → sha256 duplicate
|
|
second.with_suffix(second.suffix + ".json").write_text(
|
|
'{"category": "patreon", "id": "222", "title": "Post 222"}'
|
|
)
|
|
r2 = importer.attach_in_place(second, artist=artist)
|
|
assert r2.status == "skipped"
|
|
assert r2.skip_reason.value == "duplicate_hash"
|
|
assert r2.image_id == r1.image_id
|
|
|
|
linked = db_sync.execute(
|
|
select(Post.external_post_id)
|
|
.join(ImageProvenance, ImageProvenance.post_id == Post.id)
|
|
.where(ImageProvenance.image_record_id == r1.image_id)
|
|
).scalars().all()
|
|
assert set(linked) == {"111", "222"} # shows on BOTH posts
|
|
|
|
primary = db_sync.execute(
|
|
select(Post.external_post_id)
|
|
.join(ImageRecord, ImageRecord.primary_post_id == Post.id)
|
|
.where(ImageRecord.id == r1.image_id)
|
|
).scalar_one()
|
|
assert primary == "111" # primary stays on the first post
|
|
|
|
|
|
def test_attach_in_place_duplicate_attachment_links_both_posts(importer, db_sync):
|
|
"""The same non-art file attached to two posts gets a row on EACH post over a
|
|
single sha-addressed blob — neither post is left bare. Operator-flagged
|
|
2026-06-08 (1589 empty Anduo shells from globally-unique attachment sha)."""
|
|
from backend.app.models import Post, PostAttachment
|
|
|
|
images_root = importer.images_root
|
|
artist = Artist(name="Iris", slug="iris")
|
|
db_sync.add(artist)
|
|
db_sync.flush()
|
|
|
|
payload = b"the same readme creator attaches to every post"
|
|
a = images_root / "iris" / "patreon" / "post_a" / "readme.txt"
|
|
a.parent.mkdir(parents=True, exist_ok=True)
|
|
a.write_bytes(payload)
|
|
a.with_suffix(a.suffix + ".json").write_text(
|
|
'{"category": "patreon", "id": "301", "title": "Post 301"}'
|
|
)
|
|
assert importer.attach_in_place(a, artist=artist).status == "attached"
|
|
|
|
b = images_root / "iris" / "patreon" / "post_b" / "readme.txt"
|
|
b.parent.mkdir(parents=True, exist_ok=True)
|
|
b.write_bytes(payload)
|
|
b.with_suffix(b.suffix + ".json").write_text(
|
|
'{"category": "patreon", "id": "302", "title": "Post 302"}'
|
|
)
|
|
assert importer.attach_in_place(b, artist=artist).status == "attached"
|
|
|
|
linked = db_sync.execute(
|
|
select(Post.external_post_id)
|
|
.join(PostAttachment, PostAttachment.post_id == Post.id)
|
|
.where(PostAttachment.original_filename == "readme.txt")
|
|
).scalars().all()
|
|
assert set(linked) == {"301", "302"} # one row per post, both present
|
|
|
|
|
|
def test_attach_in_place_post_first_skips_post_fields(importer, db_sync):
|
|
"""Post-first (#856, milestone #67): with post_first set (native path), the
|
|
per-media import links provenance + localization (source_filehash) but does
|
|
NOT write the post body/title — the post-record owns those. Guards the import
|
|
side even if a body sneaks into the sidecar."""
|
|
from backend.app.models import ImageProvenance, Post
|
|
|
|
importer.post_first = True
|
|
images_root = importer.images_root
|
|
artist = Artist(name="Pat", slug="pat")
|
|
db_sync.add(artist)
|
|
db_sync.flush()
|
|
|
|
target = images_root / "pat" / "patreon" / "post1" / "img.jpg"
|
|
_make_jpg(target)
|
|
cdn = "https://cdn.patreon.com/" + "a" * 32 + "/x.jpg"
|
|
target.with_suffix(target.suffix + ".json").write_text(
|
|
'{"category": "patreon", "id": "900", "title": "Should NOT apply", '
|
|
'"content": "<p>body should NOT apply</p>", '
|
|
f'"source_url": "{cdn}"}}'
|
|
)
|
|
|
|
result = importer.attach_in_place(target, artist=artist)
|
|
assert result.status == "imported"
|
|
|
|
# Image-specific work STILL happens: localization filehash + provenance link.
|
|
rec = db_sync.execute(
|
|
select(ImageRecord).where(ImageRecord.id == result.image_id)
|
|
).scalar_one()
|
|
assert rec.source_filehash == "a" * 32
|
|
assert rec.primary_post_id is not None
|
|
|
|
post = db_sync.get(Post, rec.primary_post_id)
|
|
assert post.external_post_id == "900"
|
|
# Post-first: body/title NOT written by the media path (post-record owns them).
|
|
assert post.description is None
|
|
assert post.post_title is None
|
|
# Provenance link exists regardless of post_first.
|
|
prov = db_sync.execute(
|
|
select(ImageProvenance).where(
|
|
ImageProvenance.image_record_id == result.image_id
|
|
)
|
|
).scalars().all()
|
|
assert len(prov) == 1
|
|
|
|
|
|
def test_attach_in_place_post_first_false_applies_post_fields(importer, db_sync):
|
|
"""Default (gallery-dl path): the sidecar IS the body source, so post fields
|
|
ARE applied. Guards that the post_first flag actually gates this — not a no-op."""
|
|
from backend.app.models import Post
|
|
|
|
assert importer.post_first is False # default
|
|
images_root = importer.images_root
|
|
artist = Artist(name="Gal", slug="gal")
|
|
db_sync.add(artist)
|
|
db_sync.flush()
|
|
|
|
target = images_root / "gal" / "patreon" / "post1" / "img.jpg"
|
|
_make_jpg(target)
|
|
target.with_suffix(target.suffix + ".json").write_text(
|
|
'{"category": "patreon", "id": "901", "title": "Applies", '
|
|
'"content": "<p>body applies</p>"}'
|
|
)
|
|
|
|
result = importer.attach_in_place(target, artist=artist)
|
|
assert result.status == "imported"
|
|
rec = db_sync.get(ImageRecord, result.image_id)
|
|
post = db_sync.get(Post, rec.primary_post_id)
|
|
assert post.description == "<p>body applies</p>"
|
|
assert post.post_title == "Applies"
|
|
|
|
|
|
def _fake_video_probe(monkeypatch, dims):
|
|
"""Patch safe_probe.probe_video to return controlled (w,h,duration) keyed by
|
|
filename — so video dedup (#871) can be tested without real mp4s/ffprobe."""
|
|
def _probe(path, **_kw):
|
|
w, h, d = dims[Path(path).name]
|
|
return ProbeResult(ok=True, width=w, height=h, duration=d)
|
|
monkeypatch.setattr(safe_probe, "probe_video", _probe)
|
|
|
|
|
|
def _mkvideo(images_root, rel, content):
|
|
p = images_root / rel
|
|
p.parent.mkdir(parents=True, exist_ok=True)
|
|
p.write_bytes(content)
|
|
return p
|
|
|
|
|
|
def test_attach_in_place_video_dedup_skips_smaller(importer, db_sync, monkeypatch):
|
|
"""#871 Tier-1: a same-artist video with matching duration+aspect but smaller
|
|
resolution is a near-dup of the existing larger video → skipped + linked, not
|
|
re-imported (the #859 'same video from multiple sources' case)."""
|
|
images_root = importer.images_root
|
|
artist = Artist(name="Vid", slug="vid")
|
|
db_sync.add(artist)
|
|
db_sync.flush()
|
|
_fake_video_probe(monkeypatch, {
|
|
"a.mp4": (1920, 1080, 300.0),
|
|
"b.mp4": (1280, 720, 300.4), # within 1s, same 16:9 aspect, smaller
|
|
})
|
|
|
|
big = _mkvideo(images_root, "vid/patreon/p/a.mp4", b"AAAA")
|
|
r1 = importer.attach_in_place(big, artist=artist)
|
|
assert r1.status == "imported"
|
|
|
|
small = _mkvideo(images_root, "vid/patreon/p2/b.mp4", b"BBBB")
|
|
r2 = importer.attach_in_place(small, artist=artist)
|
|
assert r2.status == "skipped"
|
|
assert r2.skip_reason.value == "duplicate_phash"
|
|
assert r2.image_id == r1.image_id
|
|
assert db_sync.execute(
|
|
select(func.count()).select_from(ImageRecord)
|
|
).scalar_one() == 1
|
|
|
|
|
|
def test_attach_in_place_video_dedup_supersedes_larger(importer, db_sync, monkeypatch):
|
|
"""A higher-res re-download supersedes the smaller existing video, keeping the
|
|
row id + adopting the new duration."""
|
|
images_root = importer.images_root
|
|
artist = Artist(name="Vid2", slug="vid2")
|
|
db_sync.add(artist)
|
|
db_sync.flush()
|
|
_fake_video_probe(monkeypatch, {
|
|
"s.mp4": (640, 360, 120.0),
|
|
"l.mp4": (1920, 1080, 120.2), # within 1s, same aspect, larger
|
|
})
|
|
|
|
small = _mkvideo(images_root, "vid2/p/s.mp4", b"SSSS")
|
|
r1 = importer.attach_in_place(small, artist=artist)
|
|
assert r1.status == "imported"
|
|
|
|
big = _mkvideo(images_root, "vid2/p/l.mp4", b"LLLL")
|
|
r2 = importer.attach_in_place(big, artist=artist)
|
|
assert r2.status == "superseded"
|
|
assert r2.image_id == r1.image_id
|
|
|
|
db_sync.expire_all()
|
|
rec = db_sync.get(ImageRecord, r1.image_id)
|
|
assert rec.width == 1920
|
|
assert rec.duration_seconds == 120.2
|
|
assert db_sync.execute(
|
|
select(func.count()).select_from(ImageRecord)
|
|
).scalar_one() == 1
|
|
|
|
|
|
def test_attach_in_place_video_distinct_durations_not_merged(importer, db_sync, monkeypatch):
|
|
"""Guard against false-merge: two videos with clearly different durations stay
|
|
separate even at the same artist + resolution."""
|
|
images_root = importer.images_root
|
|
artist = Artist(name="Vid3", slug="vid3")
|
|
db_sync.add(artist)
|
|
db_sync.flush()
|
|
_fake_video_probe(monkeypatch, {
|
|
"a.mp4": (1920, 1080, 60.0),
|
|
"b.mp4": (1920, 1080, 200.0), # 140s apart — different content
|
|
})
|
|
|
|
a = _mkvideo(images_root, "vid3/p/a.mp4", b"AAAA")
|
|
b = _mkvideo(images_root, "vid3/p/b.mp4", b"BBBB")
|
|
assert importer.attach_in_place(a, artist=artist).status == "imported"
|
|
assert importer.attach_in_place(b, artist=artist).status == "imported"
|
|
assert db_sync.execute(
|
|
select(func.count()).select_from(ImageRecord)
|
|
).scalar_one() == 2
|
|
|
|
|
|
def test_attach_in_place_invalid_image_returns_skipped(importer):
|
|
images_root = importer.images_root
|
|
bad = images_root / "fred" / "bad.jpg"
|
|
bad.parent.mkdir(parents=True, exist_ok=True)
|
|
bad.write_bytes(b"not a jpeg, just some bytes")
|
|
|
|
result = importer.attach_in_place(bad)
|
|
assert result.status == "skipped"
|
|
assert result.skip_reason.value == "invalid_image"
|
|
|
|
|
|
def test_attach_in_place_non_media_routes_to_attachment(importer, db_sync):
|
|
"""FC-2d-iii dispatch parity for the download path. A non-media file
|
|
(.pdf, .txt etc.) downloaded by gallery-dl must become a PostAttachment,
|
|
not bounce back as `skipped+invalid_image` — the latter flipped
|
|
otherwise-successful runs to status="error" downstream. Operator-flagged
|
|
2026-06-02 (Lustria patreon OST zip)."""
|
|
from backend.app.models import PostAttachment
|
|
|
|
images_root = importer.images_root
|
|
artist = Artist(name="Gus", slug="gus")
|
|
db_sync.add(artist)
|
|
db_sync.flush()
|
|
|
|
txt = images_root / "gus" / "patreon" / "post" / "notes.txt"
|
|
txt.parent.mkdir(parents=True, exist_ok=True)
|
|
txt.write_bytes(b"some non-media payload that the importer should preserve")
|
|
|
|
result = importer.attach_in_place(txt, artist=artist)
|
|
assert result.status == "attached"
|
|
|
|
row = db_sync.execute(
|
|
select(PostAttachment).where(PostAttachment.original_filename == "notes.txt")
|
|
).scalar_one()
|
|
assert row.ext == ".txt"
|
|
assert row.artist_id == artist.id
|