From f154603811ad483555e8a12cc3302bfc3994eb08 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Mon, 15 Jun 2026 22:17:36 -0400 Subject: [PATCH] feat(import): Tier-1 video near-dup by duration+aspect (#871) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Videos deduped on sha256 only (pHash is images-only), so a different encode/remux of the same clip imported as a distinct record — the "same video from multiple sources" clutter surfaced by #859. Tier-1 metadata fingerprint: identity = container duration (±1.0s) + matching aspect ratio, scoped to the same artist; quality axis = pixel dimensions (mirrors image pHash: larger_exists→skip+link, smaller_exists→supersede). Codec/bitrate are deliberately NOT part of identity (the point is matching across re-encodes). Tight tolerances because a wrong video merge is destructive. - image_record.duration_seconds (Float, nullable; migration 0052). NULL for images. - safe_probe.probe_video also reads format=duration (one extra ffprobe field on the call that already runs); ProbeResult.duration. - _find_similar_video(duration,w,h,artist) shared by both import pipelines. - _import_media (filesystem/archive path): captures duration, video near-dup branch, persists duration. - attach_in_place (download path — handles #859's videos, previously didn't probe video at all): best-effort probe for dims+duration (LENIENT — never newly rejects a downloaded video on probe failure), video near-dup branch, persists duration. - _supersede carries duration onto the kept row. Reuses SkipReason.duplicate_phash so the existing download/external dup-cleanup (path-safe unlink, #859) applies unchanged. Tests: skip-smaller, supersede-larger (+ duration adopted), and distinct-durations-not-merged (false-merge guard). Follow-up (Phase 2, #871): a backfill to re-probe NULL-duration existing videos so the current library participates in dedup; retroactive merge of existing dups is a separate destructive maintenance action. Co-Authored-By: Claude Opus 4.8 --- .../versions/0052_image_duration_seconds.py | 32 +++++ backend/app/models/image_record.py | 5 + backend/app/services/importer.py | 120 +++++++++++++++++- backend/app/utils/safe_probe.py | 18 ++- tests/test_importer_attach_in_place.py | 98 +++++++++++++- 5 files changed, 267 insertions(+), 6 deletions(-) create mode 100644 alembic/versions/0052_image_duration_seconds.py diff --git a/alembic/versions/0052_image_duration_seconds.py b/alembic/versions/0052_image_duration_seconds.py new file mode 100644 index 0000000..ec2a180 --- /dev/null +++ b/alembic/versions/0052_image_duration_seconds.py @@ -0,0 +1,32 @@ +"""image_record: duration_seconds (Tier-1 video near-dup key) + +#871. Videos previously deduped on sha256 only (pHash is images-only), so a +different encode/remux of the same video imported as a distinct record. Persist +the container duration so the importer can treat same-artist videos with matching +duration (+ aspect ratio) as the same content and dedup/supersede like images. +NULL for images and for video rows imported before this column existed (a +backfill re-probes those so they participate in dedup). + +Revision ID: 0052 +Revises: 0051 +Create Date: 2026-06-16 +""" +from typing import Sequence, Union + +import sqlalchemy as sa +from alembic import op + +revision: str = "0052" +down_revision: Union[str, None] = "0051" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.add_column( + "image_record", sa.Column("duration_seconds", sa.Float(), nullable=True) + ) + + +def downgrade() -> None: + op.drop_column("image_record", "duration_seconds") diff --git a/backend/app/models/image_record.py b/backend/app/models/image_record.py index 07cb0ec..b3ba120 100644 --- a/backend/app/models/image_record.py +++ b/backend/app/models/image_record.py @@ -13,6 +13,7 @@ from sqlalchemy import ( BigInteger, DateTime, Enum, + Float, ForeignKey, Integer, String, @@ -39,6 +40,10 @@ class ImageRecord(Base): mime: Mapped[str] = mapped_column(String(64), nullable=False) width: Mapped[int | None] = mapped_column(Integer, nullable=True) height: Mapped[int | None] = mapped_column(Integer, nullable=True) + # Video container duration (seconds); NULL for images. The Tier-1 video + # near-dup key (#871): two videos of the same artist with matching duration + # (+ aspect) are the same content across re-encodes — dedup like image pHash. + duration_seconds: Mapped[float | None] = mapped_column(Float, nullable=True) # Integrity verification status. FC-2e populates this; FC-2a leaves rows at 'unknown'. # Values: 'unknown' (default), 'ok', 'corrupt', 'failed_verification'. diff --git a/backend/app/services/importer.py b/backend/app/services/importer.py index fa81129..b8e4e29 100644 --- a/backend/app/services/importer.py +++ b/backend/app/services/importer.py @@ -85,6 +85,15 @@ ALL_EXTS = IMAGE_EXTS | VIDEO_EXTS # deep-nested archive can't recurse forever; the bomb-probe re-runs per level. _ARCHIVE_MAX_DEPTH = 3 +# Tier-1 video near-dup (#871). pHash is images-only, so videos deduped on sha256 +# alone — a different encode/remux of the same clip imported as a distinct record. +# Identity = container duration within a tight tolerance + matching aspect ratio, +# scoped to the same artist (codec/bitrate are NOT part of identity — the point is +# to match ACROSS re-encodes). Quality axis for supersede = pixel dimensions, same +# as image pHash. Tight tolerances because a wrong video merge is destructive. +_VIDEO_DUP_DURATION_TOL_SECONDS = 1.0 +_VIDEO_DUP_ASPECT_TOL = 0.02 + def is_supported(path: Path) -> bool: return path.suffix.lower() in ALL_EXTS @@ -604,6 +613,48 @@ class Importer: archive_path.name, depth, exc, ) + @staticmethod + def _video_aspect_matches(w, h, cw, ch) -> bool: + """True when two (w,h) pairs share an aspect ratio within tolerance. + Missing dims → don't block (duration is the primary signal).""" + if not (w and h and cw and ch): + return True + return abs((w / h) - (cw / ch)) <= _VIDEO_DUP_ASPECT_TOL + + def _find_similar_video( + self, duration: float | None, width: int | None, height: int | None, + artist_id: int | None, + ) -> tuple[str, int | None]: + """Tier-1 video near-dup (#871). Find a same-artist video whose duration + matches within tolerance AND whose aspect ratio matches — treat it as the + same content (a different encode/remux). Returns find_similar's contract: + ("none"|"larger_exists"|"smaller_exists", id), quality judged by pixel + dimensions so a higher-res copy supersedes a smaller one.""" + if duration is None or artist_id is None: + return ("none", None) + lo = duration - _VIDEO_DUP_DURATION_TOL_SECONDS + hi = duration + _VIDEO_DUP_DURATION_TOL_SECONDS + rows = self.session.execute( + select(ImageRecord.id, ImageRecord.width, ImageRecord.height) + .where( + ImageRecord.artist_id == artist_id, + ImageRecord.mime.like("video/%"), + ImageRecord.duration_seconds.is_not(None), + ImageRecord.duration_seconds >= lo, + ImageRecord.duration_seconds <= hi, + ) + ).all() + w, h = width or 0, height or 0 + for cid, cw, ch in rows: + if not self._video_aspect_matches(width, height, cw, ch): + continue + cw0, ch0 = cw or 0, ch or 0 + if cw0 >= w and ch0 >= h: + return ("larger_exists", cid) + if w > cw0 or h > ch0: + return ("smaller_exists", cid) + return ("none", None) + def _import_media( self, source: Path, attribution_path: Path, *, explicit_source: Source | None = None, @@ -622,13 +673,13 @@ class Importer: # Compute file dimensions (images only) and apply filters. width = height = None + duration = None # video container duration (Tier-1 near-dup key, #871) has_alpha = False if is_video(source): # Layer-3 isolation: validate the container via ffprobe (a # separate process) before the rest of the pipeline touches # it. A corrupt video that would crash a decoder is rejected - # cleanly here, and we capture width/height for free (the - # importer didn't previously record video dimensions). + # cleanly here, and we capture width/height + duration for free. probe = safe_probe.probe_video(source) if not probe.ok: if probe.crashed: @@ -640,7 +691,7 @@ class Importer: status="skipped", skip_reason=SkipReason.invalid_image, error=probe.reason, ) - width, height = probe.width, probe.height + width, height, duration = probe.width, probe.height, probe.duration else: try: with Image.open(source) as im: @@ -752,6 +803,32 @@ class Importer: return ImportResult( status="superseded", image_id=match_id ) + elif duration is not None: + # Tier-1 video near-dup (#871): same-artist, matching duration+aspect + # → same content across re-encodes. Mirror the image flow. + rel, match_id = self._find_similar_video( + duration, width, height, + path_artist.id if path_artist else None, + ) + if rel == "larger_exists": + larger = self.session.get(ImageRecord, match_id) + if larger is not None: + self._apply_sidecar( + larger, attribution_path, path_artist, + explicit_source=explicit_source, + ) + self.session.commit() + return ImportResult( + status="skipped", skip_reason=SkipReason.duplicate_phash, + image_id=match_id, + error="video near-duplicate (duration+aspect) of existing equal/larger video", + ) + if rel == "smaller_exists": + target = self.session.get(ImageRecord, match_id) + self._supersede( + target, source, sha, None, width, height, duration=duration, + ) + return ImportResult(status="superseded", image_id=match_id) dest = self._copy_to_library(source, sha, attribution_path) @@ -763,6 +840,7 @@ class Importer: mime=_mime_for(source), width=width, height=height, + duration_seconds=duration, origin="imported_filesystem", integrity_status="unknown", ) @@ -956,6 +1034,7 @@ class Importer: # Format / dimension / transparency filters (mirror _import_media). width = height = None + duration = None # video container duration (Tier-1 near-dup key, #871) has_alpha = False if not is_video(path): try: @@ -987,6 +1066,14 @@ class Importer: status="skipped", skip_reason=SkipReason.too_transparent, error=f"{pct:.2%} transparent", ) + else: + # Best-effort probe for dims + duration so downloaded videos can dedup + # (#871). LENIENT: unlike _import_media this path does not reject on a + # probe failure — preserve the existing "import the downloaded video" + # behavior; we just skip dedup when we can't read the duration. + vp = safe_probe.probe_video(path) + if vp.ok: + width, height, duration = vp.width, vp.height, vp.duration # Hash dedup sha = _sha256_of(path) @@ -1045,6 +1132,30 @@ class Importer: new_path=path, artist=artist, source_row=source, ) return ImportResult(status="superseded", image_id=match_id) + elif duration is not None: + # Tier-1 video near-dup (#871): same content re-downloaded from another + # source/encode. artist is the subscription artist (passed explicitly). + rel, match_id = self._find_similar_video( + duration, width, height, artist.id if artist else None, + ) + if rel == "larger_exists": + larger = self.session.get(ImageRecord, match_id) + if larger is not None: + self._apply_sidecar(larger, path, artist, explicit_source=source) + self.session.commit() + return ImportResult( + status="skipped", skip_reason=SkipReason.duplicate_phash, + image_id=match_id, + error="video near-duplicate (duration+aspect) of existing equal/larger video", + ) + if rel == "smaller_exists": + target = self.session.get(ImageRecord, match_id) + self._supersede( + target, path, sha, None, width, height, + new_path=path, artist=artist, source_row=source, + duration=duration, + ) + return ImportResult(status="superseded", image_id=match_id) # Create record — the path IS where the file lives. record = ImageRecord( @@ -1055,6 +1166,7 @@ class Importer: mime=_mime_for(path), width=width, height=height, + duration_seconds=duration, origin="downloaded", integrity_status="unknown", ) @@ -1270,6 +1382,7 @@ class Importer: *, new_path: Path | None = None, artist: Artist | None = None, source_row: Source | None = None, + duration: float | None = None, ) -> None: """Replace `existing`'s file with the larger `source`, keeping the row id (so tags/series/curation stay attached). ML is cleared so @@ -1302,6 +1415,7 @@ class Importer: existing.mime = _mime_for(source) existing.width = width existing.height = height + existing.duration_seconds = duration # #871: keep the kept copy's duration existing.thumbnail_path = None existing.integrity_status = "unknown" existing.tagger_model_version = None diff --git a/backend/app/utils/safe_probe.py b/backend/app/utils/safe_probe.py index beb0bc1..4b1af6c 100644 --- a/backend/app/utils/safe_probe.py +++ b/backend/app/utils/safe_probe.py @@ -60,6 +60,9 @@ class ProbeResult: reason: str | None = None width: int | None = None height: int | None = None + # Container duration in seconds (videos only) — the Tier-1 video near-dup + # key (#871). None when not a video / ffprobe didn't report it. + duration: float | None = None def probe_video(path: Path, *, timeout: float = VIDEO_PROBE_TIMEOUT_SECONDS) -> ProbeResult: @@ -69,7 +72,10 @@ def probe_video(path: Path, *, timeout: float = VIDEO_PROBE_TIMEOUT_SECONDS) -> [ "ffprobe", "-v", "error", "-select_streams", "v:0", - "-show_entries", "stream=width,height", + # format=duration is the reliable container duration (stream + # duration is often N/A for remuxed/streamed mp4) — the #871 + # video near-dup key. + "-show_entries", "stream=width,height:format=duration", "-of", "json", str(path), ], capture_output=True, text=True, timeout=timeout, @@ -87,13 +93,21 @@ def probe_video(path: Path, *, timeout: float = VIDEO_PROBE_TIMEOUT_SECONDS) -> reason=f"ffprobe rejected the file: {out.stderr.strip()[:200]}", ) try: - streams = (json.loads(out.stdout) or {}).get("streams") or [] + parsed = json.loads(out.stdout) or {} except json.JSONDecodeError as exc: return ProbeResult(ok=False, crashed=False, reason=f"ffprobe output parse failed: {exc}") + streams = parsed.get("streams") or [] if not streams: return ProbeResult(ok=False, crashed=False, reason="no decodable video stream") + duration = None + raw_dur = (parsed.get("format") or {}).get("duration") + try: + duration = float(raw_dur) if raw_dur is not None else None + except (TypeError, ValueError): + duration = None return ProbeResult( ok=True, width=streams[0].get("width"), height=streams[0].get("height"), + duration=duration, ) diff --git a/tests/test_importer_attach_in_place.py b/tests/test_importer_attach_in_place.py index f12c653..ab08a06 100644 --- a/tests/test_importer_attach_in_place.py +++ b/tests/test_importer_attach_in_place.py @@ -3,11 +3,13 @@ from pathlib import Path import pytest -from sqlalchemy import select +from sqlalchemy import func, select from backend.app.models import Artist, ImageRecord, ImportSettings from backend.app.services.importer import Importer from backend.app.services.thumbnailer import Thumbnailer +from backend.app.utils import safe_probe +from backend.app.utils.safe_probe import ProbeResult pytestmark = pytest.mark.integration @@ -303,6 +305,100 @@ def test_attach_in_place_post_first_false_applies_post_fields(importer, db_sync) assert post.post_title == "Applies" +def _fake_video_probe(monkeypatch, dims): + """Patch safe_probe.probe_video to return controlled (w,h,duration) keyed by + filename — so video dedup (#871) can be tested without real mp4s/ffprobe.""" + def _probe(path, **_kw): + w, h, d = dims[Path(path).name] + return ProbeResult(ok=True, width=w, height=h, duration=d) + monkeypatch.setattr(safe_probe, "probe_video", _probe) + + +def _mkvideo(images_root, rel, content): + p = images_root / rel + p.parent.mkdir(parents=True, exist_ok=True) + p.write_bytes(content) + return p + + +def test_attach_in_place_video_dedup_skips_smaller(importer, db_sync, monkeypatch): + """#871 Tier-1: a same-artist video with matching duration+aspect but smaller + resolution is a near-dup of the existing larger video → skipped + linked, not + re-imported (the #859 'same video from multiple sources' case).""" + images_root = importer.images_root + artist = Artist(name="Vid", slug="vid") + db_sync.add(artist) + db_sync.flush() + _fake_video_probe(monkeypatch, { + "a.mp4": (1920, 1080, 300.0), + "b.mp4": (1280, 720, 300.4), # within 1s, same 16:9 aspect, smaller + }) + + big = _mkvideo(images_root, "vid/patreon/p/a.mp4", b"AAAA") + r1 = importer.attach_in_place(big, artist=artist) + assert r1.status == "imported" + + small = _mkvideo(images_root, "vid/patreon/p2/b.mp4", b"BBBB") + r2 = importer.attach_in_place(small, artist=artist) + assert r2.status == "skipped" + assert r2.skip_reason.value == "duplicate_phash" + assert r2.image_id == r1.image_id + assert db_sync.execute( + select(func.count()).select_from(ImageRecord) + ).scalar_one() == 1 + + +def test_attach_in_place_video_dedup_supersedes_larger(importer, db_sync, monkeypatch): + """A higher-res re-download supersedes the smaller existing video, keeping the + row id + adopting the new duration.""" + images_root = importer.images_root + artist = Artist(name="Vid2", slug="vid2") + db_sync.add(artist) + db_sync.flush() + _fake_video_probe(monkeypatch, { + "s.mp4": (640, 360, 120.0), + "l.mp4": (1920, 1080, 120.2), # within 1s, same aspect, larger + }) + + small = _mkvideo(images_root, "vid2/p/s.mp4", b"SSSS") + r1 = importer.attach_in_place(small, artist=artist) + assert r1.status == "imported" + + big = _mkvideo(images_root, "vid2/p/l.mp4", b"LLLL") + r2 = importer.attach_in_place(big, artist=artist) + assert r2.status == "superseded" + assert r2.image_id == r1.image_id + + db_sync.expire_all() + rec = db_sync.get(ImageRecord, r1.image_id) + assert rec.width == 1920 + assert rec.duration_seconds == 120.2 + assert db_sync.execute( + select(func.count()).select_from(ImageRecord) + ).scalar_one() == 1 + + +def test_attach_in_place_video_distinct_durations_not_merged(importer, db_sync, monkeypatch): + """Guard against false-merge: two videos with clearly different durations stay + separate even at the same artist + resolution.""" + images_root = importer.images_root + artist = Artist(name="Vid3", slug="vid3") + db_sync.add(artist) + db_sync.flush() + _fake_video_probe(monkeypatch, { + "a.mp4": (1920, 1080, 60.0), + "b.mp4": (1920, 1080, 200.0), # 140s apart — different content + }) + + a = _mkvideo(images_root, "vid3/p/a.mp4", b"AAAA") + b = _mkvideo(images_root, "vid3/p/b.mp4", b"BBBB") + assert importer.attach_in_place(a, artist=artist).status == "imported" + assert importer.attach_in_place(b, artist=artist).status == "imported" + assert db_sync.execute( + select(func.count()).select_from(ImageRecord) + ).scalar_one() == 2 + + def test_attach_in_place_invalid_image_returns_skipped(importer): images_root = importer.images_root bad = images_root / "fred" / "bad.jpg"