feat(import): Tier-1 video near-dup by duration+aspect (#871)

Videos deduped on sha256 only (pHash is images-only), so a different encode/remux
of the same clip imported as a distinct record — the "same video from multiple
sources" clutter surfaced by #859.

Tier-1 metadata fingerprint: identity = container duration (±1.0s) + matching
aspect ratio, scoped to the same artist; quality axis = pixel dimensions (mirrors
image pHash: larger_exists→skip+link, smaller_exists→supersede). Codec/bitrate
are deliberately NOT part of identity (the point is matching across re-encodes).
Tight tolerances because a wrong video merge is destructive.

- image_record.duration_seconds (Float, nullable; migration 0052). NULL for images.
- safe_probe.probe_video also reads format=duration (one extra ffprobe field on the
  call that already runs); ProbeResult.duration.
- _find_similar_video(duration,w,h,artist) shared by both import pipelines.
- _import_media (filesystem/archive path): captures duration, video near-dup
  branch, persists duration.
- attach_in_place (download path — handles #859's videos, previously didn't probe
  video at all): best-effort probe for dims+duration (LENIENT — never newly rejects
  a downloaded video on probe failure), video near-dup branch, persists duration.
- _supersede carries duration onto the kept row.

Reuses SkipReason.duplicate_phash so the existing download/external dup-cleanup
(path-safe unlink, #859) applies unchanged. Tests: skip-smaller, supersede-larger
(+ duration adopted), and distinct-durations-not-merged (false-merge guard).

Follow-up (Phase 2, #871): a backfill to re-probe NULL-duration existing videos so
the current library participates in dedup; retroactive merge of existing dups is a
separate destructive maintenance action.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

alembic/versions/0052_image_duration_seconds.py

@@ -0,0 +1,32 @@
+"""image_record: duration_seconds (Tier-1 video near-dup key)
+
+#871. Videos previously deduped on sha256 only (pHash is images-only), so a
+different encode/remux of the same video imported as a distinct record. Persist
+the container duration so the importer can treat same-artist videos with matching
+duration (+ aspect ratio) as the same content and dedup/supersede like images.
+NULL for images and for video rows imported before this column existed (a
+backfill re-probes those so they participate in dedup).
+
+Revision ID: 0052
+Revises: 0051
+Create Date: 2026-06-16
+"""
+from typing import Sequence, Union
+
+import sqlalchemy as sa
+from alembic import op
+
+revision: str = "0052"
+down_revision: Union[str, None] = "0051"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    op.add_column(
+        "image_record", sa.Column("duration_seconds", sa.Float(), nullable=True)
+    )
+
+
+def downgrade() -> None:
+    op.drop_column("image_record", "duration_seconds")

backend/app/models/image_record.py

@@ -13,6 +13,7 @@ from sqlalchemy import (
     BigInteger,
     DateTime,
     Enum,
+    Float,
     ForeignKey,
     Integer,
     String,
@@ -39,6 +40,10 @@ class ImageRecord(Base):
     mime: Mapped[str] = mapped_column(String(64), nullable=False)
     width: Mapped[int | None] = mapped_column(Integer, nullable=True)
     height: Mapped[int | None] = mapped_column(Integer, nullable=True)
+    # Video container duration (seconds); NULL for images. The Tier-1 video
+    # near-dup key (#871): two videos of the same artist with matching duration
+    # (+ aspect) are the same content across re-encodes — dedup like image pHash.
+    duration_seconds: Mapped[float | None] = mapped_column(Float, nullable=True)
 
     # Integrity verification status. FC-2e populates this; FC-2a leaves rows at 'unknown'.
     # Values: 'unknown' (default), 'ok', 'corrupt', 'failed_verification'.

backend/app/services/importer.py

@@ -85,6 +85,15 @@ ALL_EXTS = IMAGE_EXTS | VIDEO_EXTS
 # deep-nested archive can't recurse forever; the bomb-probe re-runs per level.
 _ARCHIVE_MAX_DEPTH = 3
 
+# Tier-1 video near-dup (#871). pHash is images-only, so videos deduped on sha256
+# alone — a different encode/remux of the same clip imported as a distinct record.
+# Identity = container duration within a tight tolerance + matching aspect ratio,
+# scoped to the same artist (codec/bitrate are NOT part of identity — the point is
+# to match ACROSS re-encodes). Quality axis for supersede = pixel dimensions, same
+# as image pHash. Tight tolerances because a wrong video merge is destructive.
+_VIDEO_DUP_DURATION_TOL_SECONDS = 1.0
+_VIDEO_DUP_ASPECT_TOL = 0.02
+
 
 def is_supported(path: Path) -> bool:
     return path.suffix.lower() in ALL_EXTS
@@ -604,6 +613,48 @@ class Importer:
                 archive_path.name, depth, exc,
             )
 
+    @staticmethod
+    def _video_aspect_matches(w, h, cw, ch) -> bool:
+        """True when two (w,h) pairs share an aspect ratio within tolerance.
+        Missing dims → don't block (duration is the primary signal)."""
+        if not (w and h and cw and ch):
+            return True
+        return abs((w / h) - (cw / ch)) <= _VIDEO_DUP_ASPECT_TOL
+
+    def _find_similar_video(
+        self, duration: float | None, width: int | None, height: int | None,
+        artist_id: int | None,
+    ) -> tuple[str, int | None]:
+        """Tier-1 video near-dup (#871). Find a same-artist video whose duration
+        matches within tolerance AND whose aspect ratio matches — treat it as the
+        same content (a different encode/remux). Returns find_similar's contract:
+        ("none"|"larger_exists"|"smaller_exists", id), quality judged by pixel
+        dimensions so a higher-res copy supersedes a smaller one."""
+        if duration is None or artist_id is None:
+            return ("none", None)
+        lo = duration - _VIDEO_DUP_DURATION_TOL_SECONDS
+        hi = duration + _VIDEO_DUP_DURATION_TOL_SECONDS
+        rows = self.session.execute(
+            select(ImageRecord.id, ImageRecord.width, ImageRecord.height)
+            .where(
+                ImageRecord.artist_id == artist_id,
+                ImageRecord.mime.like("video/%"),
+                ImageRecord.duration_seconds.is_not(None),
+                ImageRecord.duration_seconds >= lo,
+                ImageRecord.duration_seconds <= hi,
+            )
+        ).all()
+        w, h = width or 0, height or 0
+        for cid, cw, ch in rows:
+            if not self._video_aspect_matches(width, height, cw, ch):
+                continue
+            cw0, ch0 = cw or 0, ch or 0
+            if cw0 >= w and ch0 >= h:
+                return ("larger_exists", cid)
+            if w > cw0 or h > ch0:
+                return ("smaller_exists", cid)
+        return ("none", None)
+
     def _import_media(
         self, source: Path, attribution_path: Path,
         *, explicit_source: Source | None = None,
@@ -622,13 +673,13 @@ class Importer:
 
         # Compute file dimensions (images only) and apply filters.
         width = height = None
+        duration = None  # video container duration (Tier-1 near-dup key, #871)
         has_alpha = False
         if is_video(source):
             # Layer-3 isolation: validate the container via ffprobe (a
             # separate process) before the rest of the pipeline touches
             # it. A corrupt video that would crash a decoder is rejected
-            # cleanly here, and we capture width/height for free (the
+            # cleanly here, and we capture width/height + duration for free.
-            # importer didn't previously record video dimensions).
             probe = safe_probe.probe_video(source)
             if not probe.ok:
                 if probe.crashed:
@@ -640,7 +691,7 @@ class Importer:
                     status="skipped", skip_reason=SkipReason.invalid_image,
                     error=probe.reason,
                 )
-            width, height = probe.width, probe.height
+            width, height, duration = probe.width, probe.height, probe.duration
         else:
             try:
                 with Image.open(source) as im:
@@ -752,6 +803,32 @@ class Importer:
                     return ImportResult(
                         status="superseded", image_id=match_id
                     )
+        elif duration is not None:
+            # Tier-1 video near-dup (#871): same-artist, matching duration+aspect
+            # → same content across re-encodes. Mirror the image flow.
+            rel, match_id = self._find_similar_video(
+                duration, width, height,
+                path_artist.id if path_artist else None,
+            )
+            if rel == "larger_exists":
+                larger = self.session.get(ImageRecord, match_id)
+                if larger is not None:
+                    self._apply_sidecar(
+                        larger, attribution_path, path_artist,
+                        explicit_source=explicit_source,
+                    )
+                    self.session.commit()
+                return ImportResult(
+                    status="skipped", skip_reason=SkipReason.duplicate_phash,
+                    image_id=match_id,
+                    error="video near-duplicate (duration+aspect) of existing equal/larger video",
+                )
+            if rel == "smaller_exists":
+                target = self.session.get(ImageRecord, match_id)
+                self._supersede(
+                    target, source, sha, None, width, height, duration=duration,
+                )
+                return ImportResult(status="superseded", image_id=match_id)
 
         dest = self._copy_to_library(source, sha, attribution_path)
 
@@ -763,6 +840,7 @@ class Importer:
             mime=_mime_for(source),
             width=width,
             height=height,
+            duration_seconds=duration,
             origin="imported_filesystem",
             integrity_status="unknown",
         )
@@ -956,6 +1034,7 @@ class Importer:
 
         # Format / dimension / transparency filters (mirror _import_media).
         width = height = None
+        duration = None  # video container duration (Tier-1 near-dup key, #871)
         has_alpha = False
         if not is_video(path):
             try:
@@ -987,6 +1066,14 @@ class Importer:
                         status="skipped", skip_reason=SkipReason.too_transparent,
                         error=f"{pct:.2%} transparent",
                     )
+        else:
+            # Best-effort probe for dims + duration so downloaded videos can dedup
+            # (#871). LENIENT: unlike _import_media this path does not reject on a
+            # probe failure — preserve the existing "import the downloaded video"
+            # behavior; we just skip dedup when we can't read the duration.
+            vp = safe_probe.probe_video(path)
+            if vp.ok:
+                width, height, duration = vp.width, vp.height, vp.duration
 
         # Hash dedup
         sha = _sha256_of(path)
@@ -1045,6 +1132,30 @@ class Importer:
                         new_path=path, artist=artist, source_row=source,
                     )
                     return ImportResult(status="superseded", image_id=match_id)
+        elif duration is not None:
+            # Tier-1 video near-dup (#871): same content re-downloaded from another
+            # source/encode. artist is the subscription artist (passed explicitly).
+            rel, match_id = self._find_similar_video(
+                duration, width, height, artist.id if artist else None,
+            )
+            if rel == "larger_exists":
+                larger = self.session.get(ImageRecord, match_id)
+                if larger is not None:
+                    self._apply_sidecar(larger, path, artist, explicit_source=source)
+                    self.session.commit()
+                return ImportResult(
+                    status="skipped", skip_reason=SkipReason.duplicate_phash,
+                    image_id=match_id,
+                    error="video near-duplicate (duration+aspect) of existing equal/larger video",
+                )
+            if rel == "smaller_exists":
+                target = self.session.get(ImageRecord, match_id)
+                self._supersede(
+                    target, path, sha, None, width, height,
+                    new_path=path, artist=artist, source_row=source,
+                    duration=duration,
+                )
+                return ImportResult(status="superseded", image_id=match_id)
 
         # Create record — the path IS where the file lives.
         record = ImageRecord(
@@ -1055,6 +1166,7 @@ class Importer:
             mime=_mime_for(path),
             width=width,
             height=height,
+            duration_seconds=duration,
             origin="downloaded",
             integrity_status="unknown",
         )
@@ -1270,6 +1382,7 @@ class Importer:
         *, new_path: Path | None = None,
         artist: Artist | None = None,
         source_row: Source | None = None,
+        duration: float | None = None,
     ) -> None:
         """Replace `existing`'s file with the larger `source`, keeping the
         row id (so tags/series/curation stay attached). ML is cleared so
@@ -1302,6 +1415,7 @@ class Importer:
         existing.mime = _mime_for(source)
         existing.width = width
         existing.height = height
+        existing.duration_seconds = duration  # #871: keep the kept copy's duration
         existing.thumbnail_path = None
         existing.integrity_status = "unknown"
         existing.tagger_model_version = None

backend/app/utils/safe_probe.py

@@ -60,6 +60,9 @@ class ProbeResult:
     reason: str | None = None
     width: int | None = None
     height: int | None = None
+    # Container duration in seconds (videos only) — the Tier-1 video near-dup
+    # key (#871). None when not a video / ffprobe didn't report it.
+    duration: float | None = None
 
 
 def probe_video(path: Path, *, timeout: float = VIDEO_PROBE_TIMEOUT_SECONDS) -> ProbeResult:
@@ -69,7 +72,10 @@ def probe_video(path: Path, *, timeout: float = VIDEO_PROBE_TIMEOUT_SECONDS) ->
             [
                 "ffprobe", "-v", "error",
                 "-select_streams", "v:0",
-                "-show_entries", "stream=width,height",
+                # format=duration is the reliable container duration (stream
+                # duration is often N/A for remuxed/streamed mp4) — the #871
+                # video near-dup key.
+                "-show_entries", "stream=width,height:format=duration",
                 "-of", "json", str(path),
             ],
             capture_output=True, text=True, timeout=timeout,
@@ -87,13 +93,21 @@ def probe_video(path: Path, *, timeout: float = VIDEO_PROBE_TIMEOUT_SECONDS) ->
             reason=f"ffprobe rejected the file: {out.stderr.strip()[:200]}",
         )
     try:
-        streams = (json.loads(out.stdout) or {}).get("streams") or []
+        parsed = json.loads(out.stdout) or {}
     except json.JSONDecodeError as exc:
         return ProbeResult(ok=False, crashed=False, reason=f"ffprobe output parse failed: {exc}")
+    streams = parsed.get("streams") or []
     if not streams:
         return ProbeResult(ok=False, crashed=False, reason="no decodable video stream")
+    duration = None
+    raw_dur = (parsed.get("format") or {}).get("duration")
+    try:
+        duration = float(raw_dur) if raw_dur is not None else None
+    except (TypeError, ValueError):
+        duration = None
     return ProbeResult(
         ok=True, width=streams[0].get("width"), height=streams[0].get("height"),
+        duration=duration,
     )
 
 

tests/test_importer_attach_in_place.py

@@ -3,11 +3,13 @@
 from pathlib import Path
 
 import pytest
-from sqlalchemy import select
+from sqlalchemy import func, select
 
 from backend.app.models import Artist, ImageRecord, ImportSettings
 from backend.app.services.importer import Importer
 from backend.app.services.thumbnailer import Thumbnailer
+from backend.app.utils import safe_probe
+from backend.app.utils.safe_probe import ProbeResult
 
 pytestmark = pytest.mark.integration
 
@@ -303,6 +305,100 @@ def test_attach_in_place_post_first_false_applies_post_fields(importer, db_sync)
     assert post.post_title == "Applies"
 
 
+def _fake_video_probe(monkeypatch, dims):
+    """Patch safe_probe.probe_video to return controlled (w,h,duration) keyed by
+    filename — so video dedup (#871) can be tested without real mp4s/ffprobe."""
+    def _probe(path, **_kw):
+        w, h, d = dims[Path(path).name]
+        return ProbeResult(ok=True, width=w, height=h, duration=d)
+    monkeypatch.setattr(safe_probe, "probe_video", _probe)
+
+
+def _mkvideo(images_root, rel, content):
+    p = images_root / rel
+    p.parent.mkdir(parents=True, exist_ok=True)
+    p.write_bytes(content)
+    return p
+
+
+def test_attach_in_place_video_dedup_skips_smaller(importer, db_sync, monkeypatch):
+    """#871 Tier-1: a same-artist video with matching duration+aspect but smaller
+    resolution is a near-dup of the existing larger video → skipped + linked, not
+    re-imported (the #859 'same video from multiple sources' case)."""
+    images_root = importer.images_root
+    artist = Artist(name="Vid", slug="vid")
+    db_sync.add(artist)
+    db_sync.flush()
+    _fake_video_probe(monkeypatch, {
+        "a.mp4": (1920, 1080, 300.0),
+        "b.mp4": (1280, 720, 300.4),  # within 1s, same 16:9 aspect, smaller
+    })
+
+    big = _mkvideo(images_root, "vid/patreon/p/a.mp4", b"AAAA")
+    r1 = importer.attach_in_place(big, artist=artist)
+    assert r1.status == "imported"
+
+    small = _mkvideo(images_root, "vid/patreon/p2/b.mp4", b"BBBB")
+    r2 = importer.attach_in_place(small, artist=artist)
+    assert r2.status == "skipped"
+    assert r2.skip_reason.value == "duplicate_phash"
+    assert r2.image_id == r1.image_id
+    assert db_sync.execute(
+        select(func.count()).select_from(ImageRecord)
+    ).scalar_one() == 1
+
+
+def test_attach_in_place_video_dedup_supersedes_larger(importer, db_sync, monkeypatch):
+    """A higher-res re-download supersedes the smaller existing video, keeping the
+    row id + adopting the new duration."""
+    images_root = importer.images_root
+    artist = Artist(name="Vid2", slug="vid2")
+    db_sync.add(artist)
+    db_sync.flush()
+    _fake_video_probe(monkeypatch, {
+        "s.mp4": (640, 360, 120.0),
+        "l.mp4": (1920, 1080, 120.2),  # within 1s, same aspect, larger
+    })
+
+    small = _mkvideo(images_root, "vid2/p/s.mp4", b"SSSS")
+    r1 = importer.attach_in_place(small, artist=artist)
+    assert r1.status == "imported"
+
+    big = _mkvideo(images_root, "vid2/p/l.mp4", b"LLLL")
+    r2 = importer.attach_in_place(big, artist=artist)
+    assert r2.status == "superseded"
+    assert r2.image_id == r1.image_id
+
+    db_sync.expire_all()
+    rec = db_sync.get(ImageRecord, r1.image_id)
+    assert rec.width == 1920
+    assert rec.duration_seconds == 120.2
+    assert db_sync.execute(
+        select(func.count()).select_from(ImageRecord)
+    ).scalar_one() == 1
+
+
+def test_attach_in_place_video_distinct_durations_not_merged(importer, db_sync, monkeypatch):
+    """Guard against false-merge: two videos with clearly different durations stay
+    separate even at the same artist + resolution."""
+    images_root = importer.images_root
+    artist = Artist(name="Vid3", slug="vid3")
+    db_sync.add(artist)
+    db_sync.flush()
+    _fake_video_probe(monkeypatch, {
+        "a.mp4": (1920, 1080, 60.0),
+        "b.mp4": (1920, 1080, 200.0),  # 140s apart — different content
+    })
+
+    a = _mkvideo(images_root, "vid3/p/a.mp4", b"AAAA")
+    b = _mkvideo(images_root, "vid3/p/b.mp4", b"BBBB")
+    assert importer.attach_in_place(a, artist=artist).status == "imported"
+    assert importer.attach_in_place(b, artist=artist).status == "imported"
+    assert db_sync.execute(
+        select(func.count()).select_from(ImageRecord)
+    ).scalar_one() == 2
+
+
 def test_attach_in_place_invalid_image_returns_skipped(importer):
     images_root = importer.images_root
     bad = images_root / "fred" / "bad.jpg"