From a510378603e42dda88201ca11be5ba20226a1ca1 Mon Sep 17 00:00:00 2001
From: Bryan Van Deusen <bryan.vandeusen@fabledsword.com>
Date: Sun, 17 May 2026 22:05:43 -0400
Subject: [PATCH] feat(phash): importer near-dup dispatch + in-place
 supersession (clear ML)

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
---
 backend/app/services/importer.py |  90 +++++++++++++++++++-
 tests/test_phash_dedup.py        | 138 +++++++++++++++++++++++++++++++
 2 files changed, 226 insertions(+), 2 deletions(-)
 create mode 100644 tests/test_phash_dedup.py

diff --git a/backend/app/services/importer.py b/backend/app/services/importer.py
index abc3754..c45e971 100644
--- a/backend/app/services/importer.py
+++ b/backend/app/services/importer.py
@@ -21,6 +21,7 @@ from sqlalchemy.orm import Session
 from ..models import Artist, ImageRecord, ImportSettings, Tag, TagKind
 from ..models.tag import image_tag
 from ..utils.paths import derive_subdir, derive_top_level_artist, hash_suffixed_name
+from ..utils.phash import compute_phash, find_similar
 from ..utils.slug import slugify
 from .thumbnailer import Thumbnailer
 
@@ -30,12 +31,13 @@ class SkipReason(StrEnum):
     too_transparent = "too_transparent"
     single_color = "single_color"
     duplicate_hash = "duplicate_hash"
+    duplicate_phash = "duplicate_phash"
     invalid_image = "invalid_image"
 
 
 @dataclass(frozen=True)
 class ImportResult:
-    status: str  # 'imported' | 'skipped' | 'failed'
+    status: str  # 'imported' | 'skipped' | 'failed' | 'superseded'
     image_id: int | None = None
     skip_reason: SkipReason | None = None
     error: str | None = None
@@ -152,7 +154,7 @@ class Importer:
                         error=f"{pct:.2%} transparent",
                     )
 
-        # Hash dedup.
+        # Hash dedup (exact).
         sha = _sha256_of(source)
         existing_stmt = select(ImageRecord).where(ImageRecord.sha256 == sha)
         existing = self.session.execute(existing_stmt).scalar_one_or_none()
@@ -162,6 +164,42 @@ class Importer:
                 image_id=existing.id, error="sha256 already present",
             )
 
+        # Perceptual near-dup (images only; videos keep phash NULL).
+        phash = None
+        if not is_video(source):
+            with Image.open(source) as im:
+                phash = compute_phash(im)
+            if phash is not None:
+                cand_rows = self.session.execute(
+                    select(
+                        ImageRecord.phash,
+                        ImageRecord.width,
+                        ImageRecord.height,
+                        ImageRecord.id,
+                    ).where(ImageRecord.phash.is_not(None))
+                ).all()
+                candidates = [
+                    (c.phash, c.width or 0, c.height or 0, c.id)
+                    for c in cand_rows
+                ]
+                rel, match_id = find_similar(
+                    phash, width or 0, height or 0,
+                    candidates, self.settings.phash_threshold,
+                )
+                if rel == "larger_exists":
+                    return ImportResult(
+                        status="skipped",
+                        skip_reason=SkipReason.duplicate_phash,
+                        image_id=match_id,
+                        error="perceptual near-duplicate of larger existing image",
+                    )
+                if rel == "smaller_exists":
+                    target = self.session.get(ImageRecord, match_id)
+                    self._supersede(target, source, sha, phash, width, height)
+                    return ImportResult(
+                        status="superseded", image_id=match_id
+                    )
+
         # Destination path.
         subdir = derive_subdir(source, self.import_root)
         dest_dir = self.images_root / subdir if subdir else self.images_root
@@ -177,6 +215,7 @@ class Importer:
         record = ImageRecord(
             path=str(dest),
             sha256=sha,
+            phash=phash,
             size_bytes=dest.stat().st_size,
             mime=_mime_for(source),
             width=width,
@@ -232,6 +271,53 @@ class Importer:
                 )
             )
 
+    def _supersede(
+        self, existing: ImageRecord, source: Path, sha: str,
+        phash: str, width: int | None, height: int | None,
+    ) -> None:
+        """Replace `existing`'s file with the larger `source`, keeping the
+        row id (so tags/series/curation stay attached). ML is cleared so
+        the import task re-derives it on the new pixels."""
+        subdir = derive_subdir(source, self.import_root)
+        dest_dir = self.images_root / subdir if subdir else self.images_root
+        dest_dir.mkdir(parents=True, exist_ok=True)
+        dest = dest_dir / hash_suffixed_name(source.stem, sha, source.suffix)
+        partial = dest.with_suffix(dest.suffix + ".partial")
+        shutil.copy2(source, partial)
+        partial.rename(dest)
+
+        old_path = existing.path
+        old_thumb = existing.thumbnail_path
+
+        existing.path = str(dest)
+        existing.sha256 = sha
+        existing.phash = phash
+        existing.size_bytes = dest.stat().st_size
+        existing.mime = _mime_for(source)
+        existing.width = width
+        existing.height = height
+        existing.thumbnail_path = None
+        existing.integrity_status = "unknown"
+        existing.tagger_predictions = None
+        existing.tagger_model_version = None
+        existing.siglip_embedding = None
+        existing.siglip_model_version = None
+        existing.centroid_scores = None
+        # created_at intentionally preserved; updated_at auto-bumps.
+        self.session.flush()
+        self.session.commit()
+
+        for stale in (old_path, old_thumb):
+            if not stale:
+                continue
+            try:
+                p = Path(stale)
+                if p.exists():
+                    p.unlink()
+            except Exception:
+                # Benign orphan; the DB swap already committed. Don't undo it.
+                pass
+
     def _transparency_pct(self, source: Path) -> float:
         """Fraction of fully-transparent pixels in the image. 0.0 if no alpha."""
         with Image.open(source) as im:
diff --git a/tests/test_phash_dedup.py b/tests/test_phash_dedup.py
new file mode 100644
index 0000000..d92ea78
--- /dev/null
+++ b/tests/test_phash_dedup.py
@@ -0,0 +1,138 @@
+"""pHash near-dup + supersession importer tests.
+
+Mirrors tests/test_importer.py exactly: the Importer is sync, driven via
+the db_sync savepoint Session; tests are plain `def` (not asyncio).
+"""
+
+from pathlib import Path
+
+import pytest
+from PIL import Image
+from sqlalchemy import func, select
+
+from backend.app.models import ImageRecord, ImportSettings, Tag, TagKind
+from backend.app.models.tag import image_tag
+from backend.app.services.importer import Importer, SkipReason
+from backend.app.services.thumbnailer import Thumbnailer
+
+pytestmark = pytest.mark.integration
+
+
+@pytest.fixture
+def import_layout(tmp_path):
+    import_root = tmp_path / "import"
+    images_root = tmp_path / "images"
+    import_root.mkdir()
+    images_root.mkdir()
+    return import_root, images_root
+
+
+@pytest.fixture
+def importer(db_sync, import_layout):
+    import_root, images_root = import_layout
+    settings = db_sync.execute(
+        select(ImportSettings).where(ImportSettings.id == 1)
+    ).scalar_one()
+    return Importer(
+        session=db_sync,
+        images_root=images_root,
+        import_root=import_root,
+        thumbnailer=Thumbnailer(images_root=images_root),
+        settings=settings,
+    )
+
+
+def _set_threshold(importer, n):
+    importer.settings.phash_threshold = n
+    importer.session.flush()
+
+
+def _write(path: Path, color, size):
+    path.parent.mkdir(parents=True, exist_ok=True)
+    Image.new("RGB", size, color).save(path)
+
+
+def test_non_similar_imports_with_phash(importer, import_layout):
+    import_root, _ = import_layout
+    src = import_root / "a.png"
+    _write(src, (12, 200, 60), (300, 300))
+    r = importer.import_one(src)
+    assert r.status == "imported"
+    row = importer.session.get(ImageRecord, r.image_id)
+    assert row.phash is not None and len(row.phash) == 16
+
+
+def test_larger_existing_skips_new_phash_dup(importer, import_layout):
+    import_root, _ = import_layout
+    big = import_root / "big.png"
+    _write(big, (40, 90, 160), (800, 800))
+    r1 = importer.import_one(big)
+    assert r1.status == "imported"
+    small = import_root / "small.png"
+    _write(small, (40, 90, 160), (200, 200))  # same look, smaller
+    r2 = importer.import_one(small)
+    assert r2.status == "skipped"
+    assert r2.skip_reason == SkipReason.duplicate_phash
+    assert r2.image_id == r1.image_id
+    count = importer.session.execute(
+        select(func.count()).select_from(ImageRecord)
+    ).scalar_one()
+    assert count == 1
+
+
+def test_smaller_existing_is_superseded(importer, import_layout):
+    import_root, _ = import_layout
+    small = import_root / "small.png"
+    _write(small, (200, 30, 30), (200, 200))
+    r1 = importer.import_one(small)
+    eid = r1.image_id
+    old = importer.session.get(ImageRecord, eid)
+    old_path = old.path
+    tag = Tag(name="keepme", kind=TagKind.general)
+    importer.session.add(tag)
+    importer.session.flush()
+    importer.session.execute(
+        image_tag.insert().values(
+            image_record_id=eid, tag_id=tag.id, source="manual"
+        )
+    )
+    old.tagger_predictions = {"x": 1}
+    old.siglip_embedding = [0.0] * 1152
+    old.integrity_status = "ok"
+    importer.session.commit()
+
+    big = import_root / "big.png"
+    _write(big, (200, 30, 30), (900, 900))  # same look, larger
+    r2 = importer.import_one(big)
+    assert r2.status == "superseded"
+    assert r2.image_id == eid
+
+    importer.session.expire_all()
+    row = importer.session.get(ImageRecord, eid)
+    assert row.id == eid
+    assert row.width == 900 and row.height == 900
+    assert row.path != old_path
+    assert row.phash is not None
+    assert row.integrity_status == "unknown"
+    assert row.tagger_predictions is None
+    assert row.siglip_embedding is None
+    linked = importer.session.execute(
+        select(image_tag.c.tag_id).where(
+            image_tag.c.image_record_id == eid
+        )
+    ).scalars().all()
+    assert tag.id in linked
+    assert not Path(old_path).exists()
+    assert Path(row.path).exists()
+
+
+def test_threshold_controls_match(importer, import_layout):
+    import_root, _ = import_layout
+    _set_threshold(importer, 0)
+    a = import_root / "a.png"
+    _write(a, (100, 100, 100), (200, 200))
+    importer.import_one(a)
+    b = import_root / "b.png"
+    _write(b, (104, 100, 100), (900, 900))  # tiny tweak
+    r = importer.import_one(b)
+    assert r.status == "imported"  # threshold 0 → not treated as near-dup