feat(deep-scan): Importer(deep=) re-derive on sha-match; import_file wires batch.scan_mode

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

backend/app/services/importer.py

@@ -117,12 +117,14 @@ class Importer:
         import_root: Path,
         thumbnailer: Thumbnailer,
         settings: ImportSettings,
+        deep: bool = False,
     ):
         self.session = session
         self.images_root = images_root
         self.import_root = import_root
         self.thumbnailer = thumbnailer
         self.settings = settings
+        self.deep = deep
         self.attachments = AttachmentStore(images_root)
 
     def import_one(self, source: Path) -> ImportResult:
@@ -290,10 +292,12 @@ class Importer:
         existing_stmt = select(ImageRecord).where(ImageRecord.sha256 == sha)
         existing = self.session.execute(existing_stmt).scalar_one_or_none()
         if existing:
-            return ImportResult(
-                status="skipped", skip_reason=SkipReason.duplicate_hash,
-                image_id=existing.id, error="sha256 already present",
-            )
+            if not self.deep:
+                return ImportResult(
+                    status="skipped", skip_reason=SkipReason.duplicate_hash,
+                    image_id=existing.id, error="sha256 already present",
+                )
+            return self._deep_rederive(existing, source, attribution_path)
 
         # Perceptual near-dup (images only; videos keep phash NULL).
         phash = None
@@ -374,6 +378,35 @@ class Importer:
         self.session.commit()
         return ImportResult(status="imported", image_id=record.id)
 
+    def _deep_rederive(
+        self, existing: ImageRecord, source: Path, attribution_path: Path
+    ) -> ImportResult:
+        """Deep scan: backfill phash/provenance/artist on an
+        already-imported record. METADATA ONLY — never re-runs the pHash
+        near-dup / supersede path. NULL-only, idempotent."""
+        if existing.phash is None and not is_video(source):
+            try:
+                with Image.open(source) as im:
+                    ph = compute_phash(im)
+                if ph is not None:
+                    existing.phash = ph
+            except Exception as exc:
+                log.warning("deep rephash failed for %s: %s", source, exc)
+
+        artist = None
+        name = derive_top_level_artist(attribution_path, self.import_root)
+        if name:
+            artist = self._upsert_artist(name)
+            if existing.artist_id is None:
+                existing.artist_id = artist.id
+
+        self._apply_sidecar(existing, attribution_path, artist)
+        self.session.commit()
+        return ImportResult(
+            status="skipped", skip_reason=SkipReason.duplicate_hash,
+            image_id=existing.id, error="deep: re-derived",
+        )
+
     def _upsert_artist(self, name: str) -> Artist:
         slug = slugify(name)
         artist = self.session.execute(

backend/app/tasks/import_file.py

@@ -55,12 +55,15 @@ def import_media_file(self, import_task_id: int) -> dict:
             select(ImportSettings).where(ImportSettings.id == 1)
         ).scalar_one()
         import_root = Path(settings.import_scan_path)
+        batch = session.get(ImportBatch, task.batch_id)
+        deep = bool(batch and batch.scan_mode == "deep")
         importer = Importer(
             session=session,
             images_root=IMAGES_ROOT,
             import_root=import_root,
             thumbnailer=Thumbnailer(images_root=IMAGES_ROOT),
             settings=settings,
+            deep=deep,
         )
 
         try:

tests/test_importer_deep.py

@@ -0,0 +1,99 @@
+"""FC-2d-vi: Importer(deep=True) re-derives on an already-imported row."""
+
+import json
+
+import pytest
+from PIL import Image
+from sqlalchemy import func, select
+
+from backend.app.models import (
+    ImageProvenance,
+    ImageRecord,
+    ImportSettings,
+    Post,
+)
+from backend.app.services.importer import Importer
+from backend.app.services.thumbnailer import Thumbnailer
+
+pytestmark = pytest.mark.integration
+
+
+@pytest.fixture
+def import_layout(tmp_path):
+    import_root = tmp_path / "import"
+    images_root = tmp_path / "images"
+    import_root.mkdir()
+    images_root.mkdir()
+    return import_root, images_root
+
+
+def _mk(db_sync, import_layout, *, deep):
+    import_root, images_root = import_layout
+    settings = db_sync.execute(
+        select(ImportSettings).where(ImportSettings.id == 1)
+    ).scalar_one()
+    return Importer(
+        session=db_sync, images_root=images_root, import_root=import_root,
+        thumbnailer=Thumbnailer(images_root=images_root), settings=settings,
+        deep=deep,
+    )
+
+
+def _img(path):
+    path.parent.mkdir(parents=True, exist_ok=True)
+    Image.new("RGB", (50, 50), (90, 30, 200)).save(path, "JPEG")
+
+
+def test_deep_rederives_phash_and_provenance(db_sync, import_layout):
+    import_root, _ = import_layout
+    src = import_root / "Mae" / "p.jpg"
+    _img(src)
+
+    # First import (quick). Images get a phash on import; null it +
+    # primary_post_id to simulate a pre-FC-2d-i+ii/v record, then add a
+    # sidecar to simulate provenance that didn't exist at first import.
+    quick = _mk(db_sync, import_layout, deep=False)
+    r1 = quick.import_one(src)
+    assert r1.status == "imported"
+    rec = db_sync.get(ImageRecord, r1.image_id)
+    rec.phash = None
+    rec.primary_post_id = None
+    db_sync.commit()
+    src.with_suffix(".jpg.json").write_text(json.dumps(
+        {"category": "patreon", "id": "9", "title": "P"}))
+
+    deep = _mk(db_sync, import_layout, deep=True)
+    r2 = deep.import_one(src)
+    assert r2.status == "skipped"
+    assert "deep" in (r2.error or "")
+    assert r2.image_id == rec.id
+
+    db_sync.expire_all()
+    rec2 = db_sync.get(ImageRecord, rec.id)
+    assert rec2.phash is not None
+    assert db_sync.execute(
+        select(func.count()).select_from(ImageRecord)
+    ).scalar_one() == 1  # no new record
+    post = db_sync.execute(select(Post)).scalar_one()
+    assert db_sync.execute(
+        select(func.count()).select_from(ImageProvenance)
+        .where(ImageProvenance.image_record_id == rec.id)
+    ).scalar_one() == 1
+    assert rec2.primary_post_id == post.id
+
+    # Idempotent: a second deep pass adds no duplicate provenance.
+    _mk(db_sync, import_layout, deep=True).import_one(src)
+    db_sync.expire_all()
+    assert db_sync.execute(
+        select(func.count()).select_from(ImageProvenance)
+    ).scalar_one() == 1
+
+
+def test_deep_false_is_plain_skip(db_sync, import_layout):
+    import_root, _ = import_layout
+    src = import_root / "Mae" / "q.jpg"
+    _img(src)
+    _mk(db_sync, import_layout, deep=False).import_one(src)
+    r = _mk(db_sync, import_layout, deep=False).import_one(src)
+    assert r.status == "skipped"
+    assert r.error == "sha256 already present"