diff --git a/backend/app/services/importer.py b/backend/app/services/importer.py index dab757a..88a77e0 100644 --- a/backend/app/services/importer.py +++ b/backend/app/services/importer.py @@ -117,12 +117,14 @@ class Importer: import_root: Path, thumbnailer: Thumbnailer, settings: ImportSettings, + deep: bool = False, ): self.session = session self.images_root = images_root self.import_root = import_root self.thumbnailer = thumbnailer self.settings = settings + self.deep = deep self.attachments = AttachmentStore(images_root) def import_one(self, source: Path) -> ImportResult: @@ -290,10 +292,12 @@ class Importer: existing_stmt = select(ImageRecord).where(ImageRecord.sha256 == sha) existing = self.session.execute(existing_stmt).scalar_one_or_none() if existing: - return ImportResult( - status="skipped", skip_reason=SkipReason.duplicate_hash, - image_id=existing.id, error="sha256 already present", - ) + if not self.deep: + return ImportResult( + status="skipped", skip_reason=SkipReason.duplicate_hash, + image_id=existing.id, error="sha256 already present", + ) + return self._deep_rederive(existing, source, attribution_path) # Perceptual near-dup (images only; videos keep phash NULL). phash = None @@ -374,6 +378,35 @@ class Importer: self.session.commit() return ImportResult(status="imported", image_id=record.id) + def _deep_rederive( + self, existing: ImageRecord, source: Path, attribution_path: Path + ) -> ImportResult: + """Deep scan: backfill phash/provenance/artist on an + already-imported record. METADATA ONLY — never re-runs the pHash + near-dup / supersede path. NULL-only, idempotent.""" + if existing.phash is None and not is_video(source): + try: + with Image.open(source) as im: + ph = compute_phash(im) + if ph is not None: + existing.phash = ph + except Exception as exc: + log.warning("deep rephash failed for %s: %s", source, exc) + + artist = None + name = derive_top_level_artist(attribution_path, self.import_root) + if name: + artist = self._upsert_artist(name) + if existing.artist_id is None: + existing.artist_id = artist.id + + self._apply_sidecar(existing, attribution_path, artist) + self.session.commit() + return ImportResult( + status="skipped", skip_reason=SkipReason.duplicate_hash, + image_id=existing.id, error="deep: re-derived", + ) + def _upsert_artist(self, name: str) -> Artist: slug = slugify(name) artist = self.session.execute( diff --git a/backend/app/tasks/import_file.py b/backend/app/tasks/import_file.py index c1469f0..d53de74 100644 --- a/backend/app/tasks/import_file.py +++ b/backend/app/tasks/import_file.py @@ -55,12 +55,15 @@ def import_media_file(self, import_task_id: int) -> dict: select(ImportSettings).where(ImportSettings.id == 1) ).scalar_one() import_root = Path(settings.import_scan_path) + batch = session.get(ImportBatch, task.batch_id) + deep = bool(batch and batch.scan_mode == "deep") importer = Importer( session=session, images_root=IMAGES_ROOT, import_root=import_root, thumbnailer=Thumbnailer(images_root=IMAGES_ROOT), settings=settings, + deep=deep, ) try: diff --git a/tests/test_importer_deep.py b/tests/test_importer_deep.py new file mode 100644 index 0000000..b94f722 --- /dev/null +++ b/tests/test_importer_deep.py @@ -0,0 +1,99 @@ +"""FC-2d-vi: Importer(deep=True) re-derives on an already-imported row.""" + +import json + +import pytest +from PIL import Image +from sqlalchemy import func, select + +from backend.app.models import ( + ImageProvenance, + ImageRecord, + ImportSettings, + Post, +) +from backend.app.services.importer import Importer +from backend.app.services.thumbnailer import Thumbnailer + +pytestmark = pytest.mark.integration + + +@pytest.fixture +def import_layout(tmp_path): + import_root = tmp_path / "import" + images_root = tmp_path / "images" + import_root.mkdir() + images_root.mkdir() + return import_root, images_root + + +def _mk(db_sync, import_layout, *, deep): + import_root, images_root = import_layout + settings = db_sync.execute( + select(ImportSettings).where(ImportSettings.id == 1) + ).scalar_one() + return Importer( + session=db_sync, images_root=images_root, import_root=import_root, + thumbnailer=Thumbnailer(images_root=images_root), settings=settings, + deep=deep, + ) + + +def _img(path): + path.parent.mkdir(parents=True, exist_ok=True) + Image.new("RGB", (50, 50), (90, 30, 200)).save(path, "JPEG") + + +def test_deep_rederives_phash_and_provenance(db_sync, import_layout): + import_root, _ = import_layout + src = import_root / "Mae" / "p.jpg" + _img(src) + + # First import (quick). Images get a phash on import; null it + + # primary_post_id to simulate a pre-FC-2d-i+ii/v record, then add a + # sidecar to simulate provenance that didn't exist at first import. + quick = _mk(db_sync, import_layout, deep=False) + r1 = quick.import_one(src) + assert r1.status == "imported" + rec = db_sync.get(ImageRecord, r1.image_id) + rec.phash = None + rec.primary_post_id = None + db_sync.commit() + src.with_suffix(".jpg.json").write_text(json.dumps( + {"category": "patreon", "id": "9", "title": "P"})) + + deep = _mk(db_sync, import_layout, deep=True) + r2 = deep.import_one(src) + assert r2.status == "skipped" + assert "deep" in (r2.error or "") + assert r2.image_id == rec.id + + db_sync.expire_all() + rec2 = db_sync.get(ImageRecord, rec.id) + assert rec2.phash is not None + assert db_sync.execute( + select(func.count()).select_from(ImageRecord) + ).scalar_one() == 1 # no new record + post = db_sync.execute(select(Post)).scalar_one() + assert db_sync.execute( + select(func.count()).select_from(ImageProvenance) + .where(ImageProvenance.image_record_id == rec.id) + ).scalar_one() == 1 + assert rec2.primary_post_id == post.id + + # Idempotent: a second deep pass adds no duplicate provenance. + _mk(db_sync, import_layout, deep=True).import_one(src) + db_sync.expire_all() + assert db_sync.execute( + select(func.count()).select_from(ImageProvenance) + ).scalar_one() == 1 + + +def test_deep_false_is_plain_skip(db_sync, import_layout): + import_root, _ = import_layout + src = import_root / "Mae" / "q.jpg" + _img(src) + _mk(db_sync, import_layout, deep=False).import_one(src) + r = _mk(db_sync, import_layout, deep=False).import_one(src) + assert r.status == "skipped" + assert r.error == "sha256 already present"