FabledCurator/tests/test_phash_dedup.py

"""pHash near-dup + supersession importer tests.

Mirrors tests/test_importer.py exactly: the Importer is sync, driven via
the db_sync savepoint Session; tests are plain `def` (not asyncio).
"""

from pathlib import Path

import pytest
from PIL import Image
from sqlalchemy import func, select

from backend.app.models import (
    ImagePrediction,
    ImageProvenance,
    ImageRecord,
    ImportSettings,
    Post,
    Tag,
    TagKind,
)
from backend.app.models.tag import image_tag
from backend.app.services.importer import Importer, SkipReason
from backend.app.services.thumbnailer import Thumbnailer

pytestmark = pytest.mark.integration


@pytest.fixture
def import_layout(tmp_path):
    import_root = tmp_path / "import"
    images_root = tmp_path / "images"
    import_root.mkdir()
    images_root.mkdir()
    return import_root, images_root


@pytest.fixture
def importer(db_sync, import_layout):
    import_root, images_root = import_layout
    settings = db_sync.execute(
        select(ImportSettings).where(ImportSettings.id == 1)
    ).scalar_one()
    return Importer(
        session=db_sync,
        images_root=images_root,
        import_root=import_root,
        thumbnailer=Thumbnailer(images_root=images_root),
        settings=settings,
    )


def _set_threshold(importer, n):
    importer.settings.phash_threshold = n
    importer.session.flush()


def _write(path: Path, color, size):
    path.parent.mkdir(parents=True, exist_ok=True)
    Image.new("RGB", size, color).save(path)


def _write_split(path: Path, orient, size):
    """Structured (half black / half white) image — solid colors all
    phash-collapse to the same value, so a real threshold test needs DCT
    content. Vertical vs horizontal split = structurally far apart."""
    path.parent.mkdir(parents=True, exist_ok=True)
    w, h = size
    im = Image.new("L", size, 0)
    px = im.load()
    for y in range(h):
        for x in range(w):
            if (x / w if orient == "v" else y / h) >= 0.5:
                px[x, y] = 255
    im.convert("RGB").save(path)


def test_non_similar_imports_with_phash(importer, import_layout):
    import_root, _ = import_layout
    src = import_root / "a.png"
    _write(src, (12, 200, 60), (300, 300))
    r = importer.import_one(src)
    assert r.status == "imported"
    row = importer.session.get(ImageRecord, r.image_id)
    assert row.phash is not None and len(row.phash) == 16


def test_larger_existing_skips_new_phash_dup(importer, import_layout):
    import_root, _ = import_layout
    big = import_root / "big.png"
    _write(big, (40, 90, 160), (800, 800))
    r1 = importer.import_one(big)
    assert r1.status == "imported"
    small = import_root / "small.png"
    _write(small, (40, 90, 160), (200, 200))  # same look, smaller
    r2 = importer.import_one(small)
    assert r2.status == "skipped"
    assert r2.skip_reason == SkipReason.duplicate_phash
    assert r2.image_id == r1.image_id
    count = importer.session.execute(
        select(func.count()).select_from(ImageRecord)
    ).scalar_one()
    assert count == 1


def test_smaller_existing_is_superseded(importer, import_layout):
    import_root, _ = import_layout
    small = import_root / "small.png"
    _write(small, (200, 30, 30), (200, 200))
    r1 = importer.import_one(small)
    eid = r1.image_id
    old = importer.session.get(ImageRecord, eid)
    old_path = old.path
    tag = Tag(name="keepme", kind=TagKind.general)
    importer.session.add(tag)
    importer.session.flush()
    importer.session.execute(
        image_tag.insert().values(
            image_record_id=eid, tag_id=tag.id, source="manual"
        )
    )
    importer.session.add(
        ImagePrediction(
            image_record_id=eid, raw_name="x", category="general", score=0.9
        )
    )
    old.siglip_embedding = [0.0] * 1152
    old.integrity_status = "ok"
    importer.session.commit()

    big = import_root / "big.png"
    _write(big, (200, 30, 30), (900, 900))  # same look, larger
    r2 = importer.import_one(big)
    assert r2.status == "superseded"
    assert r2.image_id == eid

    importer.session.expire_all()
    row = importer.session.get(ImageRecord, eid)
    assert row.id == eid
    assert row.width == 900 and row.height == 900
    assert row.path != old_path
    assert row.phash is not None
    assert row.integrity_status == "unknown"
    # #768: re-import clears the normalized predictions too
    assert importer.session.execute(
        select(func.count()).select_from(ImagePrediction)
        .where(ImagePrediction.image_record_id == eid)
    ).scalar_one() == 0
    assert row.siglip_embedding is None
    linked = importer.session.execute(
        select(image_tag.c.tag_id).where(
            image_tag.c.image_record_id == eid
        )
    ).scalars().all()
    assert tag.id in linked
    assert not Path(old_path).exists()
    assert Path(row.path).exists()


def test_supersede_applies_new_file_sidecar(importer, import_layout):
    """Operator-flagged 2026-05-25: scanning the GS download dir should
    supersede smaller IR-migrated images AND wire up the GS sidecar's
    Post/Source/ImageProvenance. Previously _supersede swapped the file
    but ignored the sidecar entirely."""
    import json
    import_root, _ = import_layout

    # Stage 1: a small, sidecar-less image (the "IR migration" precondition).
    small = import_root / "ir-migration-folder" / "small.png"
    _write(small, (60, 130, 200), (200, 200))
    r1 = importer.import_one(small)
    assert r1.status == "imported"
    eid = r1.image_id

    # Stage 2: a larger version of the same image (same phash) WITH a
    # gallery-dl JSON sidecar adjacent. Live in a separate folder to
    # simulate the GS download dir.
    big = import_root / "Maewix" / "patreon" / "01_big.png"
    _write(big, (60, 130, 200), (900, 900))
    sidecar_path = big.with_suffix(big.suffix + ".json")
    sidecar_path.parent.mkdir(parents=True, exist_ok=True)
    sidecar_path.write_text(json.dumps({
        "category": "patreon",
        "id": 555,
        "url": "https://www.patreon.com/posts/555",
        "title": "Set 1",
        "content": "<p>The big version</p>",
        "page_count": 1,
        "published_at": "2025-08-01T00:00:00+00:00",
        "artist": "Maewix",
    }))

    r2 = importer.import_one(big)
    assert r2.status == "superseded"
    assert r2.image_id == eid

    importer.session.expire_all()
    # Row preserved, file replaced, sidecar metadata wired up.
    row = importer.session.get(ImageRecord, eid)
    assert row.width == 900 and row.height == 900

    post = importer.session.execute(
        select(Post).where(Post.external_post_id == "555")
    ).scalar_one()
    assert post.post_title == "Set 1"
    assert "big version" in (post.description or "")

    # Filesystem sidecars no longer create a synthetic Source (alembic 0030).
    # The Post sits null-source; the platform context is captured in the
    # sidecar JSON / raw_metadata, not in a phantom Source row.
    assert post.source_id is None
    assert post.artist_id is not None

    prov_count = importer.session.execute(
        select(func.count(ImageProvenance.id))
        .where(ImageProvenance.image_record_id == eid)
        .where(ImageProvenance.post_id == post.id)
    ).scalar_one()
    assert prov_count == 1


def test_threshold_controls_match(importer, import_layout):
    # Structurally distinct images (orthogonal splits) are far apart in
    # phash space, so a tight threshold keeps them independent rather than
    # collapsing the larger one into a supersede.
    import_root, _ = import_layout
    _set_threshold(importer, 0)
    a = import_root / "a.png"
    _write_split(a, "v", (200, 200))
    importer.import_one(a)
    b = import_root / "b.png"
    _write_split(b, "h", (900, 900))  # different structure, larger
    r = importer.import_one(b)
    assert r.status == "imported"  # threshold 0 + far → independent import


def test_import_task_maps_superseded_to_complete_and_requeues():
    from backend.app.services.importer import ImportResult
    from backend.app.tasks.import_file import _map_result_to_status

    assert _map_result_to_status(
        ImportResult(status="superseded", image_id=5)
    ) == ("complete", True)
    assert _map_result_to_status(
        ImportResult(status="imported", image_id=5)
    ) == ("complete", True)
    assert _map_result_to_status(
        ImportResult(status="skipped", skip_reason=SkipReason.duplicate_phash)
    ) == ("skipped", False)
    assert _map_result_to_status(
        ImportResult(status="failed", error="boom")
    ) == ("failed", False)
    # Refreshed (deep scan): complete + no ML/thumb re-derive (pixels
    # unchanged). Added 2026-05-25 alongside ImportBatch.refreshed
    # counter so deep scan reports "X refreshed" instead of "no work".
    assert _map_result_to_status(
        ImportResult(status="refreshed", image_id=5)
    ) == ("complete", False)