fix: variant artwork was dropped as a near-duplicate even at threshold 0 (4223)

The operator reported a 15-image variant pack landing as 3 records, then
reported variants STILL being dropped with phash_threshold at 0 — the floor
of the dial. No setting could have fixed it: at hash_size=8 a pHash is 64
bits of coarse light/dark layout, so two variants sharing a composition
produce the SAME bits. Distance 0 meant "identical hash", not "identical
image", and the dial was simultaneously too coarse to keep variants and too
tight to catch a re-encoded rescale.

The hash no longer decides a merge on its own. find_similar now runs three
gates, cheapest first: the threshold proposes candidates, aspect ratio
(ASPECT_TOL, matching the tier-1 video path) rejects crops and re-canvases,
and a pixel-level confirm on the two files accepts. Every gate fails closed —
unknown dimensions, an unreadable candidate, a hash of the wrong width all
mean "not a duplicate", because too strict keeps a redundant copy the
operator can see while too loose deletes artwork only a source re-walk
returns.

- utils/phash.py: HASH_SIZE 8 -> 16 (256-bit, what ImageRepo always used);
  aspect_matches, fingerprint/fingerprint_path/fingerprints_match (PIL-only,
  mean drift + changed-pixel fraction), find_similar gains `confirm`.
- importer: _pixel_confirmer supplies gate 3 on both dedup sites, lazily and
  cached, so a non-matching import costs no extra I/O.
- 0098: widens image_record.phash to 64 chars and NULLs every value — a
  stored 64-bit hash cannot be compared to a 256-bit one, and backfill_phash
  is NULL-only, keyset-paginated and now on the daily beat, so the library
  re-hashes itself. Dedup degrades to sha256 until it finishes.
- phash_threshold counts bits and the denominator went 64 -> 256, so the
  setting is reset to the new default of 24 (there is no honest carry-over)
  and the slider is rescaled to 0-64.
- gallery_service dup_threshold 8 -> 32: the same fraction of the hash, so
  the Explore rail keeps the variance the operator tuned in on 2026-07-01.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01LVjrnpQjRgHdvq95rASoiR

tests/test_api_settings.py

@@ -258,7 +258,7 @@ async def test_patch_rejects_non_object(client):
 async def test_import_settings_phash_threshold_default(client):
     resp = await client.get("/api/settings/import")
     assert resp.status_code == 200
-    assert (await resp.get_json())["phash_threshold"] == 10
+    assert (await resp.get_json())["phash_threshold"] == 24
 
 
 @pytest.mark.asyncio

tests/test_gallery_similar.py

@@ -146,12 +146,17 @@ async def test_similar_collapses_near_duplicate_phashes(db):
     dupes = []
     for n in range(2, 7):                       # 5 near-identical reposts
         r = await _img(db, n, _vec(1, 0.01 * n))
-        r.phash = "ffffffffffffffff"            # identical perceptual hash
+        r.phash = "f" * 64                      # identical perceptual hash
         dupes.append(r)
+    # 64 hex chars = the 256-bit hash utils.phash emits (#4223, migration
+    # 0098). The widths must match the real ones: _diversify_similar's
+    # dup_threshold moved 8 -> 32 with the hash, so 64-bit fixtures would
+    # now read as near-duplicates of each other and collapse the very rows
+    # this asserts come through.
     distinct_a = await _img(db, 7, _vec(1, 1))
-    distinct_a.phash = "0000000000000000"
+    distinct_a.phash = "0" * 64                 # 256 bits away from the dupes
     distinct_b = await _img(db, 8, _vec(0, 1))
-    distinct_b.phash = "0f0f0f0f0f0f0f0f"
+    distinct_b.phash = "0f" * 32                # 128 bits away
     await db.flush()
 
     res = await GalleryService(db).similar(src.id, limit=10)

tests/test_phash_dedup.py

@@ -81,7 +81,7 @@ def test_non_similar_imports_with_phash(importer, import_layout):
     r = importer.import_one(src)
     assert r.status == "imported"
     row = importer.session.get(ImageRecord, r.image_id)
-    assert row.phash is not None and len(row.phash) == 16
+    assert row.phash is not None and len(row.phash) == 64  # 256-bit
 
 
 def test_larger_existing_skips_new_phash_dup(importer, import_layout):
@@ -223,6 +223,74 @@ def test_threshold_controls_match(importer, import_layout):
     assert r.status == "imported"  # threshold 0 + far → independent import
 
 
+# --- The three gates (#4223) ------------------------------------------------
+#
+# Each of these opens the hash gate all the way (threshold 256 = every
+# candidate passes) so the test is about the gate named in its title, and not
+# about whether two fixtures happen to hash apart. That is the regression
+# being guarded: the operator ran the dial down to 0 and STILL lost variants,
+# because the hash was never the thing that could tell them apart.
+
+
+def _wide_open(importer):
+    _set_threshold(importer, 256)
+
+
+def test_variant_survives_a_wide_open_threshold(importer, import_layout):
+    """Same aspect, same size, different picture — only the pixel confirm can
+    save it, and it must."""
+    import_root, _ = import_layout
+    a = import_root / "v.png"
+    _write_split(a, "v", (400, 400))
+    _wide_open(importer)
+    assert importer.import_one(a).status == "imported"
+
+    b = import_root / "h.png"
+    _write_split(b, "h", (400, 400))
+    assert importer.import_one(b).status == "imported"
+    assert importer.session.execute(
+        select(func.count()).select_from(ImageRecord)
+    ).scalar_one() == 2
+
+
+def test_rescale_still_supersedes_at_a_wide_open_threshold(importer, import_layout):
+    """The other half of the deal: the merge the operator DOES want still
+    happens, and keeps the higher resolution."""
+    import_root, _ = import_layout
+    small = import_root / "small.png"
+    _write_split(small, "v", (200, 200))
+    _wide_open(importer)
+    r1 = importer.import_one(small)
+    assert r1.status == "imported"
+
+    big = import_root / "big.png"
+    _write_split(big, "v", (900, 900))
+    r2 = importer.import_one(big)
+    assert r2.status == "superseded"
+    assert r2.image_id == r1.image_id
+
+    importer.session.expire_all()
+    row = importer.session.get(ImageRecord, r1.image_id)
+    assert row.width == 900 and row.height == 900
+
+
+def test_different_aspect_is_never_a_duplicate(importer, import_layout):
+    """Solid colours are pixel-identical once fingerprinted, so the aspect
+    gate is the only thing standing between a crop and a supersede."""
+    import_root, _ = import_layout
+    square = import_root / "square.png"
+    _write(square, (90, 40, 180), (400, 400))
+    _wide_open(importer)
+    assert importer.import_one(square).status == "imported"
+
+    wide = import_root / "wide.png"
+    _write(wide, (90, 40, 180), (800, 400))
+    assert importer.import_one(wide).status == "imported"
+    assert importer.session.execute(
+        select(func.count()).select_from(ImageRecord)
+    ).scalar_one() == 2
+
+
 def test_import_task_maps_superseded_to_complete_and_requeues():
     from backend.app.services.importer import ImportResult
     from backend.app.tasks.import_file import _map_result_to_status

tests/test_phash_util.py

@@ -3,7 +3,13 @@ import io
 import imagehash
 from PIL import Image
 
-from backend.app.utils.phash import compute_phash, find_similar
+from backend.app.utils.phash import (
+    aspect_matches,
+    compute_phash,
+    find_similar,
+    fingerprint,
+    fingerprints_match,
+)
 
 
 def _img(color, size=(64, 64)):
@@ -25,7 +31,7 @@ def _split(orient, size=64):
 def test_compute_phash_stable_and_hex():
     h1 = compute_phash(_img((10, 120, 200)))
     h2 = compute_phash(_img((10, 120, 200)))
-    assert isinstance(h1, str) and len(h1) == 16  # hash_size=8 -> 64-bit -> 16 hex
+    assert isinstance(h1, str) and len(h1) == 64  # hash_size=16 -> 256-bit -> 64 hex
     assert h1 == h2
 
 
@@ -62,3 +68,73 @@ def test_find_similar_threshold_boundary_inclusive_and_first_match():
     assert rel == "larger_exists"
     rel2, _ = find_similar(h, 10, 10, [(far, 999, 999, 1)], threshold=d - 1)
     assert rel2 == "none"
+
+
+# --- Gate 2: aspect ratio (#4223) -------------------------------------------
+
+def test_aspect_matches_tolerates_rounding_but_not_a_crop():
+    assert aspect_matches(1000, 1000, 250, 250)
+    assert aspect_matches(1999, 1000, 1000, 500)     # off-by-one on a rescale
+    assert not aspect_matches(1000, 1000, 1000, 500)  # 1:1 vs 2:1
+    # Unmeasurable is not a proven duplicate — the gate fails closed.
+    assert not aspect_matches(1000, 1000, None, None)
+    assert not aspect_matches(0, 0, 100, 100)
+
+
+def test_find_similar_skips_a_hash_twin_with_a_different_aspect():
+    """A crop or re-canvas keeps the composition, so it can land inside the
+    threshold. Dimensions are what say it is not a rescale."""
+    h = compute_phash(_img((123, 50, 7)))
+    rel, mid = find_similar(h, 100, 100, [(h, 400, 200, 9)], threshold=64)
+    assert rel == "none" and mid is None
+
+
+# --- Gate 3: the pixel confirm (#4223) --------------------------------------
+
+def test_fingerprints_match_across_a_rescale():
+    assert fingerprints_match(
+        fingerprint(_split("v", 64)), fingerprint(_split("v", 512))
+    )
+
+
+def test_fingerprints_reject_a_local_change():
+    """The case the hash cannot see: same composition, one region redrawn."""
+    base = _split("v", 128)
+    variant = base.copy()
+    for y in range(20, 60):
+        for x in range(80, 120):        # a patch on the white half
+            variant.putpixel((x, y), (0, 0, 0))
+    assert not fingerprints_match(fingerprint(base), fingerprint(variant))
+
+
+def test_fingerprints_match_is_false_when_either_side_is_missing():
+    assert not fingerprints_match(fingerprint(_split("v")), None)
+    assert not fingerprints_match(None, None)
+
+
+def test_find_similar_confirm_can_veto_a_hash_and_aspect_match():
+    h = compute_phash(_img((123, 50, 7)))
+    cand = [(h, 400, 400, 9)]
+    rel, mid = find_similar(h, 100, 100, cand, threshold=64, confirm=lambda _: True)
+    assert rel == "larger_exists" and mid == 9
+    rel2, mid2 = find_similar(h, 100, 100, cand, threshold=64, confirm=lambda _: False)
+    assert rel2 == "none" and mid2 is None
+
+
+def test_find_similar_keeps_looking_after_a_veto():
+    """A vetoed candidate must not end the search, or one false pre-filter hit
+    would hide the real duplicate sitting behind it."""
+    h = compute_phash(_img((123, 50, 7)))
+    candidates = [(h, 400, 400, 1), (h, 400, 400, 2)]
+    rel, mid = find_similar(
+        h, 100, 100, candidates, threshold=64, confirm=lambda cid: cid == 2
+    )
+    assert rel == "larger_exists" and mid == 2
+
+
+def test_find_similar_skips_a_hash_of_the_wrong_width():
+    """Mid-re-hash (migration 0098): a leftover 64-bit hash cannot be compared
+    to a 256-bit one. Skipping it degrades to no dedup, never to a merge."""
+    h = compute_phash(_img((123, 50, 7)))
+    rel, mid = find_similar(h, 100, 100, [("ffffffffffffffff", 400, 400, 3)], threshold=64)
+    assert rel == "none" and mid is None