"""What the near-duplicate gates would decide about a folder of real artwork. Read-only. Opens image files, touches no database, imports nothing from the app but `utils/phash.py` itself — so what it reports is what the importer would actually do, not a re-implementation that could drift from it. ## Why this exists Issue #4223 replaced a single 64-bit pHash comparison with three gates (threshold -> aspect ratio -> pixel confirm). The threshold is the operator's dial and the aspect tolerance mirrors the video path, but the three pixel constants — FINGERPRINT_MAX_MEAN_DIFF, FINGERPRINT_CHANGED_LEVEL, FINGERPRINT_MAX_CHANGED_FRACTION — were chosen without ever measuring real artwork, because FC verifies in CI and CI has only synthetic fixtures. This prints the measurements those constants should have been chosen from. Point it at a folder whose right answer you already know — a variant set that should stay whole, or an image you have at two resolutions that should collapse — and read the MARGIN column. A pair that lands just inside a limit is the one that will flip on the next slightly-different file. ## Usage python scripts/phash_gate_report.py [--threshold N] [--recursive] python scripts/phash_gate_report.py [...] No local Python environment needed — run it inside the published image, which already has PIL and imagehash, with the folder mounted read-only: docker run --rm --entrypoint python -e PYTHONPATH=/app \ -v /path/to/art:/data:ro -v "$PWD/scripts":/scripts:ro \ git.fabledsword.com/bvandeusen/fabledcurator:dev \ /scripts/phash_gate_report.py /data (The image ships `backend/` at /app but not `scripts/`, hence the second mount and PYTHONPATH. The `:ro` on the art folder is the point — this reads.) `:dev` is the rolling channel this branch publishes to (rule 147) — the same bytes that would run in the app, without merging anything to main. """ import argparse import itertools import sys from pathlib import Path import imagehash from PIL import Image sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) from backend.app.utils.phash import ( # noqa: E402 ASPECT_TOL, FINGERPRINT_MAX_CHANGED_FRACTION, FINGERPRINT_MAX_MEAN_DIFF, HASH_SIZE, aspect_matches, compute_phash, find_similar, fingerprint, fingerprint_diff, fingerprints_match, ) IMAGE_EXTS = {".png", ".jpg", ".jpeg", ".gif", ".webp", ".bmp", ".tif", ".tiff"} DEFAULT_THRESHOLD = 24 class Shot: """One image, measured once.""" def __init__(self, path: Path): self.path = path self.phash = None self.width = 0 self.height = 0 self.fingerprint = None self.error = None try: with Image.open(path) as im: self.width, self.height = im.size self.phash = compute_phash(im) self.fingerprint = fingerprint(im) except Exception as exc: self.error = str(exc) @property def ok(self) -> bool: return self.phash is not None and self.fingerprint is not None @property def label(self) -> str: return f"{self.path.name} ({self.width}x{self.height})" def collect(paths, recursive: bool) -> list[Path]: found: list[Path] = [] for p in paths: p = Path(p) if p.is_dir(): walk = p.rglob("*") if recursive else p.glob("*") found += sorted(f for f in walk if f.suffix.lower() in IMAGE_EXTS) elif p.is_file(): found.append(p) return found def verdict(a: Shot, b: Shot, threshold: int) -> tuple[str, str]: """Run the REAL find_similar for "a is being imported, b is in the library". Returns (verdict, the gate that decided it).""" rel, _ = find_similar( a.phash, a.width, a.height, [(b.phash, b.width, b.height, 1)], threshold, confirm=lambda _: fingerprints_match(a.fingerprint, b.fingerprint), ) if rel == "larger_exists": return ("DROP", f"{a.path.name} dropped; {b.path.name} kept (>= in both)") if rel == "smaller_exists": return ("SUPERSEDE", f"{a.path.name} replaces {b.path.name}'s file") # Not a match — say which gate refused, cheapest first, since that is the # constant to move if the answer is wrong. dist = imagehash.hex_to_hash(a.phash) - imagehash.hex_to_hash(b.phash) if dist > threshold: return ("KEEP BOTH", f"hash distance {dist} > threshold {threshold}") if not aspect_matches(a.width, a.height, b.width, b.height): return ("KEEP BOTH", "aspect ratios differ") return ("KEEP BOTH", "pixels differ") def main() -> int: ap = argparse.ArgumentParser(description=__doc__.split("\n")[0]) ap.add_argument("paths", nargs="+", help="a directory, or two or more files") ap.add_argument("--threshold", type=int, default=DEFAULT_THRESHOLD, help=f"phash_threshold to simulate (default {DEFAULT_THRESHOLD})") ap.add_argument("--recursive", action="store_true", help="walk subdirectories") ap.add_argument("--limit", type=int, default=60, help="refuse more than this many images (pairs grow as N^2)") args = ap.parse_args() files = collect(args.paths, args.recursive) if len(files) < 2: print(f"Need at least 2 images; found {len(files)}.", file=sys.stderr) return 2 if len(files) > args.limit: print( f"{len(files)} images would be {len(files) * (len(files) - 1) // 2} " f"pairs. Narrow the folder or raise --limit.", file=sys.stderr ) return 2 print(f"hash_size={HASH_SIZE} ({HASH_SIZE * HASH_SIZE} bits) " f"threshold={args.threshold} aspect_tol={ASPECT_TOL}") print(f"pixel limits: mean <= {FINGERPRINT_MAX_MEAN_DIFF}, " f"changed <= {FINGERPRINT_MAX_CHANGED_FRACTION:.1%}\n") shots = [Shot(f) for f in files] for s in shots: if not s.ok: print(f" ! unreadable, excluded: {s.path.name} — {s.error}") shots = [s for s in shots if s.ok] if len(shots) < 2: print("Not enough readable images.", file=sys.stderr) return 2 print(f"{'verdict':<10} {'dist':>5} {'mean':>7} {'changed':>8} pair") print("-" * 78) counts: dict[str, int] = {} rows = [] for a, b in itertools.combinations(shots, 2): v, why = verdict(a, b, args.threshold) counts[v] = counts.get(v, 0) + 1 dist = imagehash.hex_to_hash(a.phash) - imagehash.hex_to_hash(b.phash) measured = fingerprint_diff(a.fingerprint, b.fingerprint) mean, changed = measured if measured else (float("nan"), float("nan")) rows.append((v, dist, mean, changed, a, b, why)) # Closest pairs first: the interesting decisions are the near-misses at # both limits, not the obvious strangers at the bottom of the list. for v, dist, mean, changed, a, b, why in sorted(rows, key=lambda r: r[1]): print(f"{v:<10} {dist:>5} {mean:>7.2f} {changed:>7.2%} " f"{a.label} vs {b.label}") print(f"{'':<10} {'':>5} {'':>7} {'':>8} -> {why}") print("\n" + " ".join(f"{k}: {n}" for k, n in sorted(counts.items()))) print( "\nRead the margin, not just the verdict. A pair you consider the SAME " f"image should sit far under mean {FINGERPRINT_MAX_MEAN_DIFF} / changed " f"{FINGERPRINT_MAX_CHANGED_FRACTION:.0%}; a pair you consider DIFFERENT " "artwork\nshould sit far over. Anything that only just cleared a limit " "is what will flip on the next file, and is the reason to move a constant." ) return 0 if __name__ == "__main__": raise SystemExit(main())