From 3313c3b10a1360596e4bd6038a82cf62166e32c8 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Mon, 21 Sep 2026 06:30:54 -0400 Subject: [PATCH] feat: a report that shows what the near-dup gates decide about real artwork (4223) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The three pixel constants added with the #4223 fix were chosen without ever measuring real files — CI only has synthetic split/solid fixtures, and FC verifies nowhere else. This prints the measurements they should have been chosen from: per pair, the hash distance, the mean drift, the changed-pixel fraction, the verdict, and which gate produced it. It drives the real find_similar with the real confirm rather than restating the decision, so it cannot drift from what the importer does. Read-only: opens files, touches no database. Also splits fingerprint_diff out of fingerprints_match — same computation, now returning the numbers instead of only the boolean, so the report can show how far a pair sat from a limit rather than which side of it it fell on. Runs inside the published :dev image (PIL + imagehash already there, no local env needed) with the art folder mounted read-only — rule 147's channel, so nothing has to reach main to be tried. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01LVjrnpQjRgHdvq95rASoiR --- backend/app/utils/phash.py | 42 +++++--- scripts/phash_gate_report.py | 197 +++++++++++++++++++++++++++++++++++ 2 files changed, 226 insertions(+), 13 deletions(-) create mode 100644 scripts/phash_gate_report.py diff --git a/backend/app/utils/phash.py b/backend/app/utils/phash.py index b92f991..4592b3a 100644 --- a/backend/app/utils/phash.py +++ b/backend/app/utils/phash.py @@ -122,6 +122,31 @@ def fingerprint_path(path) -> Image.Image | None: return None +def fingerprint_diff( + a, b, *, changed_level: int = FINGERPRINT_CHANGED_LEVEL, +) -> tuple[float, float] | None: + """(mean absolute difference, fraction of pixels past `changed_level`) for + two fingerprints. None if either is missing or the comparison fails. + + This is the MEASUREMENT behind `fingerprints_match`, split out so the + calibration report (scripts/phash_gate_report.py) can show how far a pair + sat from the limits instead of only which side of them it fell on. The + constants were chosen without a real-library sample; the numbers this + returns are what moves them. + """ + if a is None or b is None: + return None + try: + diff = ImageChops.difference(a, b) + hist = diff.histogram() + total = sum(hist) + if not total: + return None + return (ImageStat.Stat(diff).mean[0], sum(hist[changed_level:]) / total) + except Exception: + return None + + def fingerprints_match( a, b, *, @@ -132,20 +157,11 @@ def fingerprints_match( """True when two fingerprints are the same picture: no large global drift AND no meaningful local region that differs. False on any failure.""" - if a is None or b is None: - return False - try: - diff = ImageChops.difference(a, b) - if ImageStat.Stat(diff).mean[0] > max_mean_diff: - return False - hist = diff.histogram() - total = sum(hist) - if not total: - return False - changed = sum(hist[changed_level:]) - return (changed / total) <= max_changed_fraction - except Exception: + measured = fingerprint_diff(a, b, changed_level=changed_level) + if measured is None: return False + mean, changed_fraction = measured + return mean <= max_mean_diff and changed_fraction <= max_changed_fraction def aspect_matches( diff --git a/scripts/phash_gate_report.py b/scripts/phash_gate_report.py new file mode 100644 index 0000000..c193986 --- /dev/null +++ b/scripts/phash_gate_report.py @@ -0,0 +1,197 @@ +"""What the near-duplicate gates would decide about a folder of real artwork. + +Read-only. Opens image files, touches no database, imports nothing from the +app but `utils/phash.py` itself — so what it reports is what the importer +would actually do, not a re-implementation that could drift from it. + +## Why this exists + +Issue #4223 replaced a single 64-bit pHash comparison with three gates +(threshold -> aspect ratio -> pixel confirm). The threshold is the operator's +dial and the aspect tolerance mirrors the video path, but the three pixel +constants — FINGERPRINT_MAX_MEAN_DIFF, FINGERPRINT_CHANGED_LEVEL, +FINGERPRINT_MAX_CHANGED_FRACTION — were chosen without ever measuring real +artwork, because FC verifies in CI and CI has only synthetic fixtures. This +prints the measurements those constants should have been chosen from. + +Point it at a folder whose right answer you already know — a variant set that +should stay whole, or an image you have at two resolutions that should +collapse — and read the MARGIN column. A pair that lands just inside a limit +is the one that will flip on the next slightly-different file. + +## Usage + + python scripts/phash_gate_report.py [--threshold N] [--recursive] + python scripts/phash_gate_report.py [...] + +No local Python environment needed — run it inside the published image, which +already has PIL and imagehash, with the folder mounted read-only: + + docker run --rm --entrypoint python -e PYTHONPATH=/app \ + -v /path/to/art:/data:ro -v "$PWD/scripts":/scripts:ro \ + git.fabledsword.com/bvandeusen/fabledcurator:dev \ + /scripts/phash_gate_report.py /data + +(The image ships `backend/` at /app but not `scripts/`, hence the second +mount and PYTHONPATH. The `:ro` on the art folder is the point — this reads.) + +`:dev` is the rolling channel this branch publishes to (rule 147) — the same +bytes that would run in the app, without merging anything to main. +""" + +import argparse +import itertools +import sys +from pathlib import Path + +import imagehash +from PIL import Image + +sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) + +from backend.app.utils.phash import ( # noqa: E402 + ASPECT_TOL, + FINGERPRINT_MAX_CHANGED_FRACTION, + FINGERPRINT_MAX_MEAN_DIFF, + HASH_SIZE, + aspect_matches, + compute_phash, + find_similar, + fingerprint, + fingerprint_diff, + fingerprints_match, +) + +IMAGE_EXTS = {".png", ".jpg", ".jpeg", ".gif", ".webp", ".bmp", ".tif", ".tiff"} +DEFAULT_THRESHOLD = 24 + + +class Shot: + """One image, measured once.""" + + def __init__(self, path: Path): + self.path = path + self.phash = None + self.width = 0 + self.height = 0 + self.fingerprint = None + self.error = None + try: + with Image.open(path) as im: + self.width, self.height = im.size + self.phash = compute_phash(im) + self.fingerprint = fingerprint(im) + except Exception as exc: + self.error = str(exc) + + @property + def ok(self) -> bool: + return self.phash is not None and self.fingerprint is not None + + @property + def label(self) -> str: + return f"{self.path.name} ({self.width}x{self.height})" + + +def collect(paths, recursive: bool) -> list[Path]: + found: list[Path] = [] + for p in paths: + p = Path(p) + if p.is_dir(): + walk = p.rglob("*") if recursive else p.glob("*") + found += sorted(f for f in walk if f.suffix.lower() in IMAGE_EXTS) + elif p.is_file(): + found.append(p) + return found + + +def verdict(a: Shot, b: Shot, threshold: int) -> tuple[str, str]: + """Run the REAL find_similar for "a is being imported, b is in the + library". Returns (verdict, the gate that decided it).""" + rel, _ = find_similar( + a.phash, a.width, a.height, + [(b.phash, b.width, b.height, 1)], + threshold, + confirm=lambda _: fingerprints_match(a.fingerprint, b.fingerprint), + ) + if rel == "larger_exists": + return ("DROP", f"{a.path.name} dropped; {b.path.name} kept (>= in both)") + if rel == "smaller_exists": + return ("SUPERSEDE", f"{a.path.name} replaces {b.path.name}'s file") + # Not a match — say which gate refused, cheapest first, since that is the + # constant to move if the answer is wrong. + dist = imagehash.hex_to_hash(a.phash) - imagehash.hex_to_hash(b.phash) + if dist > threshold: + return ("KEEP BOTH", f"hash distance {dist} > threshold {threshold}") + if not aspect_matches(a.width, a.height, b.width, b.height): + return ("KEEP BOTH", "aspect ratios differ") + return ("KEEP BOTH", "pixels differ") + + +def main() -> int: + ap = argparse.ArgumentParser(description=__doc__.split("\n")[0]) + ap.add_argument("paths", nargs="+", help="a directory, or two or more files") + ap.add_argument("--threshold", type=int, default=DEFAULT_THRESHOLD, + help=f"phash_threshold to simulate (default {DEFAULT_THRESHOLD})") + ap.add_argument("--recursive", action="store_true", help="walk subdirectories") + ap.add_argument("--limit", type=int, default=60, + help="refuse more than this many images (pairs grow as N^2)") + args = ap.parse_args() + + files = collect(args.paths, args.recursive) + if len(files) < 2: + print(f"Need at least 2 images; found {len(files)}.", file=sys.stderr) + return 2 + if len(files) > args.limit: + print( + f"{len(files)} images would be {len(files) * (len(files) - 1) // 2} " + f"pairs. Narrow the folder or raise --limit.", file=sys.stderr + ) + return 2 + + print(f"hash_size={HASH_SIZE} ({HASH_SIZE * HASH_SIZE} bits) " + f"threshold={args.threshold} aspect_tol={ASPECT_TOL}") + print(f"pixel limits: mean <= {FINGERPRINT_MAX_MEAN_DIFF}, " + f"changed <= {FINGERPRINT_MAX_CHANGED_FRACTION:.1%}\n") + + shots = [Shot(f) for f in files] + for s in shots: + if not s.ok: + print(f" ! unreadable, excluded: {s.path.name} — {s.error}") + shots = [s for s in shots if s.ok] + if len(shots) < 2: + print("Not enough readable images.", file=sys.stderr) + return 2 + + print(f"{'verdict':<10} {'dist':>5} {'mean':>7} {'changed':>8} pair") + print("-" * 78) + counts: dict[str, int] = {} + rows = [] + for a, b in itertools.combinations(shots, 2): + v, why = verdict(a, b, args.threshold) + counts[v] = counts.get(v, 0) + 1 + dist = imagehash.hex_to_hash(a.phash) - imagehash.hex_to_hash(b.phash) + measured = fingerprint_diff(a.fingerprint, b.fingerprint) + mean, changed = measured if measured else (float("nan"), float("nan")) + rows.append((v, dist, mean, changed, a, b, why)) + + # Closest pairs first: the interesting decisions are the near-misses at + # both limits, not the obvious strangers at the bottom of the list. + for v, dist, mean, changed, a, b, why in sorted(rows, key=lambda r: r[1]): + print(f"{v:<10} {dist:>5} {mean:>7.2f} {changed:>7.2%} " + f"{a.label} vs {b.label}") + print(f"{'':<10} {'':>5} {'':>7} {'':>8} -> {why}") + + print("\n" + " ".join(f"{k}: {n}" for k, n in sorted(counts.items()))) + print( + "\nRead the margin, not just the verdict. A pair you consider the SAME " + f"image should sit far under mean {FINGERPRINT_MAX_MEAN_DIFF} / changed " + f"{FINGERPRINT_MAX_CHANGED_FRACTION:.0%}; a pair you consider DIFFERENT " + "artwork\nshould sit far over. Anything that only just cleared a limit " + "is what will flip on the next file, and is the reason to move a constant." + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main())