feat(library): the duplicate matcher — a pure comparison over fingerprints (M400 #3909)

Decides whether tracks are proposed as one recording. No database, no
files, so every rule is falsifiable in a unit test.

Two tiers:
- exact: equal audio_stream_sha256 (identical encoded audio bytes). No
  threshold and no false positives.
- acoustic: chromaprint fingerprints that agree once aligned. Two
  fingerprints can start at slightly different points in the audio
  (padding trimmed differently), so offsets within ±120 items (~15s)
  are voted on using items that share their high 14 bits. Bit-error
  rate is then measured over the overlap at the winning offset. The
  approach and both constants follow AcoustID's pg_acoustid; it was
  reimplemented from that description and no code was copied.

No verdict below ~10s of overlap, or for low-information fingerprints
(silence, a sustained tone). Two such tracks agree without being one
recording.

Grouping uses complete linkage: a track joins a group only if it
matches every member. Otherwise A close to B and B close to C would
merge A and C, which are not close, and it means any member can be the
survivor. Other rules:
- durations must be within 3s
- acoustic groups are capped at 8, and larger clusters are reported
  and discarded as a likely shared jingle
- an exact group absorbed into an acoustic one takes the acoustic tier
- output does not depend on input order

The acoustic threshold is 0.15 bit-error rate: deliberately
conservative, since the operator's concern is different recordings of
one song being merged, and an instrumental shares its vocal's harmony.
It is unmeasured, and needs calibrating against real pairs once the
backfill has populated fingerprints (#3913 exposes it).

Nothing calls this yet; the sweep (#3910) does.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01SQ31KQpYbStyK5y58UmPLH

internal/library/duplicate_match_test.go

@@ -0,0 +1,241 @@
+package library
+
+import (
+	"math"
+	"math/rand/v2"
+	"reflect"
+	"testing"
+)
+
+// printLen is a realistic fingerprint length: fpcalc's 120s at ~8 items/second.
+const printLen = 960
+
+// randomPrint is a deterministic stand-in for one recording's fingerprint.
+func randomPrint(seed uint64, n int) []int32 {
+	r := rand.New(rand.NewPCG(seed, seed^0x9e3779b97f4a7c15))
+	fp := make([]int32, n)
+	for i := range fp {
+		fp[i] = int32(r.Uint32())
+	}
+	return fp
+}
+
+// withBitNoise flips exactly round(fraction × all bits) distinct bits — the
+// same recording through a different encoder, at a known bit-error rate.
+func withBitNoise(fp []int32, fraction float64, seed uint64) []int32 {
+	out := append([]int32(nil), fp...)
+	r := rand.New(rand.NewPCG(seed, seed^0x243f6a8885a308d3))
+	total := 32 * len(fp)
+	for _, pos := range r.Perm(total)[:int(math.Round(fraction*float64(total)))] {
+		out[pos/32] ^= int32(uint32(1) << (pos % 32))
+	}
+	return out
+}
+
+func constantPrint(v int32, n int) []int32 {
+	fp := make([]int32, n)
+	for i := range fp {
+		fp[i] = v
+	}
+	return fp
+}
+
+func TestCompareChromaprint(t *testing.T) {
+	base := randomPrint(1, printLen)
+
+	t.Run("identical", func(t *testing.T) {
+		got, ok := compareChromaprint(base, base)
+		if !ok || got.BitErrorRate != 0 || got.Offset != 0 || got.Overlap != printLen {
+			t.Fatalf("got %+v ok=%v, want an exact alignment", got, ok)
+		}
+	})
+
+	t.Run("re-encoded: known bit noise is measured exactly", func(t *testing.T) {
+		got, ok := compareChromaprint(base, withBitNoise(base, 0.03, 2))
+		if !ok {
+			t.Fatal("a re-encode was not comparable")
+		}
+		if want := math.Round(0.03*32*printLen) / (32 * printLen); got.BitErrorRate != want {
+			t.Fatalf("BitErrorRate = %v, want %v", got.BitErrorRate, want)
+		}
+	})
+
+	// b starts 40 items later in the same audio: b[j] = a[j+40], so a[i] aligns
+	// with b[i-40].
+	t.Run("offset inside the window is recovered", func(t *testing.T) {
+		got, ok := compareChromaprint(base, base[40:])
+		if !ok || got.Offset != -40 || got.BitErrorRate != 0 || got.Overlap != printLen-40 {
+			t.Fatalf("got %+v ok=%v, want offset -40 with no error", got, ok)
+		}
+	})
+
+	t.Run("offset beyond the window never matches", func(t *testing.T) {
+		got, ok := compareChromaprint(base, base[200:])
+		if ok && got.BitErrorRate <= defaultAcousticMaxBitErrorRate {
+			t.Fatalf("a 200-item shift matched: %+v", got)
+		}
+	})
+
+	t.Run("unrelated recordings sit near 0.5", func(t *testing.T) {
+		got, ok := compareChromaprint(base, randomPrint(99, printLen))
+		if ok && got.BitErrorRate < 0.4 {
+			t.Fatalf("unrelated fingerprints scored %v", got.BitErrorRate)
+		}
+	})
+
+	t.Run("too short an overlap gives no verdict", func(t *testing.T) {
+		if got, ok := compareChromaprint(base, base[:minOverlapItems-1]); ok {
+			t.Fatalf("a %d-item fingerprint was compared: %+v", minOverlapItems-1, got)
+		}
+	})
+
+	// Two near-silent tracks agree perfectly without being one recording. The
+	// information floor is the only thing standing between them and a merge.
+	t.Run("low-information fingerprints give no verdict", func(t *testing.T) {
+		silence := constantPrint(0x1234, printLen)
+		if got, ok := compareChromaprint(silence, silence); ok {
+			t.Fatalf("silence compared as a match: %+v", got)
+		}
+	})
+
+	t.Run("the threshold separates close from not close", func(t *testing.T) {
+		near, _ := compareChromaprint(base, withBitNoise(base, 0.10, 3))
+		far, _ := compareChromaprint(base, withBitNoise(base, 0.20, 4))
+		if near.BitErrorRate > defaultAcousticMaxBitErrorRate {
+			t.Errorf("10%% noise (%v) is over the threshold", near.BitErrorRate)
+		}
+		if far.BitErrorRate <= defaultAcousticMaxBitErrorRate {
+			t.Errorf("20%% noise (%v) is under the threshold", far.BitErrorRate)
+		}
+	})
+}
+
+func TestGroupDuplicates_ExactTier(t *testing.T) {
+	hash := []byte("sha256-of-www-instrumental-bytes")
+	res := groupDuplicates([]fingerprintCandidate{
+		{ID: "www-01", DurationMs: 215000, StreamSHA256: hash},
+		{ID: "www-02", DurationMs: 215000, StreamSHA256: hash},
+		{ID: "lovesick", DurationMs: 198000, StreamSHA256: []byte("another")},
+	}, defaultAcousticMaxBitErrorRate)
+	want := []duplicateGroup{{Tier: tierExact, Members: []string{"www-01", "www-02"}}}
+	if !reflect.DeepEqual(res.Groups, want) {
+		t.Fatalf("groups = %+v, want %+v", res.Groups, want)
+	}
+}
+
+func TestGroupDuplicates_AcousticPair(t *testing.T) {
+	p := randomPrint(10, printLen)
+	res := groupDuplicates([]fingerprintCandidate{
+		{ID: "album", DurationMs: 240000, Chromaprint: p},
+		{ID: "compilation", DurationMs: 241000, Chromaprint: withBitNoise(p, 0.05, 11)},
+	}, defaultAcousticMaxBitErrorRate)
+	if len(res.Groups) != 1 || res.Groups[0].Tier != tierAcoustic ||
+		!reflect.DeepEqual(res.Groups[0].Members, []string{"album", "compilation"}) {
+		t.Fatalf("groups = %+v, want one acoustic pair", res.Groups)
+	}
+	if got := res.Groups[0].WorstBitErrorRate; math.Abs(got-0.05) > 0.001 {
+		t.Fatalf("WorstBitErrorRate = %v, want about 0.05", got)
+	}
+}
+
+// A is close to B and B is close to C, but A and C are not close. Under
+// single linkage all three would be proposed as one recording; complete linkage
+// must keep C out.
+func TestGroupDuplicates_NoChaining(t *testing.T) {
+	a := randomPrint(20, printLen)
+	b := withBitNoise(a, 0.10, 21)
+	c := withBitNoise(b, 0.10, 22)
+	if s, _ := compareChromaprint(a, c); s.BitErrorRate <= defaultAcousticMaxBitErrorRate {
+		t.Fatalf("fixture broken: A and C are close (%v), so this cannot test chaining", s.BitErrorRate)
+	}
+	res := groupDuplicates([]fingerprintCandidate{
+		{ID: "a", DurationMs: 200000, Chromaprint: a},
+		{ID: "b", DurationMs: 200000, Chromaprint: b},
+		{ID: "c", DurationMs: 200000, Chromaprint: c},
+	}, defaultAcousticMaxBitErrorRate)
+	if len(res.Groups) != 1 || !reflect.DeepEqual(res.Groups[0].Members, []string{"a", "b"}) {
+		t.Fatalf("groups = %+v, want only {a, b}", res.Groups)
+	}
+}
+
+func TestGroupDuplicates_DurationTolerance(t *testing.T) {
+	p := randomPrint(30, printLen)
+	res := groupDuplicates([]fingerprintCandidate{
+		{ID: "edit", DurationMs: 200000, Chromaprint: p},
+		{ID: "extended", DurationMs: 200000 + durationToleranceMs + 1, Chromaprint: p},
+	}, defaultAcousticMaxBitErrorRate)
+	if len(res.Groups) != 0 {
+		t.Fatalf("tracks %dms apart were grouped: %+v", durationToleranceMs+1, res.Groups)
+	}
+}
+
+// Nine tracks that all match are far likelier a shared jingle than nine copies
+// of one recording. The cluster must be reported, not proposed.
+func TestGroupDuplicates_OversizeClusterIsDiscarded(t *testing.T) {
+	p := randomPrint(40, printLen)
+	var cands []fingerprintCandidate
+	for i := range maxAcousticGroupSize + 1 {
+		cands = append(cands, fingerprintCandidate{
+			ID: string(rune('a' + i)), DurationMs: 30000, Chromaprint: withBitNoise(p, 0.01, uint64(100+i)),
+		})
+	}
+	res := groupDuplicates(cands, defaultAcousticMaxBitErrorRate)
+	if len(res.Groups) != 0 || res.OversizeClusters != 1 {
+		t.Fatalf("groups = %+v, oversize = %d; want none proposed and 1 oversize", res.Groups, res.OversizeClusters)
+	}
+}
+
+// Two byte-identical copies plus a re-encode of the same recording are one
+// group, and it is only as certain as its weakest link.
+func TestGroupDuplicates_ExactGroupAbsorbedIntoAcoustic(t *testing.T) {
+	p := randomPrint(50, printLen)
+	hash := []byte("same-bytes")
+	res := groupDuplicates([]fingerprintCandidate{
+		{ID: "x1", DurationMs: 180000, StreamSHA256: hash, Chromaprint: p},
+		{ID: "x2", DurationMs: 180000, StreamSHA256: hash, Chromaprint: p},
+		{ID: "y", DurationMs: 180000, StreamSHA256: []byte("other-bytes"), Chromaprint: withBitNoise(p, 0.03, 51)},
+	}, defaultAcousticMaxBitErrorRate)
+	want := []string{"x1", "x2", "y"}
+	if len(res.Groups) != 1 || res.Groups[0].Tier != tierAcoustic || !reflect.DeepEqual(res.Groups[0].Members, want) {
+		t.Fatalf("groups = %+v, want one acoustic group %v", res.Groups, want)
+	}
+}
+
+func TestGroupDuplicates_UnrelatedTracksNeverGroup(t *testing.T) {
+	var cands []fingerprintCandidate
+	for i := range 6 {
+		cands = append(cands, fingerprintCandidate{
+			ID: string(rune('a' + i)), DurationMs: 210000, Chromaprint: randomPrint(uint64(60+i), printLen),
+		})
+	}
+	if res := groupDuplicates(cands, defaultAcousticMaxBitErrorRate); len(res.Groups) != 0 {
+		t.Fatalf("unrelated recordings were grouped: %+v", res.Groups)
+	}
+}
+
+func TestGroupDuplicates_OrderIndependent(t *testing.T) {
+	p := randomPrint(70, printLen)
+	q := randomPrint(71, printLen)
+	hash := []byte("identical")
+	cands := []fingerprintCandidate{
+		{ID: "p1", DurationMs: 200000, Chromaprint: p},
+		{ID: "p2", DurationMs: 201000, Chromaprint: withBitNoise(p, 0.04, 72)},
+		{ID: "q1", DurationMs: 150000, Chromaprint: q},
+		{ID: "q2", DurationMs: 150500, Chromaprint: withBitNoise(q, 0.02, 73)},
+		{ID: "h1", DurationMs: 90000, StreamSHA256: hash},
+		{ID: "h2", DurationMs: 90000, StreamSHA256: hash},
+		{ID: "lone", DurationMs: 200000, Chromaprint: randomPrint(74, printLen)},
+	}
+	want := groupDuplicates(cands, defaultAcousticMaxBitErrorRate)
+	if len(want.Groups) != 3 {
+		t.Fatalf("fixture broken: %d groups, want 3 (p, q, h)", len(want.Groups))
+	}
+	r := rand.New(rand.NewPCG(75, 76))
+	for range 20 {
+		shuffled := append([]fingerprintCandidate(nil), cands...)
+		r.Shuffle(len(shuffled), func(i, j int) { shuffled[i], shuffled[j] = shuffled[j], shuffled[i] })
+		if got := groupDuplicates(shuffled, defaultAcousticMaxBitErrorRate); !reflect.DeepEqual(got, want) {
+			t.Fatalf("input order changed the result:\n got  %+v\n want %+v", got, want)
+		}
+	}
+}