diff --git a/internal/library/duplicate_match.go b/internal/library/duplicate_match.go new file mode 100644 index 00000000..75539e88 --- /dev/null +++ b/internal/library/duplicate_match.go @@ -0,0 +1,357 @@ +package library + +import ( + "math" + "math/bits" + "sort" +) + +// Duplicate matching (M400 #3909). +// +// Pure functions over fingerprints: no database, no files. This is the part that +// decides whether two tracks in the operator's library are proposed as one +// recording, so every rule in it has to be falsifiable in a unit test. +// +// Two tiers, answering different questions: +// +// exact equal audio_stream_sha256 — the same encoded audio bytes. No score, +// no threshold, no false positives (the #3885 pair). +// acoustic chromaprint fingerprints that agree closely once aligned — the +// same recording at another bitrate or in another codec. +// +// The acoustic comparison follows the approach of AcoustID's pg_acoustid +// (acoustid_compare.c): vote on the relative offset between two fingerprints +// using items that agree in their high bits, then measure disagreement at the +// winning offset. Reimplemented from that description; no code was copied. +// The alignment window and match-bit width below are taken from it. + +// maxAlignOffsetItems bounds how far apart two fingerprints may be shifted and +// still be compared: ±120 items, about 15 seconds at chromaprint's ~8 items per +// second. Covers a leading silence trimmed differently or a short intro; the +// same bound pg_acoustid uses (ACOUSTID_MAX_ALIGN_OFFSET). +const maxAlignOffsetItems = 120 + +// alignMatchBits is how many high bits two items must share to vote for an +// offset. Matching whole 32-bit items would miss the same recording at another +// bitrate, whose low bits are noisier; 14 is pg_acoustid's MATCH_BITS. +const alignMatchBits = 14 + +// minOverlapItems is the least overlap worth a verdict: about 10 seconds. A few +// items agreeing perfectly is not evidence that two recordings are one. +const minOverlapItems = 80 + +// minDistinctFraction rejects low-information fingerprints before they can +// match. Near-silence, a sustained tone or a click track produces the same few +// items over and over, and two such tracks agree closely without being the +// same recording. Real music is overwhelmingly distinct item to item, so this +// floor only catches the pathological case. A judgment value, not a measured +// one — revisit if the sweep reports real tracks refused for it. +const minDistinctFraction = 0.3 + +// defaultAcousticMaxBitErrorRate is the most disagreement two aligned +// fingerprints may show and still be proposed as one recording. Unrelated audio +// sits near 0.5; the same recording re-encoded lands well under 0.1. +// +// Deliberately conservative. The operator's stated worry is the opposite of a +// missed duplicate: "the same song can appear in different albums, usually it's +// a different recording", and an instrumental shares its vocal version's +// harmony, which chroma features capture. A false merge is the failure that +// matters, and the report is reviewed anyway. This is an unmeasured default: +// calibrate it against real pairs once the backfill (#3908) has populated the +// library, then expose it in Settings (#3913). +const defaultAcousticMaxBitErrorRate = 0.15 + +// durationToleranceMs is how far apart two tracks' durations may be and still be +// compared. Encoders pad and trim a little; different edits differ by more. +const durationToleranceMs = 3000 + +// maxAcousticGroupSize caps an acoustic group. A cluster bigger than this is far +// more likely a shared jingle, a skit or a low-information pattern than eight +// copies of one recording, and proposing it would bury the real duplicates. +// Exact-tier groups are not capped: identical bytes are identical however many. +const maxAcousticGroupSize = 8 + +// acousticScore is the result of comparing two fingerprints. +type acousticScore struct { + // Offset is how many items b is shifted against a: b[i+Offset] aligns with + // a[i]. + Offset int + // Overlap is how many aligned items were compared. + Overlap int + // BitErrorRate is the fraction of differing bits over the overlap, 0..1. + BitErrorRate float64 +} + +// compareChromaprint aligns two raw fingerprints and measures how much they +// disagree. ok is false when no verdict is possible: no offset gathered any +// votes, the overlap at the best offset is too short, or either side carries +// too little information to mean anything. +func compareChromaprint(a, b []int32) (acousticScore, bool) { + if len(a) < minOverlapItems || len(b) < minOverlapItems { + return acousticScore{}, false + } + if !informative(a) || !informative(b) { + return acousticScore{}, false + } + + offset, ok := bestOffset(a, b) + if !ok { + return acousticScore{}, false + } + + // a[i] aligns with b[i+offset]; walk the indices valid on both sides. + start := max(0, -offset) + end := min(len(a), len(b)-offset) + overlap := end - start + if overlap < minOverlapItems { + return acousticScore{}, false + } + errBits := 0 + for i := start; i < end; i++ { + errBits += bits.OnesCount32(uint32(a[i]) ^ uint32(b[i+offset])) + } + return acousticScore{ + Offset: offset, + Overlap: overlap, + BitErrorRate: float64(errBits) / float64(32*overlap), + }, true +} + +// bestOffset returns the relative shift most items agree on. +func bestOffset(a, b []int32) (int, bool) { + // Index a's items by their high bits. Each bucket keeps only a few + // positions: a value repeating many times is uninformative, and letting it + // vote once per repeat would make every pairing O(n²). + const keepPerBucket = 4 + positions := make(map[uint32][]int, len(a)) + for i, v := range a { + key := uint32(v) >> (32 - alignMatchBits) + if p := positions[key]; len(p) < keepPerBucket { + positions[key] = append(p, i) + } + } + + votes := make([]int, 2*maxAlignOffsetItems+1) + for j, v := range b { + for _, i := range positions[uint32(v)>>(32-alignMatchBits)] { + off := j - i + if off >= -maxAlignOffsetItems && off <= maxAlignOffsetItems { + votes[off+maxAlignOffsetItems]++ + } + } + } + + best, bestVotes := 0, 0 + for k, n := range votes { + // Strictly greater keeps the smallest shift on a tie, which is the more + // likely truth and keeps the result deterministic. + if n > bestVotes || (n == bestVotes && n > 0 && abs(k-maxAlignOffsetItems) < abs(best)) { + best, bestVotes = k-maxAlignOffsetItems, n + } + } + return best, bestVotes > 0 +} + +// informative reports whether a fingerprint varies enough to be compared. +func informative(fp []int32) bool { + seen := make(map[int32]struct{}, len(fp)) + for _, v := range fp { + seen[v] = struct{}{} + } + return float64(len(seen)) >= minDistinctFraction*float64(len(fp)) +} + +// fingerprintCandidate is one track as the grouping sees it. +type fingerprintCandidate struct { + ID string + DurationMs int32 + StreamSHA256 []byte + Chromaprint []int32 +} + +// duplicateTier names what a group's evidence is. +type duplicateTier string + +const ( + tierExact duplicateTier = "exact" + tierAcoustic duplicateTier = "acoustic" +) + +// duplicateGroup is a set of tracks proposed as one recording. Members are +// sorted by ID. +type duplicateGroup struct { + Tier duplicateTier + Members []string + // WorstBitErrorRate is the largest disagreement between any two members of + // an acoustic group — the weakest evidence the group rests on. Zero for + // exact groups. + WorstBitErrorRate float64 +} + +// groupingResult is what one grouping pass found. +type groupingResult struct { + Groups []duplicateGroup + // OversizeClusters counts acoustic clusters discarded for exceeding + // maxAcousticGroupSize. Reported rather than silent: a sudden rise means the + // cap or the information floor needs attention. + OversizeClusters int +} + +// groupDuplicates proposes duplicate groups among candidates. +// +// Exact groups come first: tracks sharing an audio stream hash. Each exact group +// is then treated as a single unit for the acoustic pass, so its members are +// never compared with each other again. +// +// Acoustic grouping is COMPLETE-LINKAGE: a unit joins a group only if it matches +// every unit already in it, within the duration tolerance and the bit-error +// limit. Single-linkage would let a chain of near-misses — A close to B, B close +// to C — drag A and C, which are not close, into one proposed merge. Complete +// linkage also means any member can be chosen as the survivor (#3911). +// +// When an acoustic group absorbs an exact group, the result is tier acoustic: +// a group is only as certain as its weakest link. +// +// The output does not depend on input order. +func groupDuplicates(cands []fingerprintCandidate, maxBitErrorRate float64) groupingResult { + var res groupingResult + + // Exact tier. + byHash := map[string][]fingerprintCandidate{} + var noHash []fingerprintCandidate + for _, c := range cands { + if len(c.StreamSHA256) == 0 { + noHash = append(noHash, c) + continue + } + k := string(c.StreamSHA256) + byHash[k] = append(byHash[k], c) + } + + // A unit is one exact group, or one track with no exact duplicate. + type unit struct { + members []fingerprintCandidate + durationMs int32 + print []int32 + exact bool + } + var units []unit + for _, group := range byHash { + sortCandidates(group) + u := unit{members: group, durationMs: group[0].DurationMs, exact: len(group) > 1} + for _, m := range group { + if len(m.Chromaprint) > 0 { + u.print = m.Chromaprint + break + } + } + units = append(units, u) + } + for _, c := range noHash { + units = append(units, unit{members: []fingerprintCandidate{c}, durationMs: c.DurationMs, print: c.Chromaprint}) + } + + // Deterministic order: duration, then the first member's ID. Sorting by + // duration also lets the scan below stop as soon as durations are too far + // apart, which is the blocking #3910 relies on. + sort.Slice(units, func(i, j int) bool { + if units[i].durationMs != units[j].durationMs { + return units[i].durationMs < units[j].durationMs + } + return units[i].members[0].ID < units[j].members[0].ID + }) + + assigned := make([]bool, len(units)) + for i := range units { + if assigned[i] || len(units[i].print) == 0 { + continue + } + group := []int{i} + worst := 0.0 + for j := i + 1; j < len(units); j++ { + if units[j].durationMs-units[i].durationMs > durationToleranceMs { + break + } + if assigned[j] || len(units[j].print) == 0 { + continue + } + // Complete linkage: j must match every member so far. + joined, worstWithJ := true, worst + for _, g := range group { + if abs32(units[j].durationMs-units[g].durationMs) > durationToleranceMs { + joined = false + break + } + score, ok := compareChromaprint(units[g].print, units[j].print) + if !ok || score.BitErrorRate > maxBitErrorRate { + joined = false + break + } + worstWithJ = math.Max(worstWithJ, score.BitErrorRate) + } + if joined { + group = append(group, j) + worst = worstWithJ + } + } + + if len(group) == 1 { + continue + } + // Count units, not tracks: an absorbed exact group is one piece of + // acoustic evidence however many identical files it holds. + if len(group) > maxAcousticGroupSize { + res.OversizeClusters++ + for _, g := range group { + assigned[g] = true + } + continue + } + var members []string + for _, g := range group { + assigned[g] = true + for _, m := range units[g].members { + members = append(members, m.ID) + } + } + sort.Strings(members) + res.Groups = append(res.Groups, duplicateGroup{ + Tier: tierAcoustic, Members: members, WorstBitErrorRate: worst, + }) + } + + // Exact groups that no acoustic group absorbed stand on their own. + for i, u := range units { + if assigned[i] || !u.exact { + continue + } + members := make([]string, len(u.members)) + for k, m := range u.members { + members[k] = m.ID + } + res.Groups = append(res.Groups, duplicateGroup{Tier: tierExact, Members: members}) + } + + sort.Slice(res.Groups, func(i, j int) bool { + return res.Groups[i].Members[0] < res.Groups[j].Members[0] + }) + return res +} + +func sortCandidates(cs []fingerprintCandidate) { + sort.Slice(cs, func(i, j int) bool { return cs[i].ID < cs[j].ID }) +} + +func abs(n int) int { + if n < 0 { + return -n + } + return n +} + +func abs32(n int32) int32 { + if n < 0 { + return -n + } + return n +} diff --git a/internal/library/duplicate_match_test.go b/internal/library/duplicate_match_test.go new file mode 100644 index 00000000..f0d2decf --- /dev/null +++ b/internal/library/duplicate_match_test.go @@ -0,0 +1,241 @@ +package library + +import ( + "math" + "math/rand/v2" + "reflect" + "testing" +) + +// printLen is a realistic fingerprint length: fpcalc's 120s at ~8 items/second. +const printLen = 960 + +// randomPrint is a deterministic stand-in for one recording's fingerprint. +func randomPrint(seed uint64, n int) []int32 { + r := rand.New(rand.NewPCG(seed, seed^0x9e3779b97f4a7c15)) + fp := make([]int32, n) + for i := range fp { + fp[i] = int32(r.Uint32()) + } + return fp +} + +// withBitNoise flips exactly round(fraction × all bits) distinct bits — the +// same recording through a different encoder, at a known bit-error rate. +func withBitNoise(fp []int32, fraction float64, seed uint64) []int32 { + out := append([]int32(nil), fp...) + r := rand.New(rand.NewPCG(seed, seed^0x243f6a8885a308d3)) + total := 32 * len(fp) + for _, pos := range r.Perm(total)[:int(math.Round(fraction*float64(total)))] { + out[pos/32] ^= int32(uint32(1) << (pos % 32)) + } + return out +} + +func constantPrint(v int32, n int) []int32 { + fp := make([]int32, n) + for i := range fp { + fp[i] = v + } + return fp +} + +func TestCompareChromaprint(t *testing.T) { + base := randomPrint(1, printLen) + + t.Run("identical", func(t *testing.T) { + got, ok := compareChromaprint(base, base) + if !ok || got.BitErrorRate != 0 || got.Offset != 0 || got.Overlap != printLen { + t.Fatalf("got %+v ok=%v, want an exact alignment", got, ok) + } + }) + + t.Run("re-encoded: known bit noise is measured exactly", func(t *testing.T) { + got, ok := compareChromaprint(base, withBitNoise(base, 0.03, 2)) + if !ok { + t.Fatal("a re-encode was not comparable") + } + if want := math.Round(0.03*32*printLen) / (32 * printLen); got.BitErrorRate != want { + t.Fatalf("BitErrorRate = %v, want %v", got.BitErrorRate, want) + } + }) + + // b starts 40 items later in the same audio: b[j] = a[j+40], so a[i] aligns + // with b[i-40]. + t.Run("offset inside the window is recovered", func(t *testing.T) { + got, ok := compareChromaprint(base, base[40:]) + if !ok || got.Offset != -40 || got.BitErrorRate != 0 || got.Overlap != printLen-40 { + t.Fatalf("got %+v ok=%v, want offset -40 with no error", got, ok) + } + }) + + t.Run("offset beyond the window never matches", func(t *testing.T) { + got, ok := compareChromaprint(base, base[200:]) + if ok && got.BitErrorRate <= defaultAcousticMaxBitErrorRate { + t.Fatalf("a 200-item shift matched: %+v", got) + } + }) + + t.Run("unrelated recordings sit near 0.5", func(t *testing.T) { + got, ok := compareChromaprint(base, randomPrint(99, printLen)) + if ok && got.BitErrorRate < 0.4 { + t.Fatalf("unrelated fingerprints scored %v", got.BitErrorRate) + } + }) + + t.Run("too short an overlap gives no verdict", func(t *testing.T) { + if got, ok := compareChromaprint(base, base[:minOverlapItems-1]); ok { + t.Fatalf("a %d-item fingerprint was compared: %+v", minOverlapItems-1, got) + } + }) + + // Two near-silent tracks agree perfectly without being one recording. The + // information floor is the only thing standing between them and a merge. + t.Run("low-information fingerprints give no verdict", func(t *testing.T) { + silence := constantPrint(0x1234, printLen) + if got, ok := compareChromaprint(silence, silence); ok { + t.Fatalf("silence compared as a match: %+v", got) + } + }) + + t.Run("the threshold separates close from not close", func(t *testing.T) { + near, _ := compareChromaprint(base, withBitNoise(base, 0.10, 3)) + far, _ := compareChromaprint(base, withBitNoise(base, 0.20, 4)) + if near.BitErrorRate > defaultAcousticMaxBitErrorRate { + t.Errorf("10%% noise (%v) is over the threshold", near.BitErrorRate) + } + if far.BitErrorRate <= defaultAcousticMaxBitErrorRate { + t.Errorf("20%% noise (%v) is under the threshold", far.BitErrorRate) + } + }) +} + +func TestGroupDuplicates_ExactTier(t *testing.T) { + hash := []byte("sha256-of-www-instrumental-bytes") + res := groupDuplicates([]fingerprintCandidate{ + {ID: "www-01", DurationMs: 215000, StreamSHA256: hash}, + {ID: "www-02", DurationMs: 215000, StreamSHA256: hash}, + {ID: "lovesick", DurationMs: 198000, StreamSHA256: []byte("another")}, + }, defaultAcousticMaxBitErrorRate) + want := []duplicateGroup{{Tier: tierExact, Members: []string{"www-01", "www-02"}}} + if !reflect.DeepEqual(res.Groups, want) { + t.Fatalf("groups = %+v, want %+v", res.Groups, want) + } +} + +func TestGroupDuplicates_AcousticPair(t *testing.T) { + p := randomPrint(10, printLen) + res := groupDuplicates([]fingerprintCandidate{ + {ID: "album", DurationMs: 240000, Chromaprint: p}, + {ID: "compilation", DurationMs: 241000, Chromaprint: withBitNoise(p, 0.05, 11)}, + }, defaultAcousticMaxBitErrorRate) + if len(res.Groups) != 1 || res.Groups[0].Tier != tierAcoustic || + !reflect.DeepEqual(res.Groups[0].Members, []string{"album", "compilation"}) { + t.Fatalf("groups = %+v, want one acoustic pair", res.Groups) + } + if got := res.Groups[0].WorstBitErrorRate; math.Abs(got-0.05) > 0.001 { + t.Fatalf("WorstBitErrorRate = %v, want about 0.05", got) + } +} + +// A is close to B and B is close to C, but A and C are not close. Under +// single linkage all three would be proposed as one recording; complete linkage +// must keep C out. +func TestGroupDuplicates_NoChaining(t *testing.T) { + a := randomPrint(20, printLen) + b := withBitNoise(a, 0.10, 21) + c := withBitNoise(b, 0.10, 22) + if s, _ := compareChromaprint(a, c); s.BitErrorRate <= defaultAcousticMaxBitErrorRate { + t.Fatalf("fixture broken: A and C are close (%v), so this cannot test chaining", s.BitErrorRate) + } + res := groupDuplicates([]fingerprintCandidate{ + {ID: "a", DurationMs: 200000, Chromaprint: a}, + {ID: "b", DurationMs: 200000, Chromaprint: b}, + {ID: "c", DurationMs: 200000, Chromaprint: c}, + }, defaultAcousticMaxBitErrorRate) + if len(res.Groups) != 1 || !reflect.DeepEqual(res.Groups[0].Members, []string{"a", "b"}) { + t.Fatalf("groups = %+v, want only {a, b}", res.Groups) + } +} + +func TestGroupDuplicates_DurationTolerance(t *testing.T) { + p := randomPrint(30, printLen) + res := groupDuplicates([]fingerprintCandidate{ + {ID: "edit", DurationMs: 200000, Chromaprint: p}, + {ID: "extended", DurationMs: 200000 + durationToleranceMs + 1, Chromaprint: p}, + }, defaultAcousticMaxBitErrorRate) + if len(res.Groups) != 0 { + t.Fatalf("tracks %dms apart were grouped: %+v", durationToleranceMs+1, res.Groups) + } +} + +// Nine tracks that all match are far likelier a shared jingle than nine copies +// of one recording. The cluster must be reported, not proposed. +func TestGroupDuplicates_OversizeClusterIsDiscarded(t *testing.T) { + p := randomPrint(40, printLen) + var cands []fingerprintCandidate + for i := range maxAcousticGroupSize + 1 { + cands = append(cands, fingerprintCandidate{ + ID: string(rune('a' + i)), DurationMs: 30000, Chromaprint: withBitNoise(p, 0.01, uint64(100+i)), + }) + } + res := groupDuplicates(cands, defaultAcousticMaxBitErrorRate) + if len(res.Groups) != 0 || res.OversizeClusters != 1 { + t.Fatalf("groups = %+v, oversize = %d; want none proposed and 1 oversize", res.Groups, res.OversizeClusters) + } +} + +// Two byte-identical copies plus a re-encode of the same recording are one +// group, and it is only as certain as its weakest link. +func TestGroupDuplicates_ExactGroupAbsorbedIntoAcoustic(t *testing.T) { + p := randomPrint(50, printLen) + hash := []byte("same-bytes") + res := groupDuplicates([]fingerprintCandidate{ + {ID: "x1", DurationMs: 180000, StreamSHA256: hash, Chromaprint: p}, + {ID: "x2", DurationMs: 180000, StreamSHA256: hash, Chromaprint: p}, + {ID: "y", DurationMs: 180000, StreamSHA256: []byte("other-bytes"), Chromaprint: withBitNoise(p, 0.03, 51)}, + }, defaultAcousticMaxBitErrorRate) + want := []string{"x1", "x2", "y"} + if len(res.Groups) != 1 || res.Groups[0].Tier != tierAcoustic || !reflect.DeepEqual(res.Groups[0].Members, want) { + t.Fatalf("groups = %+v, want one acoustic group %v", res.Groups, want) + } +} + +func TestGroupDuplicates_UnrelatedTracksNeverGroup(t *testing.T) { + var cands []fingerprintCandidate + for i := range 6 { + cands = append(cands, fingerprintCandidate{ + ID: string(rune('a' + i)), DurationMs: 210000, Chromaprint: randomPrint(uint64(60+i), printLen), + }) + } + if res := groupDuplicates(cands, defaultAcousticMaxBitErrorRate); len(res.Groups) != 0 { + t.Fatalf("unrelated recordings were grouped: %+v", res.Groups) + } +} + +func TestGroupDuplicates_OrderIndependent(t *testing.T) { + p := randomPrint(70, printLen) + q := randomPrint(71, printLen) + hash := []byte("identical") + cands := []fingerprintCandidate{ + {ID: "p1", DurationMs: 200000, Chromaprint: p}, + {ID: "p2", DurationMs: 201000, Chromaprint: withBitNoise(p, 0.04, 72)}, + {ID: "q1", DurationMs: 150000, Chromaprint: q}, + {ID: "q2", DurationMs: 150500, Chromaprint: withBitNoise(q, 0.02, 73)}, + {ID: "h1", DurationMs: 90000, StreamSHA256: hash}, + {ID: "h2", DurationMs: 90000, StreamSHA256: hash}, + {ID: "lone", DurationMs: 200000, Chromaprint: randomPrint(74, printLen)}, + } + want := groupDuplicates(cands, defaultAcousticMaxBitErrorRate) + if len(want.Groups) != 3 { + t.Fatalf("fixture broken: %d groups, want 3 (p, q, h)", len(want.Groups)) + } + r := rand.New(rand.NewPCG(75, 76)) + for range 20 { + shuffled := append([]fingerprintCandidate(nil), cands...) + r.Shuffle(len(shuffled), func(i, j int) { shuffled[i], shuffled[j] = shuffled[j], shuffled[i] }) + if got := groupDuplicates(shuffled, defaultAcousticMaxBitErrorRate); !reflect.DeepEqual(got, want) { + t.Fatalf("input order changed the result:\n got %+v\n want %+v", got, want) + } + } +}