minstrel/internal/library/duplicate_match.go

package library

import (
	"math"
	"math/bits"
	"sort"
)

// Duplicate matching (M400 #3909).
//
// Pure functions over fingerprints: no database, no files. This is the part that
// decides whether two tracks in the operator's library are proposed as one
// recording, so every rule in it has to be falsifiable in a unit test.
//
// Two tiers, answering different questions:
//
//	exact     equal audio_stream_sha256 — the same encoded audio bytes. No score,
//	          no threshold, no false positives (the #3885 pair).
//	acoustic  chromaprint fingerprints that agree closely once aligned — the
//	          same recording at another bitrate or in another codec.
//
// The acoustic comparison follows the approach of AcoustID's pg_acoustid
// (acoustid_compare.c): vote on the relative offset between two fingerprints
// using items that agree in their high bits, then measure disagreement at the
// winning offset. Reimplemented from that description; no code was copied.
// The alignment window and match-bit width below are taken from it.

// maxAlignOffsetItems bounds how far apart two fingerprints may be shifted and
// still be compared: ±120 items, about 15 seconds at chromaprint's ~8 items per
// second. Covers a leading silence trimmed differently or a short intro; the
// same bound pg_acoustid uses (ACOUSTID_MAX_ALIGN_OFFSET).
const maxAlignOffsetItems = 120

// alignMatchBits is how many high bits two items must share to vote for an
// offset. Matching whole 32-bit items would miss the same recording at another
// bitrate, whose low bits are noisier; 14 is pg_acoustid's MATCH_BITS.
const alignMatchBits = 14

// minOverlapItems is the least overlap worth a verdict: about 10 seconds. A few
// items agreeing perfectly is not evidence that two recordings are one.
const minOverlapItems = 80

// minDistinctFraction rejects low-information fingerprints before they can
// match. Near-silence, a sustained tone or a click track produces the same few
// items over and over, and two such tracks agree closely without being the
// same recording. Real music is overwhelmingly distinct item to item, so this
// floor only catches the pathological case. A judgment value, not a measured
// one — revisit if the sweep reports real tracks refused for it.
const minDistinctFraction = 0.3

// defaultAcousticMaxBitErrorRate is the most disagreement two aligned
// fingerprints may show and still be proposed as one recording. Unrelated audio
// sits near 0.5; the same recording re-encoded lands well under 0.1.
//
// Deliberately conservative. The operator's stated worry is the opposite of a
// missed duplicate: "the same song can appear in different albums, usually it's
// a different recording", and an instrumental shares its vocal version's
// harmony, which chroma features capture. A false merge is the failure that
// matters, and the report is reviewed anyway. This is an unmeasured default:
// calibrate it against real pairs once the backfill (#3908) has populated the
// library, then expose it in Settings (#3913).
const defaultAcousticMaxBitErrorRate = 0.15

// durationToleranceMs is how far apart two tracks' durations may be and still be
// compared. Encoders pad and trim a little; different edits differ by more.
const durationToleranceMs = 3000

// maxAcousticGroupSize caps an acoustic group. A cluster bigger than this is far
// more likely a shared jingle, a skit or a low-information pattern than eight
// copies of one recording, and proposing it would bury the real duplicates.
// Exact-tier groups are not capped: identical bytes are identical however many.
const maxAcousticGroupSize = 8

// acousticScore is the result of comparing two fingerprints.
type acousticScore struct {
	// Offset is how many items b is shifted against a: b[i+Offset] aligns with
	// a[i].
	Offset int
	// Overlap is how many aligned items were compared.
	Overlap int
	// BitErrorRate is the fraction of differing bits over the overlap, 0..1.
	BitErrorRate float64
}

// compareChromaprint aligns two raw fingerprints and measures how much they
// disagree. ok is false when no verdict is possible: no offset gathered any
// votes, the overlap at the best offset is too short, or either side carries
// too little information to mean anything.
func compareChromaprint(a, b []int32) (acousticScore, bool) {
	if len(a) < minOverlapItems || len(b) < minOverlapItems {
		return acousticScore{}, false
	}
	if !informative(a) || !informative(b) {
		return acousticScore{}, false
	}

	offset, ok := bestOffset(a, b)
	if !ok {
		return acousticScore{}, false
	}

	// a[i] aligns with b[i+offset]; walk the indices valid on both sides.
	start := max(0, -offset)
	end := min(len(a), len(b)-offset)
	overlap := end - start
	if overlap < minOverlapItems {
		return acousticScore{}, false
	}
	errBits := 0
	for i := start; i < end; i++ {
		errBits += bits.OnesCount32(uint32(a[i]) ^ uint32(b[i+offset]))
	}
	return acousticScore{
		Offset:       offset,
		Overlap:      overlap,
		BitErrorRate: float64(errBits) / float64(32*overlap),
	}, true
}

// bestOffset returns the relative shift most items agree on.
func bestOffset(a, b []int32) (int, bool) {
	// Index a's items by their high bits. Each bucket keeps only a few
	// positions: a value repeating many times is uninformative, and letting it
	// vote once per repeat would make every pairing O(n²).
	const keepPerBucket = 4
	positions := make(map[uint32][]int, len(a))
	for i, v := range a {
		key := uint32(v) >> (32 - alignMatchBits)
		if p := positions[key]; len(p) < keepPerBucket {
			positions[key] = append(p, i)
		}
	}

	votes := make([]int, 2*maxAlignOffsetItems+1)
	for j, v := range b {
		for _, i := range positions[uint32(v)>>(32-alignMatchBits)] {
			off := j - i
			if off >= -maxAlignOffsetItems && off <= maxAlignOffsetItems {
				votes[off+maxAlignOffsetItems]++
			}
		}
	}

	best, bestVotes := 0, 0
	for k, n := range votes {
		// Strictly greater keeps the smallest shift on a tie, which is the more
		// likely truth and keeps the result deterministic.
		if n > bestVotes || (n == bestVotes && n > 0 && abs(k-maxAlignOffsetItems) < abs(best)) {
			best, bestVotes = k-maxAlignOffsetItems, n
		}
	}
	return best, bestVotes > 0
}

// informative reports whether a fingerprint varies enough to be compared.
func informative(fp []int32) bool {
	seen := make(map[int32]struct{}, len(fp))
	for _, v := range fp {
		seen[v] = struct{}{}
	}
	return float64(len(seen)) >= minDistinctFraction*float64(len(fp))
}

// fingerprintCandidate is one track as the grouping sees it.
type fingerprintCandidate struct {
	ID           string
	DurationMs   int32
	StreamSHA256 []byte
	Chromaprint  []int32
}

// duplicateTier names what a group's evidence is.
type duplicateTier string

const (
	tierExact    duplicateTier = "exact"
	tierAcoustic duplicateTier = "acoustic"
)

// duplicateGroup is a set of tracks proposed as one recording. Members are
// sorted by ID.
type duplicateGroup struct {
	Tier    duplicateTier
	Members []string
	// WorstBitErrorRate is the largest disagreement between any two members of
	// an acoustic group — the weakest evidence the group rests on. Zero for
	// exact groups.
	WorstBitErrorRate float64
}

// groupingResult is what one grouping pass found.
type groupingResult struct {
	Groups []duplicateGroup
	// OversizeClusters counts acoustic clusters discarded for exceeding
	// maxAcousticGroupSize. Reported rather than silent: a sudden rise means the
	// cap or the information floor needs attention.
	OversizeClusters int
}

// groupDuplicates proposes duplicate groups among candidates.
//
// Exact groups come first: tracks sharing an audio stream hash. Each exact group
// is then treated as a single unit for the acoustic pass, so its members are
// never compared with each other again.
//
// Acoustic grouping is COMPLETE-LINKAGE: a unit joins a group only if it matches
// every unit already in it, within the duration tolerance and the bit-error
// limit. Single-linkage would let a chain of near-misses — A close to B, B close
// to C — drag A and C, which are not close, into one proposed merge. Complete
// linkage also means any member can be chosen as the survivor (#3911).
//
// When an acoustic group absorbs an exact group, the result is tier acoustic:
// a group is only as certain as its weakest link.
//
// The output does not depend on input order.
func groupDuplicates(cands []fingerprintCandidate, maxBitErrorRate float64) groupingResult {
	var res groupingResult

	// Exact tier.
	byHash := map[string][]fingerprintCandidate{}
	var noHash []fingerprintCandidate
	for _, c := range cands {
		if len(c.StreamSHA256) == 0 {
			noHash = append(noHash, c)
			continue
		}
		k := string(c.StreamSHA256)
		byHash[k] = append(byHash[k], c)
	}

	// A unit is one exact group, or one track with no exact duplicate.
	type unit struct {
		members    []fingerprintCandidate
		durationMs int32
		print      []int32
		exact      bool
	}
	var units []unit
	for _, group := range byHash {
		sortCandidates(group)
		u := unit{members: group, durationMs: group[0].DurationMs, exact: len(group) > 1}
		for _, m := range group {
			if len(m.Chromaprint) > 0 {
				u.print = m.Chromaprint
				break
			}
		}
		units = append(units, u)
	}
	for _, c := range noHash {
		units = append(units, unit{members: []fingerprintCandidate{c}, durationMs: c.DurationMs, print: c.Chromaprint})
	}

	// Deterministic order: duration, then the first member's ID. Sorting by
	// duration also lets the scan below stop as soon as durations are too far
	// apart, which is the blocking #3910 relies on.
	sort.Slice(units, func(i, j int) bool {
		if units[i].durationMs != units[j].durationMs {
			return units[i].durationMs < units[j].durationMs
		}
		return units[i].members[0].ID < units[j].members[0].ID
	})

	assigned := make([]bool, len(units))
	for i := range units {
		if assigned[i] || len(units[i].print) == 0 {
			continue
		}
		group := []int{i}
		worst := 0.0
		for j := i + 1; j < len(units); j++ {
			if units[j].durationMs-units[i].durationMs > durationToleranceMs {
				break
			}
			if assigned[j] || len(units[j].print) == 0 {
				continue
			}
			// Complete linkage: j must match every member so far.
			joined, worstWithJ := true, worst
			for _, g := range group {
				if abs32(units[j].durationMs-units[g].durationMs) > durationToleranceMs {
					joined = false
					break
				}
				score, ok := compareChromaprint(units[g].print, units[j].print)
				if !ok || score.BitErrorRate > maxBitErrorRate {
					joined = false
					break
				}
				worstWithJ = math.Max(worstWithJ, score.BitErrorRate)
			}
			if joined {
				group = append(group, j)
				worst = worstWithJ
			}
		}

		if len(group) == 1 {
			continue
		}
		// Count units, not tracks: an absorbed exact group is one piece of
		// acoustic evidence however many identical files it holds.
		if len(group) > maxAcousticGroupSize {
			res.OversizeClusters++
			for _, g := range group {
				assigned[g] = true
			}
			continue
		}
		var members []string
		for _, g := range group {
			assigned[g] = true
			for _, m := range units[g].members {
				members = append(members, m.ID)
			}
		}
		sort.Strings(members)
		res.Groups = append(res.Groups, duplicateGroup{
			Tier: tierAcoustic, Members: members, WorstBitErrorRate: worst,
		})
	}

	// Exact groups that no acoustic group absorbed stand on their own.
	for i, u := range units {
		if assigned[i] || !u.exact {
			continue
		}
		members := make([]string, len(u.members))
		for k, m := range u.members {
			members[k] = m.ID
		}
		res.Groups = append(res.Groups, duplicateGroup{Tier: tierExact, Members: members})
	}

	sort.Slice(res.Groups, func(i, j int) bool {
		return res.Groups[i].Members[0] < res.Groups[j].Members[0]
	})
	return res
}

func sortCandidates(cs []fingerprintCandidate) {
	sort.Slice(cs, func(i, j int) bool { return cs[i].ID < cs[j].ID })
}

func abs(n int) int {
	if n < 0 {
		return -n
	}
	return n
}

func abs32(n int32) int32 {
	if n < 0 {
		return -n
	}
	return n
}