feat(library): the duplicate sweep — propose duplicate groups from fingerprints (M400 #3910)

Reads fingerprints, runs them through the matcher, and records
proposals in duplicate_groups (migration 0059). Nothing is merged or
deleted: a group is a proposal for the admin report (#3912).

Streaming. The whole library's fingerprints are hundreds of megabytes,
but tracks are only compared within 3s of each other in duration. So
candidates stream in (duration_ms, id) order, keyset-paged on a new
tracks(duration_ms, id) index. The grouper holds only the tracks within
3s of the oldest one not yet settled. A seed is settled once a track
arrives beyond its window, which gives the same result as grouping the
whole sorted list. groupDuplicates is rebuilt on the same streamGrouper,
so there is one grouping rule and the #3909 tests still cover it. Each
fingerprint's alignment index and variety check are computed once
instead of for every pair.

Exact duplicates are grouped library-wide in SQL. The first member the
stream meets stands in for the whole group in the acoustic pass. An
exact group caught in an oversize acoustic cluster is still proposed:
the acoustic evidence is discarded, identical bytes are not.

Re-sweeping:
- a group is identified by its sorted member ids, so finding it again
  refreshes the row in place
- a proposal whose members all sat in one dismissed group is not
  proposed again (a subset repeats the verdict; a superset is new
  evidence)
- a pending proposal no sweep has found again is retired, but only
  after a complete sweep, and only if an earlier sweep last confirmed
  it, so two overlapping sweeps cannot delete each other's findings
- dismissals are kept

DuplicateSweepWorker checks hourly and sweeps only when a fingerprint
was written after the last sweep started. TryStartDuplicateSweep guards
against two sweeps at once and reaps one stuck in flight for 2h. The
sweep row is closed on a detached context with a deadline, so a sweep
cancelled at shutdown still records that it ended.

The integration test pages one row at a time and checks:
- an acoustic pair and an exact pair are found
- a track with no fingerprint, a missing track and a near-duration
  unrelated song are left out
- a dismissed group is suppressed while the pending one refreshes
  without duplicating
- a proposal that stops holding is retired and the dismissal survives

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01SQ31KQpYbStyK5y58UmPLH

internal/library/duplicate_match.go

@@ -82,15 +82,52 @@ type acousticScore struct {
 	BitErrorRate float64
 }
 
+// preparedPrint is a fingerprint with the parts every comparison needs worked
+// out once. The sweep compares each track with every other track within a few
+// seconds of its duration, so rebuilding the alignment index for each pair would
+// dominate its cost.
+type preparedPrint struct {
+	items       []int32
+	index       map[uint32][]int
+	informative bool
+}
+
+// preparePrint indexes a fingerprint's items by their high bits and records
+// whether it varies enough to be compared at all.
+func preparePrint(fp []int32) *preparedPrint {
+	// Each bucket keeps only a few positions: a value repeating many times is
+	// uninformative, and letting it vote once per repeat would make every
+	// pairing O(n²).
+	const keepPerBucket = 4
+	p := &preparedPrint{items: fp, index: make(map[uint32][]int, len(fp))}
+	seen := make(map[int32]struct{}, len(fp))
+	for i, v := range fp {
+		seen[v] = struct{}{}
+		key := alignKey(v)
+		if pos := p.index[key]; len(pos) < keepPerBucket {
+			p.index[key] = append(pos, i)
+		}
+	}
+	p.informative = len(fp) > 0 && float64(len(seen)) >= minDistinctFraction*float64(len(fp))
+	return p
+}
+
+func alignKey(v int32) uint32 { return uint32(v) >> (32 - alignMatchBits) }
+
 // compareChromaprint aligns two raw fingerprints and measures how much they
 // disagree. ok is false when no verdict is possible: no offset gathered any
 // votes, the overlap at the best offset is too short, or either side carries
 // too little information to mean anything.
 func compareChromaprint(a, b []int32) (acousticScore, bool) {
-	if len(a) < minOverlapItems || len(b) < minOverlapItems {
+	return comparePrepared(preparePrint(a), preparePrint(b))
+}
+
+// comparePrepared is compareChromaprint over fingerprints already prepared.
+func comparePrepared(a, b *preparedPrint) (acousticScore, bool) {
+	if len(a.items) < minOverlapItems || len(b.items) < minOverlapItems {
 		return acousticScore{}, false
 	}
-	if !informative(a) || !informative(b) {
+	if !a.informative || !b.informative {
 		return acousticScore{}, false
 	}
 
@@ -101,14 +138,14 @@ func compareChromaprint(a, b []int32) (acousticScore, bool) {
 
 	// a[i] aligns with b[i+offset]; walk the indices valid on both sides.
 	start := max(0, -offset)
-	end := min(len(a), len(b)-offset)
+	end := min(len(a.items), len(b.items)-offset)
 	overlap := end - start
 	if overlap < minOverlapItems {
 		return acousticScore{}, false
 	}
 	errBits := 0
 	for i := start; i < end; i++ {
-		errBits += bits.OnesCount32(uint32(a[i]) ^ uint32(b[i+offset]))
+		errBits += bits.OnesCount32(uint32(a.items[i]) ^ uint32(b.items[i+offset]))
 	}
 	return acousticScore{
 		Offset:       offset,
@@ -118,22 +155,10 @@ func compareChromaprint(a, b []int32) (acousticScore, bool) {
 }
 
 // bestOffset returns the relative shift most items agree on.
-func bestOffset(a, b []int32) (int, bool) {
-	// Index a's items by their high bits. Each bucket keeps only a few
-	// positions: a value repeating many times is uninformative, and letting it
-	// vote once per repeat would make every pairing O(n²).
-	const keepPerBucket = 4
-	positions := make(map[uint32][]int, len(a))
-	for i, v := range a {
-		key := uint32(v) >> (32 - alignMatchBits)
-		if p := positions[key]; len(p) < keepPerBucket {
-			positions[key] = append(p, i)
-		}
-	}
-
+func bestOffset(a, b *preparedPrint) (int, bool) {
 	votes := make([]int, 2*maxAlignOffsetItems+1)
-	for j, v := range b {
-		for _, i := range positions[uint32(v)>>(32-alignMatchBits)] {
+	for j, v := range b.items {
+		for _, i := range a.index[alignKey(v)] {
 			off := j - i
 			if off >= -maxAlignOffsetItems && off <= maxAlignOffsetItems {
 				votes[off+maxAlignOffsetItems]++
@@ -152,15 +177,6 @@ func bestOffset(a, b []int32) (int, bool) {
 	return best, bestVotes > 0
 }
 
-// informative reports whether a fingerprint varies enough to be compared.
-func informative(fp []int32) bool {
-	seen := make(map[int32]struct{}, len(fp))
-	for _, v := range fp {
-		seen[v] = struct{}{}
-	}
-	return float64(len(seen)) >= minDistinctFraction*float64(len(fp))
-}
-
 // fingerprintCandidate is one track as the grouping sees it.
 type fingerprintCandidate struct {
 	ID           string
@@ -197,149 +213,206 @@ type groupingResult struct {
 	OversizeClusters int
 }
 
-// groupDuplicates proposes duplicate groups among candidates.
+// groupUnit is one thing the acoustic pass compares: a single track, or an exact
+// group standing in for all its byte-identical copies.
+type groupUnit struct {
+	ids        []string // every member, sorted
+	durationMs int32
+	sortKey    string // the representative's id: ties on duration break on it
+	print      *preparedPrint
+	exact      bool // more than one member with identical audio
+	assigned   bool
+}
+
+// streamGrouper is the acoustic pass over units arriving in (durationMs, sortKey)
+// order. It holds only the units within durationToleranceMs of the oldest one
+// not yet settled, so memory is bounded by the densest few seconds of the
+// library rather than by its size — the whole library's fingerprints would be
+// hundreds of megabytes.
 //
-// Exact groups come first: tracks sharing an audio stream hash. Each exact group
-// is then treated as a single unit for the acoustic pass, so its members are
-// never compared with each other again.
+// A seed can be settled as soon as a unit arrives beyond its window: everything
+// it could group with has already arrived, and no later seed can reach back to
+// it because seeds only look forward. That is what makes the streamed result
+// identical to running the same pass over the whole sorted list.
 //
-// Acoustic grouping is COMPLETE-LINKAGE: a unit joins a group only if it matches
-// every unit already in it, within the duration tolerance and the bit-error
-// limit. Single-linkage would let a chain of near-misses — A close to B, B close
-// to C — drag A and C, which are not close, into one proposed merge. Complete
-// linkage also means any member can be chosen as the survivor (#3911).
+// Grouping is COMPLETE-LINKAGE: a unit joins a group only if it matches every
+// unit already in it, within the duration tolerance and the bit-error limit.
+// Single-linkage would let a chain of near-misses — A close to B, B close to C —
+// drag A and C, which are not close, into one proposed merge. Complete linkage
+// also means any member can be chosen as the survivor (#3911).
 //
-// When an acoustic group absorbs an exact group, the result is tier acoustic:
-// a group is only as certain as its weakest link.
+// When an acoustic group absorbs an exact group, the result is tier acoustic: a
+// group is only as certain as its weakest link.
+type streamGrouper struct {
+	maxBitErrorRate float64
+	window          []*groupUnit
+	res             groupingResult
+}
+
+func newStreamGrouper(maxBitErrorRate float64) *streamGrouper {
+	return &streamGrouper{maxBitErrorRate: maxBitErrorRate}
+}
+
+// push adds the next unit. Units must arrive in non-decreasing
+// (durationMs, sortKey) order.
+func (g *streamGrouper) push(u *groupUnit) {
+	g.window = append(g.window, u)
+	for len(g.window) > 1 && u.durationMs-g.window[0].durationMs > durationToleranceMs {
+		g.settleOldest()
+	}
+}
+
+// finish settles every unit still waiting and returns what was found. Groups
+// are in no particular order; callers sort with sortGroups.
+func (g *streamGrouper) finish() groupingResult {
+	for len(g.window) > 0 {
+		g.settleOldest()
+	}
+	return g.res
+}
+
+func (g *streamGrouper) settleOldest() {
+	seed := g.window[0]
+	g.window[0] = nil // release it: the window's backing array outlives the slide
+	g.window = g.window[1:]
+	if seed.assigned {
+		return
+	}
+
+	group := []*groupUnit{seed}
+	worst := 0.0
+	for _, cand := range g.window {
+		if cand.durationMs-seed.durationMs > durationToleranceMs {
+			break
+		}
+		if cand.assigned {
+			continue
+		}
+		joined, worstWithCand := true, worst
+		for _, member := range group {
+			if abs32(cand.durationMs-member.durationMs) > durationToleranceMs {
+				joined = false
+				break
+			}
+			score, ok := comparePrepared(member.print, cand.print)
+			if !ok || score.BitErrorRate > g.maxBitErrorRate {
+				joined = false
+				break
+			}
+			worstWithCand = math.Max(worstWithCand, score.BitErrorRate)
+		}
+		if joined {
+			group = append(group, cand)
+			worst = worstWithCand
+		}
+	}
+
+	if len(group) == 1 {
+		if seed.exact {
+			g.res.Groups = append(g.res.Groups, duplicateGroup{Tier: tierExact, Members: seed.ids})
+		}
+		return
+	}
+	// Count units, not tracks: an absorbed exact group is one piece of acoustic
+	// evidence however many identical files it holds.
+	if len(group) > maxAcousticGroupSize {
+		g.res.OversizeClusters++
+		for _, member := range group {
+			member.assigned = true
+			// The acoustic evidence is untrustworthy; identical bytes are not.
+			// An exact group caught inside an oversize cluster is still proposed.
+			if member.exact {
+				g.res.Groups = append(g.res.Groups, duplicateGroup{Tier: tierExact, Members: member.ids})
+			}
+		}
+		return
+	}
+	var members []string
+	for _, member := range group {
+		member.assigned = true
+		members = append(members, member.ids...)
+	}
+	sort.Strings(members)
+	g.res.Groups = append(g.res.Groups, duplicateGroup{
+		Tier: tierAcoustic, Members: members, WorstBitErrorRate: worst,
+	})
+}
+
+// groupDuplicates proposes duplicate groups among candidates held in memory. It
+// runs the same streamGrouper the sweep uses, so there is one grouping rule.
+//
+// Exact groups come first: tracks sharing an audio stream hash. Each becomes a
+// single unit for the acoustic pass, represented by its member with the lowest
+// (duration, id) that has a chromaprint. That is the member the sweep's
+// duration-ordered stream meets first, which keeps the two identical. An exact
+// group with no chromaprint at all cannot be compared acoustically and stands
+// on its own.
 //
 // The output does not depend on input order.
 func groupDuplicates(cands []fingerprintCandidate, maxBitErrorRate float64) groupingResult {
-	var res groupingResult
-
-	// Exact tier.
 	byHash := map[string][]fingerprintCandidate{}
-	var noHash []fingerprintCandidate
+	var units []*groupUnit
+	var printless []duplicateGroup
 	for _, c := range cands {
-		if len(c.StreamSHA256) == 0 {
-			noHash = append(noHash, c)
+		if len(c.StreamSHA256) > 0 {
+			byHash[string(c.StreamSHA256)] = append(byHash[string(c.StreamSHA256)], c)
 			continue
 		}
-		k := string(c.StreamSHA256)
-		byHash[k] = append(byHash[k], c)
+		if len(c.Chromaprint) > 0 {
+			units = append(units, &groupUnit{
+				ids: []string{c.ID}, durationMs: c.DurationMs, sortKey: c.ID, print: preparePrint(c.Chromaprint),
+			})
+		}
 	}
-
-	// A unit is one exact group, or one track with no exact duplicate.
-	type unit struct {
-		members    []fingerprintCandidate
-		durationMs int32
-		print      []int32
-		exact      bool
-	}
-	var units []unit
 	for _, group := range byHash {
-		sortCandidates(group)
-		u := unit{members: group, durationMs: group[0].DurationMs, exact: len(group) > 1}
-		for _, m := range group {
-			if len(m.Chromaprint) > 0 {
-				u.print = m.Chromaprint
-				break
+		ids := make([]string, len(group))
+		for i, m := range group {
+			ids[i] = m.ID
+		}
+		sort.Strings(ids)
+
+		var rep *fingerprintCandidate
+		for i := range group {
+			m := &group[i]
+			if len(m.Chromaprint) == 0 {
+				continue
+			}
+			if rep == nil || m.DurationMs < rep.DurationMs || (m.DurationMs == rep.DurationMs && m.ID < rep.ID) {
+				rep = m
 			}
 		}
-		units = append(units, u)
-	}
-	for _, c := range noHash {
-		units = append(units, unit{members: []fingerprintCandidate{c}, durationMs: c.DurationMs, print: c.Chromaprint})
+		if rep == nil {
+			if len(group) > 1 {
+				printless = append(printless, duplicateGroup{Tier: tierExact, Members: ids})
+			}
+			continue
+		}
+		units = append(units, &groupUnit{
+			ids: ids, durationMs: rep.DurationMs, sortKey: rep.ID,
+			print: preparePrint(rep.Chromaprint), exact: len(group) > 1,
+		})
 	}
 
-	// Deterministic order: duration, then the first member's ID. Sorting by
-	// duration also lets the scan below stop as soon as durations are too far
-	// apart, which is the blocking #3910 relies on.
 	sort.Slice(units, func(i, j int) bool {
 		if units[i].durationMs != units[j].durationMs {
 			return units[i].durationMs < units[j].durationMs
 		}
-		return units[i].members[0].ID < units[j].members[0].ID
+		return units[i].sortKey < units[j].sortKey
 	})
-
-	assigned := make([]bool, len(units))
-	for i := range units {
-		if assigned[i] || len(units[i].print) == 0 {
-			continue
-		}
-		group := []int{i}
-		worst := 0.0
-		for j := i + 1; j < len(units); j++ {
-			if units[j].durationMs-units[i].durationMs > durationToleranceMs {
-				break
-			}
-			if assigned[j] || len(units[j].print) == 0 {
-				continue
-			}
-			// Complete linkage: j must match every member so far.
-			joined, worstWithJ := true, worst
-			for _, g := range group {
-				if abs32(units[j].durationMs-units[g].durationMs) > durationToleranceMs {
-					joined = false
-					break
-				}
-				score, ok := compareChromaprint(units[g].print, units[j].print)
-				if !ok || score.BitErrorRate > maxBitErrorRate {
-					joined = false
-					break
-				}
-				worstWithJ = math.Max(worstWithJ, score.BitErrorRate)
-			}
-			if joined {
-				group = append(group, j)
-				worst = worstWithJ
-			}
-		}
-
-		if len(group) == 1 {
-			continue
-		}
-		// Count units, not tracks: an absorbed exact group is one piece of
-		// acoustic evidence however many identical files it holds.
-		if len(group) > maxAcousticGroupSize {
-			res.OversizeClusters++
-			for _, g := range group {
-				assigned[g] = true
-			}
-			continue
-		}
-		var members []string
-		for _, g := range group {
-			assigned[g] = true
-			for _, m := range units[g].members {
-				members = append(members, m.ID)
-			}
-		}
-		sort.Strings(members)
-		res.Groups = append(res.Groups, duplicateGroup{
-			Tier: tierAcoustic, Members: members, WorstBitErrorRate: worst,
-		})
+	g := newStreamGrouper(maxBitErrorRate)
+	for _, u := range units {
+		g.push(u)
 	}
-
-	// Exact groups that no acoustic group absorbed stand on their own.
-	for i, u := range units {
-		if assigned[i] || !u.exact {
-			continue
-		}
-		members := make([]string, len(u.members))
-		for k, m := range u.members {
-			members[k] = m.ID
-		}
-		res.Groups = append(res.Groups, duplicateGroup{Tier: tierExact, Members: members})
-	}
-
-	sort.Slice(res.Groups, func(i, j int) bool {
-		return res.Groups[i].Members[0] < res.Groups[j].Members[0]
-	})
+	res := g.finish()
+	res.Groups = append(res.Groups, printless...)
+	sortGroups(res.Groups)
 	return res
 }
 
-func sortCandidates(cs []fingerprintCandidate) {
-	sort.Slice(cs, func(i, j int) bool { return cs[i].ID < cs[j].ID })
+// sortGroups orders groups by their first member. Groups are disjoint, so that
+// is a total order.
+func sortGroups(groups []duplicateGroup) {
+	sort.Slice(groups, func(i, j int) bool { return groups[i].Members[0] < groups[j].Members[0] })
 }
 
 func abs(n int) int {