feat(library): the duplicate sweep — propose duplicate groups from fingerprints (M400 #3910)

Reads fingerprints, runs them through the matcher, and records
proposals in duplicate_groups (migration 0059). Nothing is merged or
deleted: a group is a proposal for the admin report (#3912).

Streaming. The whole library's fingerprints are hundreds of megabytes,
but tracks are only compared within 3s of each other in duration. So
candidates stream in (duration_ms, id) order, keyset-paged on a new
tracks(duration_ms, id) index. The grouper holds only the tracks within
3s of the oldest one not yet settled. A seed is settled once a track
arrives beyond its window, which gives the same result as grouping the
whole sorted list. groupDuplicates is rebuilt on the same streamGrouper,
so there is one grouping rule and the #3909 tests still cover it. Each
fingerprint's alignment index and variety check are computed once
instead of for every pair.

Exact duplicates are grouped library-wide in SQL. The first member the
stream meets stands in for the whole group in the acoustic pass. An
exact group caught in an oversize acoustic cluster is still proposed:
the acoustic evidence is discarded, identical bytes are not.

Re-sweeping:
- a group is identified by its sorted member ids, so finding it again
  refreshes the row in place
- a proposal whose members all sat in one dismissed group is not
  proposed again (a subset repeats the verdict; a superset is new
  evidence)
- a pending proposal no sweep has found again is retired, but only
  after a complete sweep, and only if an earlier sweep last confirmed
  it, so two overlapping sweeps cannot delete each other's findings
- dismissals are kept

DuplicateSweepWorker checks hourly and sweeps only when a fingerprint
was written after the last sweep started. TryStartDuplicateSweep guards
against two sweeps at once and reaps one stuck in flight for 2h. The
sweep row is closed on a detached context with a deadline, so a sweep
cancelled at shutdown still records that it ended.

The integration test pages one row at a time and checks:
- an acoustic pair and an exact pair are found
- a track with no fingerprint, a missing track and a near-duration
  unrelated song are left out
- a dismissed group is suppressed while the pending one refreshes
  without duplicating
- a proposal that stops holding is retired and the dismissal survives

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01SQ31KQpYbStyK5y58UmPLH

internal/library/duplicate_sweep_test.go

@@ -0,0 +1,173 @@
+package library
+
+import (
+	"bytes"
+	"context"
+	"io"
+	"log/slog"
+	"path/filepath"
+	"sort"
+	"strings"
+	"testing"
+
+	"git.fabledsword.com/bvandeusen/minstrel/internal/db/dbq"
+	syncpkg "git.fabledsword.com/bvandeusen/minstrel/internal/sync"
+)
+
+func TestCoveredByDismissal(t *testing.T) {
+	dismissed := []map[string]struct{}{{"a": {}, "b": {}, "c": {}}}
+	for _, tc := range []struct {
+		name    string
+		members []string
+		want    bool
+	}{
+		{"the same set", []string{"a", "b", "c"}, true},
+		{"a subset of it", []string{"a", "b"}, true},
+		// A new copy joining is new evidence: ask again.
+		{"a superset of it", []string{"a", "b", "c", "d"}, false},
+		{"overlapping only in part", []string{"a", "d"}, false},
+		{"unrelated", []string{"x", "y"}, false},
+	} {
+		if got := coveredByDismissal(tc.members, dismissed); got != tc.want {
+			t.Errorf("%s: coveredByDismissal = %v, want %v", tc.name, got, tc.want)
+		}
+	}
+}
+
+// TestDuplicateSweep_Integration pins what the sweep proposes, what it leaves out,
+// and how re-sweeping treats a dismissal and a proposal that no longer holds.
+func TestDuplicateSweep_Integration(t *testing.T) {
+	pool := newPool(t)
+	ctx := context.Background()
+	q := dbq.New(pool)
+	dir := t.TempDir()
+	logger := slog.New(slog.NewTextHandler(io.Discard, nil))
+
+	// seedTrack's own track has no fingerprint row: it must be absent from the
+	// report, not grouped with every other track lacking one.
+	_, album, artist := seedTrack(t, pool, filepath.Join(dir, "unfingerprinted.mp3"))
+	hash := func(b byte) []byte { return bytes.Repeat([]byte{b}, 32) }
+	add := func(name string, durationMs int32, sum []byte, print []int32) string {
+		t.Helper()
+		tr, err := q.UpsertTrack(ctx, dbq.UpsertTrackParams{
+			Title: name, AlbumID: album.ID, ArtistID: artist.ID, DurationMs: durationMs,
+			FilePath: filepath.Join(dir, name+".mp3"), FileSize: 100, FileFormat: "mp3",
+		})
+		if err != nil {
+			t.Fatalf("track %s: %v", name, err)
+		}
+		if err := q.UpsertTrackFingerprint(ctx, dbq.UpsertTrackFingerprintParams{
+			TrackID: tr.ID, AudioStreamSha256: sum, Chromaprint: print, FingerprintVersion: fingerprintVersion,
+		}); err != nil {
+			t.Fatalf("fingerprint %s: %v", name, err)
+		}
+		return syncpkg.FormatUUID(tr.ID)
+	}
+	key := func(ids ...string) string {
+		sorted := append([]string(nil), ids...)
+		sort.Strings(sorted)
+		return strings.Join(sorted, ",")
+	}
+
+	recording := randomPrint(200, printLen)
+	onAlbum := add("recording-album", 240000, hash(1), recording)
+	onCompilation := add("recording-compilation", 241000, hash(2), withBitNoise(recording, 0.05, 201))
+	www1 := add("www-01", 215000, hash(9), randomPrint(210, printLen))
+	www2 := add("www-02", 215000, hash(9), randomPrint(210, printLen))
+	// Near-identical duration to the recording, different audio.
+	add("different-song", 240500, hash(3), randomPrint(220, printLen))
+	// Identical to the album copy, but its file is gone: nothing to compare.
+	missing := add("missing-copy", 240000, hash(4), recording)
+	if _, err := pool.Exec(ctx, "UPDATE tracks SET missing_since = now() WHERE file_path LIKE '%missing-copy.mp3'"); err != nil {
+		t.Fatalf("mark missing: %v", err)
+	}
+
+	type stored struct {
+		tier, status string
+	}
+	groups := func() map[string]stored {
+		t.Helper()
+		rows, err := pool.Query(ctx, `SELECT member_key, tier, status FROM duplicate_groups`)
+		if err != nil {
+			t.Fatalf("read groups: %v", err)
+		}
+		defer rows.Close()
+		out := map[string]stored{}
+		for rows.Next() {
+			var k string
+			var s stored
+			if err := rows.Scan(&k, &s.tier, &s.status); err != nil {
+				t.Fatalf("scan group: %v", err)
+			}
+			out[k] = s
+		}
+		return out
+	}
+
+	// 1. A page size of one forces the keyset cursor across every candidate.
+	res, err := runDuplicateSweep(ctx, pool, logger, 1)
+	if err != nil {
+		t.Fatalf("first sweep: %v", err)
+	}
+	// Five tracks carry a chromaprint and a present file.
+	if res.Candidates != 5 || res.Groups != 2 || res.Proposed != 2 {
+		t.Fatalf("first sweep = %+v, want 5 candidates, 2 groups, 2 proposed", res)
+	}
+	acousticKey, exactKey := key(onAlbum, onCompilation), key(www1, www2)
+	got := groups()
+	want := map[string]stored{
+		acousticKey: {"acoustic", "pending"},
+		exactKey:    {"exact", "pending"},
+	}
+	if len(got) != len(want) || got[acousticKey] != want[acousticKey] || got[exactKey] != want[exactKey] {
+		t.Fatalf("groups = %+v, want %+v", got, want)
+	}
+	for k := range got {
+		if strings.Contains(k, missing) {
+			t.Fatalf("a missing track was proposed: %s", k)
+		}
+	}
+
+	// 2. A dismissed group is not proposed again, and the pending one is
+	// refreshed in place rather than duplicated.
+	if _, err := pool.Exec(ctx, "UPDATE duplicate_groups SET status = 'dismissed' WHERE member_key = $1", acousticKey); err != nil {
+		t.Fatalf("dismiss: %v", err)
+	}
+	res, err = runDuplicateSweep(ctx, pool, logger, duplicateCandidatePage)
+	if err != nil {
+		t.Fatalf("second sweep: %v", err)
+	}
+	if res.Proposed != 1 || res.Suppressed != 1 {
+		t.Fatalf("second sweep = %+v, want 1 proposed, 1 suppressed", res)
+	}
+	got = groups()
+	if len(got) != 2 || got[acousticKey].status != "dismissed" || got[exactKey].status != "pending" {
+		t.Fatalf("after dismissal groups = %+v, want the dismissal kept and one pending group", got)
+	}
+
+	// 3. A proposal that no longer holds is retired; the dismissal survives it.
+	if _, err := pool.Exec(ctx,
+		"DELETE FROM track_fingerprints f USING tracks t WHERE f.track_id = t.id AND t.file_path LIKE '%www-02.mp3'"); err != nil {
+		t.Fatalf("drop fingerprint: %v", err)
+	}
+	res, err = runDuplicateSweep(ctx, pool, logger, duplicateCandidatePage)
+	if err != nil {
+		t.Fatalf("third sweep: %v", err)
+	}
+	if res.Retired != 1 {
+		t.Fatalf("third sweep = %+v, want 1 retired", res)
+	}
+	got = groups()
+	if len(got) != 1 || got[acousticKey].status != "dismissed" {
+		t.Fatalf("after retiring groups = %+v, want only the dismissal", got)
+	}
+
+	// 4. The sweep record reflects the last run.
+	last, err := q.GetLatestDuplicateSweep(ctx)
+	if err != nil {
+		t.Fatalf("latest sweep: %v", err)
+	}
+	if !last.FinishedAt.Valid || last.ErrorMessage != nil {
+		t.Fatalf("latest sweep = %+v, want finished without error", last)
+	}
+}