feat(library): the duplicate sweep — propose duplicate groups from fingerprints (M400 #3910)

Reads fingerprints, runs them through the matcher, and records
proposals in duplicate_groups (migration 0059). Nothing is merged or
deleted: a group is a proposal for the admin report (#3912).

Streaming. The whole library's fingerprints are hundreds of megabytes,
but tracks are only compared within 3s of each other in duration. So
candidates stream in (duration_ms, id) order, keyset-paged on a new
tracks(duration_ms, id) index. The grouper holds only the tracks within
3s of the oldest one not yet settled. A seed is settled once a track
arrives beyond its window, which gives the same result as grouping the
whole sorted list. groupDuplicates is rebuilt on the same streamGrouper,
so there is one grouping rule and the #3909 tests still cover it. Each
fingerprint's alignment index and variety check are computed once
instead of for every pair.

Exact duplicates are grouped library-wide in SQL. The first member the
stream meets stands in for the whole group in the acoustic pass. An
exact group caught in an oversize acoustic cluster is still proposed:
the acoustic evidence is discarded, identical bytes are not.

Re-sweeping:
- a group is identified by its sorted member ids, so finding it again
  refreshes the row in place
- a proposal whose members all sat in one dismissed group is not
  proposed again (a subset repeats the verdict; a superset is new
  evidence)
- a pending proposal no sweep has found again is retired, but only
  after a complete sweep, and only if an earlier sweep last confirmed
  it, so two overlapping sweeps cannot delete each other's findings
- dismissals are kept

DuplicateSweepWorker checks hourly and sweeps only when a fingerprint
was written after the last sweep started. TryStartDuplicateSweep guards
against two sweeps at once and reaps one stuck in flight for 2h. The
sweep row is closed on a detached context with a deadline, so a sweep
cancelled at shutdown still records that it ended.

The integration test pages one row at a time and checks:
- an acoustic pair and an exact pair are found
- a track with no fingerprint, a missing track and a near-duration
  unrelated song are left out
- a dismissed group is suppressed while the pending one refreshes
  without duplicating
- a proposal that stops holding is retired and the dismissal survives

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01SQ31KQpYbStyK5y58UmPLH

internal/db/dbq/duplicates.sql.go

@@ -0,0 +1,320 @@
+// Code generated by sqlc. DO NOT EDIT.
+// versions:
+//   sqlc v1.31.1
+// source: duplicates.sql
+
+package dbq
+
+import (
+	"context"
+
+	"github.com/jackc/pgx/v5/pgtype"
+)
+
+const addDuplicateGroupMember = `-- name: AddDuplicateGroupMember :exec
+INSERT INTO duplicate_group_members (group_id, track_id)
+VALUES ($1, $2)
+ON CONFLICT DO NOTHING
+`
+
+type AddDuplicateGroupMemberParams struct {
+	GroupID pgtype.UUID
+	TrackID pgtype.UUID
+}
+
+func (q *Queries) AddDuplicateGroupMember(ctx context.Context, arg AddDuplicateGroupMemberParams) error {
+	_, err := q.db.Exec(ctx, addDuplicateGroupMember, arg.GroupID, arg.TrackID)
+	return err
+}
+
+const deleteStalePendingDuplicateGroups = `-- name: DeleteStalePendingDuplicateGroups :execrows
+DELETE FROM duplicate_groups g
+ WHERE g.status = 'pending'
+   AND g.last_seen_sweep_id IS DISTINCT FROM $1
+   AND (g.last_seen_sweep_id IS NULL
+        OR (SELECT s.started_at FROM duplicate_sweeps s WHERE s.id = g.last_seen_sweep_id)
+           < (SELECT s.started_at FROM duplicate_sweeps s WHERE s.id = $1))
+`
+
+// A pending proposal this sweep did not find again no longer describes the
+// library: a member was re-fingerprinted, merged away or went missing. Dismissed
+// groups are kept regardless — they are the memory of a decision.
+//
+// Only proposals last confirmed by an EARLIER sweep go. Should two sweeps ever
+// overlap (a manual trigger racing the worker), neither may delete what the other
+// has just found.
+func (q *Queries) DeleteStalePendingDuplicateGroups(ctx context.Context, sweepID pgtype.UUID) (int64, error) {
+	result, err := q.db.Exec(ctx, deleteStalePendingDuplicateGroups, sweepID)
+	if err != nil {
+		return 0, err
+	}
+	return result.RowsAffected(), nil
+}
+
+const finishDuplicateSweep = `-- name: FinishDuplicateSweep :exec
+UPDATE duplicate_sweeps
+   SET finished_at       = now(),
+       candidates        = $1,
+       groups_found      = $2,
+       oversize_clusters = $3,
+       error_message     = NULLIF($4::text, '')
+ WHERE id = $5
+`
+
+type FinishDuplicateSweepParams struct {
+	Candidates       *int32
+	GroupsFound      *int32
+	OversizeClusters *int32
+	ErrorMessage     string
+	ID               pgtype.UUID
+}
+
+func (q *Queries) FinishDuplicateSweep(ctx context.Context, arg FinishDuplicateSweepParams) error {
+	_, err := q.db.Exec(ctx, finishDuplicateSweep,
+		arg.Candidates,
+		arg.GroupsFound,
+		arg.OversizeClusters,
+		arg.ErrorMessage,
+		arg.ID,
+	)
+	return err
+}
+
+const getInFlightDuplicateSweep = `-- name: GetInFlightDuplicateSweep :one
+SELECT id, started_at
+  FROM duplicate_sweeps
+ WHERE finished_at IS NULL
+ ORDER BY started_at DESC
+ LIMIT 1
+`
+
+type GetInFlightDuplicateSweepRow struct {
+	ID        pgtype.UUID
+	StartedAt pgtype.Timestamptz
+}
+
+// The guard against two sweeps at once: "in flight" is finished_at IS NULL.
+func (q *Queries) GetInFlightDuplicateSweep(ctx context.Context) (GetInFlightDuplicateSweepRow, error) {
+	row := q.db.QueryRow(ctx, getInFlightDuplicateSweep)
+	var i GetInFlightDuplicateSweepRow
+	err := row.Scan(&i.ID, &i.StartedAt)
+	return i, err
+}
+
+const getLatestDuplicateSweep = `-- name: GetLatestDuplicateSweep :one
+SELECT id, started_at, finished_at, candidates, groups_found, oversize_clusters, error_message
+  FROM duplicate_sweeps
+ ORDER BY started_at DESC
+ LIMIT 1
+`
+
+func (q *Queries) GetLatestDuplicateSweep(ctx context.Context) (DuplicateSweep, error) {
+	row := q.db.QueryRow(ctx, getLatestDuplicateSweep)
+	var i DuplicateSweep
+	err := row.Scan(
+		&i.ID,
+		&i.StartedAt,
+		&i.FinishedAt,
+		&i.Candidates,
+		&i.GroupsFound,
+		&i.OversizeClusters,
+		&i.ErrorMessage,
+	)
+	return i, err
+}
+
+const getLatestFingerprintComputedAt = `-- name: GetLatestFingerprintComputedAt :one
+SELECT max(computed_at)::timestamptz AS latest FROM track_fingerprints
+`
+
+// Whether a sweep has anything new to look at: fingerprints written since the
+// last sweep started.
+func (q *Queries) GetLatestFingerprintComputedAt(ctx context.Context) (pgtype.Timestamptz, error) {
+	row := q.db.QueryRow(ctx, getLatestFingerprintComputedAt)
+	var latest pgtype.Timestamptz
+	err := row.Scan(&latest)
+	return latest, err
+}
+
+const listDismissedDuplicateMemberSets = `-- name: ListDismissedDuplicateMemberSets :many
+SELECT g.id, array_agg(m.track_id ORDER BY m.track_id)::uuid[] AS track_ids
+  FROM duplicate_groups g
+  JOIN duplicate_group_members m ON m.group_id = g.id
+ WHERE g.status = 'dismissed'
+ GROUP BY g.id
+`
+
+type ListDismissedDuplicateMemberSetsRow struct {
+	ID       pgtype.UUID
+	TrackIds []pgtype.UUID
+}
+
+// What the operator has already said are not duplicates. A new proposal whose
+// every member sat together in one of these is not proposed again.
+func (q *Queries) ListDismissedDuplicateMemberSets(ctx context.Context) ([]ListDismissedDuplicateMemberSetsRow, error) {
+	rows, err := q.db.Query(ctx, listDismissedDuplicateMemberSets)
+	if err != nil {
+		return nil, err
+	}
+	defer rows.Close()
+	var items []ListDismissedDuplicateMemberSetsRow
+	for rows.Next() {
+		var i ListDismissedDuplicateMemberSetsRow
+		if err := rows.Scan(&i.ID, &i.TrackIds); err != nil {
+			return nil, err
+		}
+		items = append(items, i)
+	}
+	if err := rows.Err(); err != nil {
+		return nil, err
+	}
+	return items, nil
+}
+
+const listDuplicateCandidates = `-- name: ListDuplicateCandidates :many
+SELECT t.id, t.duration_ms, f.audio_stream_sha256, f.chromaprint
+  FROM tracks t
+  JOIN track_fingerprints f ON f.track_id = t.id
+ WHERE t.missing_since IS NULL
+   AND f.fingerprint_version >= $1
+   AND f.chromaprint IS NOT NULL
+   AND (t.duration_ms, t.id) > ($2::integer, $3::uuid)
+ ORDER BY t.duration_ms, t.id
+ LIMIT $4
+`
+
+type ListDuplicateCandidatesParams struct {
+	CurrentVersion  int16
+	AfterDurationMs int32
+	AfterID         pgtype.UUID
+	PageLimit       int32
+}
+
+type ListDuplicateCandidatesRow struct {
+	ID                pgtype.UUID
+	DurationMs        int32
+	AudioStreamSha256 []byte
+	Chromaprint       []int32
+}
+
+// The acoustic tier's input, one page at a time in (duration_ms, id) order so the
+// sweep holds only a sliding window of durations. Tracks without a chromaprint
+// cannot be compared acoustically and are left out; any exact duplicates among
+// them come from ListExactDuplicateHashes.
+func (q *Queries) ListDuplicateCandidates(ctx context.Context, arg ListDuplicateCandidatesParams) ([]ListDuplicateCandidatesRow, error) {
+	rows, err := q.db.Query(ctx, listDuplicateCandidates,
+		arg.CurrentVersion,
+		arg.AfterDurationMs,
+		arg.AfterID,
+		arg.PageLimit,
+	)
+	if err != nil {
+		return nil, err
+	}
+	defer rows.Close()
+	var items []ListDuplicateCandidatesRow
+	for rows.Next() {
+		var i ListDuplicateCandidatesRow
+		if err := rows.Scan(
+			&i.ID,
+			&i.DurationMs,
+			&i.AudioStreamSha256,
+			&i.Chromaprint,
+		); err != nil {
+			return nil, err
+		}
+		items = append(items, i)
+	}
+	if err := rows.Err(); err != nil {
+		return nil, err
+	}
+	return items, nil
+}
+
+const listExactDuplicateHashes = `-- name: ListExactDuplicateHashes :many
+SELECT f.audio_stream_sha256,
+       array_agg(t.id ORDER BY t.id)::uuid[] AS track_ids
+  FROM track_fingerprints f
+  JOIN tracks t ON t.id = f.track_id
+ WHERE t.missing_since IS NULL
+   AND f.fingerprint_version >= $1
+   AND f.audio_stream_sha256 IS NOT NULL
+ GROUP BY f.audio_stream_sha256
+HAVING count(*) > 1
+`
+
+type ListExactDuplicateHashesRow struct {
+	AudioStreamSha256 []byte
+	TrackIds          []pgtype.UUID
+}
+
+// The exact tier, library-wide in one pass: identical encoded audio shared by
+// more than one present track.
+func (q *Queries) ListExactDuplicateHashes(ctx context.Context, currentVersion int16) ([]ListExactDuplicateHashesRow, error) {
+	rows, err := q.db.Query(ctx, listExactDuplicateHashes, currentVersion)
+	if err != nil {
+		return nil, err
+	}
+	defer rows.Close()
+	var items []ListExactDuplicateHashesRow
+	for rows.Next() {
+		var i ListExactDuplicateHashesRow
+		if err := rows.Scan(&i.AudioStreamSha256, &i.TrackIds); err != nil {
+			return nil, err
+		}
+		items = append(items, i)
+	}
+	if err := rows.Err(); err != nil {
+		return nil, err
+	}
+	return items, nil
+}
+
+const startDuplicateSweep = `-- name: StartDuplicateSweep :one
+INSERT INTO duplicate_sweeps DEFAULT VALUES RETURNING id, started_at
+`
+
+type StartDuplicateSweepRow struct {
+	ID        pgtype.UUID
+	StartedAt pgtype.Timestamptz
+}
+
+func (q *Queries) StartDuplicateSweep(ctx context.Context) (StartDuplicateSweepRow, error) {
+	row := q.db.QueryRow(ctx, startDuplicateSweep)
+	var i StartDuplicateSweepRow
+	err := row.Scan(&i.ID, &i.StartedAt)
+	return i, err
+}
+
+const upsertDuplicateGroup = `-- name: UpsertDuplicateGroup :one
+INSERT INTO duplicate_groups (member_key, tier, worst_bit_error_rate, last_seen_sweep_id)
+VALUES ($1, $2, $3, $4)
+ON CONFLICT (member_key) DO UPDATE
+   SET tier                 = EXCLUDED.tier,
+       worst_bit_error_rate = EXCLUDED.worst_bit_error_rate,
+       last_seen_sweep_id   = EXCLUDED.last_seen_sweep_id
+ WHERE duplicate_groups.status = 'pending'
+RETURNING id
+`
+
+type UpsertDuplicateGroupParams struct {
+	MemberKey         string
+	Tier              string
+	WorstBitErrorRate *float32
+	SweepID           pgtype.UUID
+}
+
+// Proposes a group, or refreshes one already pending. A group already dismissed
+// or merged is left exactly as it is: the WHERE on the update makes the conflict
+// a no-op, and the caller sees no row.
+func (q *Queries) UpsertDuplicateGroup(ctx context.Context, arg UpsertDuplicateGroupParams) (pgtype.UUID, error) {
+	row := q.db.QueryRow(ctx, upsertDuplicateGroup,
+		arg.MemberKey,
+		arg.Tier,
+		arg.WorstBitErrorRate,
+		arg.SweepID,
+	)
+	var id pgtype.UUID
+	err := row.Scan(&id)
+	return id, err
+}