feat(library): the duplicate sweep — propose duplicate groups from fingerprints (M400 #3910)

Reads fingerprints, runs them through the matcher, and records proposals in duplicate_groups (migration 0059). Nothing is merged or deleted: a group is a proposal for the admin report (#3912). Streaming. The whole library's fingerprints are hundreds of megabytes, but tracks are only compared within 3s of each other in duration. So candidates stream in (duration_ms, id) order, keyset-paged on a new tracks(duration_ms, id) index. The grouper holds only the tracks within 3s of the oldest one not yet settled. A seed is settled once a track arrives beyond its window, which gives the same result as grouping the whole sorted list. groupDuplicates is rebuilt on the same streamGrouper, so there is one grouping rule and the #3909 tests still cover it. Each fingerprint's alignment index and variety check are computed once instead of for every pair. Exact duplicates are grouped library-wide in SQL. The first member the stream meets stands in for the whole group in the acoustic pass. An exact group caught in an oversize acoustic cluster is still proposed: the acoustic evidence is discarded, identical bytes are not. Re-sweeping: - a group is identified by its sorted member ids, so finding it again refreshes the row in place - a proposal whose members all sat in one dismissed group is not proposed again (a subset repeats the verdict; a superset is new evidence) - a pending proposal no sweep has found again is retired, but only after a complete sweep, and only if an earlier sweep last confirmed it, so two overlapping sweeps cannot delete each other's findings - dismissals are kept DuplicateSweepWorker checks hourly and sweeps only when a fingerprint was written after the last sweep started. TryStartDuplicateSweep guards against two sweeps at once and reaps one stuck in flight for 2h. The sweep row is closed on a detached context with a deadline, so a sweep cancelled at shutdown still records that it ended. The integration test pages one row at a time and checks: - an acoustic pair and an exact pair are found - a track with no fingerprint, a missing track and a near-duration unrelated song are left out - a dismissed group is suppressed while the pending one refreshes without duplicating - a proposal that stops holding is retired and the dismissal survives Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01SQ31KQpYbStyK5y58UmPLH
2026-09-11 17:00:07 -04:00
parent c06af48cd6
commit 6379b6c31d
10 changed files with 1264 additions and 146 deletions
@@ -0,0 +1,320 @@
+// Code generated by sqlc. DO NOT EDIT.
+// versions:
+//   sqlc v1.31.1
+// source: duplicates.sql
+
+package dbq
+
+import (
+	"context"
+
+	"github.com/jackc/pgx/v5/pgtype"
+)
+
+const addDuplicateGroupMember = `-- name: AddDuplicateGroupMember :exec
+INSERT INTO duplicate_group_members (group_id, track_id)
+VALUES ($1, $2)
+ON CONFLICT DO NOTHING
+`
+
+type AddDuplicateGroupMemberParams struct {
+	GroupID pgtype.UUID
+	TrackID pgtype.UUID
+}
+
+func (q *Queries) AddDuplicateGroupMember(ctx context.Context, arg AddDuplicateGroupMemberParams) error {
+	_, err := q.db.Exec(ctx, addDuplicateGroupMember, arg.GroupID, arg.TrackID)
+	return err
+}
+
+const deleteStalePendingDuplicateGroups = `-- name: DeleteStalePendingDuplicateGroups :execrows
+DELETE FROM duplicate_groups g
+ WHERE g.status = 'pending'
+   AND g.last_seen_sweep_id IS DISTINCT FROM $1
+   AND (g.last_seen_sweep_id IS NULL
+        OR (SELECT s.started_at FROM duplicate_sweeps s WHERE s.id = g.last_seen_sweep_id)
+           < (SELECT s.started_at FROM duplicate_sweeps s WHERE s.id = $1))
+`
+
+// A pending proposal this sweep did not find again no longer describes the
+// library: a member was re-fingerprinted, merged away or went missing. Dismissed
+// groups are kept regardless — they are the memory of a decision.
+//
+// Only proposals last confirmed by an EARLIER sweep go. Should two sweeps ever
+// overlap (a manual trigger racing the worker), neither may delete what the other
+// has just found.
+func (q *Queries) DeleteStalePendingDuplicateGroups(ctx context.Context, sweepID pgtype.UUID) (int64, error) {
+	result, err := q.db.Exec(ctx, deleteStalePendingDuplicateGroups, sweepID)
+	if err != nil {
+		return 0, err
+	}
+	return result.RowsAffected(), nil
+}
+
+const finishDuplicateSweep = `-- name: FinishDuplicateSweep :exec
+UPDATE duplicate_sweeps
+   SET finished_at       = now(),
+       candidates        = $1,
+       groups_found      = $2,
+       oversize_clusters = $3,
+       error_message     = NULLIF($4::text, '')
+ WHERE id = $5
+`
+
+type FinishDuplicateSweepParams struct {
+	Candidates       *int32
+	GroupsFound      *int32
+	OversizeClusters *int32
+	ErrorMessage     string
+	ID               pgtype.UUID
+}
+
+func (q *Queries) FinishDuplicateSweep(ctx context.Context, arg FinishDuplicateSweepParams) error {
+	_, err := q.db.Exec(ctx, finishDuplicateSweep,
+		arg.Candidates,
+		arg.GroupsFound,
+		arg.OversizeClusters,
+		arg.ErrorMessage,
+		arg.ID,
+	)
+	return err
+}
+
+const getInFlightDuplicateSweep = `-- name: GetInFlightDuplicateSweep :one
+SELECT id, started_at
+  FROM duplicate_sweeps
+ WHERE finished_at IS NULL
+ ORDER BY started_at DESC
+ LIMIT 1
+`
+
+type GetInFlightDuplicateSweepRow struct {
+	ID        pgtype.UUID
+	StartedAt pgtype.Timestamptz
+}
+
+// The guard against two sweeps at once: "in flight" is finished_at IS NULL.
+func (q *Queries) GetInFlightDuplicateSweep(ctx context.Context) (GetInFlightDuplicateSweepRow, error) {
+	row := q.db.QueryRow(ctx, getInFlightDuplicateSweep)
+	var i GetInFlightDuplicateSweepRow
+	err := row.Scan(&i.ID, &i.StartedAt)
+	return i, err
+}
+
+const getLatestDuplicateSweep = `-- name: GetLatestDuplicateSweep :one
+SELECT id, started_at, finished_at, candidates, groups_found, oversize_clusters, error_message
+  FROM duplicate_sweeps
+ ORDER BY started_at DESC
+ LIMIT 1
+`
+
+func (q *Queries) GetLatestDuplicateSweep(ctx context.Context) (DuplicateSweep, error) {
+	row := q.db.QueryRow(ctx, getLatestDuplicateSweep)
+	var i DuplicateSweep
+	err := row.Scan(
+		&i.ID,
+		&i.StartedAt,
+		&i.FinishedAt,
+		&i.Candidates,
+		&i.GroupsFound,
+		&i.OversizeClusters,
+		&i.ErrorMessage,
+	)
+	return i, err
+}
+
+const getLatestFingerprintComputedAt = `-- name: GetLatestFingerprintComputedAt :one
+SELECT max(computed_at)::timestamptz AS latest FROM track_fingerprints
+`
+
+// Whether a sweep has anything new to look at: fingerprints written since the
+// last sweep started.
+func (q *Queries) GetLatestFingerprintComputedAt(ctx context.Context) (pgtype.Timestamptz, error) {
+	row := q.db.QueryRow(ctx, getLatestFingerprintComputedAt)
+	var latest pgtype.Timestamptz
+	err := row.Scan(&latest)
+	return latest, err
+}
+
+const listDismissedDuplicateMemberSets = `-- name: ListDismissedDuplicateMemberSets :many
+SELECT g.id, array_agg(m.track_id ORDER BY m.track_id)::uuid[] AS track_ids
+  FROM duplicate_groups g
+  JOIN duplicate_group_members m ON m.group_id = g.id
+ WHERE g.status = 'dismissed'
+ GROUP BY g.id
+`
+
+type ListDismissedDuplicateMemberSetsRow struct {
+	ID       pgtype.UUID
+	TrackIds []pgtype.UUID
+}
+
+// What the operator has already said are not duplicates. A new proposal whose
+// every member sat together in one of these is not proposed again.
+func (q *Queries) ListDismissedDuplicateMemberSets(ctx context.Context) ([]ListDismissedDuplicateMemberSetsRow, error) {
+	rows, err := q.db.Query(ctx, listDismissedDuplicateMemberSets)
+	if err != nil {
+		return nil, err
+	}
+	defer rows.Close()
+	var items []ListDismissedDuplicateMemberSetsRow
+	for rows.Next() {
+		var i ListDismissedDuplicateMemberSetsRow
+		if err := rows.Scan(&i.ID, &i.TrackIds); err != nil {
+			return nil, err
+		}
+		items = append(items, i)
+	}
+	if err := rows.Err(); err != nil {
+		return nil, err
+	}
+	return items, nil
+}
+
+const listDuplicateCandidates = `-- name: ListDuplicateCandidates :many
+SELECT t.id, t.duration_ms, f.audio_stream_sha256, f.chromaprint
+  FROM tracks t
+  JOIN track_fingerprints f ON f.track_id = t.id
+ WHERE t.missing_since IS NULL
+   AND f.fingerprint_version >= $1
+   AND f.chromaprint IS NOT NULL
+   AND (t.duration_ms, t.id) > ($2::integer, $3::uuid)
+ ORDER BY t.duration_ms, t.id
+ LIMIT $4
+`
+
+type ListDuplicateCandidatesParams struct {
+	CurrentVersion  int16
+	AfterDurationMs int32
+	AfterID         pgtype.UUID
+	PageLimit       int32
+}
+
+type ListDuplicateCandidatesRow struct {
+	ID                pgtype.UUID
+	DurationMs        int32
+	AudioStreamSha256 []byte
+	Chromaprint       []int32
+}
+
+// The acoustic tier's input, one page at a time in (duration_ms, id) order so the
+// sweep holds only a sliding window of durations. Tracks without a chromaprint
+// cannot be compared acoustically and are left out; any exact duplicates among
+// them come from ListExactDuplicateHashes.
+func (q *Queries) ListDuplicateCandidates(ctx context.Context, arg ListDuplicateCandidatesParams) ([]ListDuplicateCandidatesRow, error) {
+	rows, err := q.db.Query(ctx, listDuplicateCandidates,
+		arg.CurrentVersion,
+		arg.AfterDurationMs,
+		arg.AfterID,
+		arg.PageLimit,
+	)
+	if err != nil {
+		return nil, err
+	}
+	defer rows.Close()
+	var items []ListDuplicateCandidatesRow
+	for rows.Next() {
+		var i ListDuplicateCandidatesRow
+		if err := rows.Scan(
+			&i.ID,
+			&i.DurationMs,
+			&i.AudioStreamSha256,
+			&i.Chromaprint,
+		); err != nil {
+			return nil, err
+		}
+		items = append(items, i)
+	}
+	if err := rows.Err(); err != nil {
+		return nil, err
+	}
+	return items, nil
+}
+
+const listExactDuplicateHashes = `-- name: ListExactDuplicateHashes :many
+SELECT f.audio_stream_sha256,
+       array_agg(t.id ORDER BY t.id)::uuid[] AS track_ids
+  FROM track_fingerprints f
+  JOIN tracks t ON t.id = f.track_id
+ WHERE t.missing_since IS NULL
+   AND f.fingerprint_version >= $1
+   AND f.audio_stream_sha256 IS NOT NULL
+ GROUP BY f.audio_stream_sha256
+HAVING count(*) > 1
+`
+
+type ListExactDuplicateHashesRow struct {
+	AudioStreamSha256 []byte
+	TrackIds          []pgtype.UUID
+}
+
+// The exact tier, library-wide in one pass: identical encoded audio shared by
+// more than one present track.
+func (q *Queries) ListExactDuplicateHashes(ctx context.Context, currentVersion int16) ([]ListExactDuplicateHashesRow, error) {
+	rows, err := q.db.Query(ctx, listExactDuplicateHashes, currentVersion)
+	if err != nil {
+		return nil, err
+	}
+	defer rows.Close()
+	var items []ListExactDuplicateHashesRow
+	for rows.Next() {
+		var i ListExactDuplicateHashesRow
+		if err := rows.Scan(&i.AudioStreamSha256, &i.TrackIds); err != nil {
+			return nil, err
+		}
+		items = append(items, i)
+	}
+	if err := rows.Err(); err != nil {
+		return nil, err
+	}
+	return items, nil
+}
+
+const startDuplicateSweep = `-- name: StartDuplicateSweep :one
+INSERT INTO duplicate_sweeps DEFAULT VALUES RETURNING id, started_at
+`
+
+type StartDuplicateSweepRow struct {
+	ID        pgtype.UUID
+	StartedAt pgtype.Timestamptz
+}
+
+func (q *Queries) StartDuplicateSweep(ctx context.Context) (StartDuplicateSweepRow, error) {
+	row := q.db.QueryRow(ctx, startDuplicateSweep)
+	var i StartDuplicateSweepRow
+	err := row.Scan(&i.ID, &i.StartedAt)
+	return i, err
+}
+
+const upsertDuplicateGroup = `-- name: UpsertDuplicateGroup :one
+INSERT INTO duplicate_groups (member_key, tier, worst_bit_error_rate, last_seen_sweep_id)
+VALUES ($1, $2, $3, $4)
+ON CONFLICT (member_key) DO UPDATE
+   SET tier                 = EXCLUDED.tier,
+       worst_bit_error_rate = EXCLUDED.worst_bit_error_rate,
+       last_seen_sweep_id   = EXCLUDED.last_seen_sweep_id
+ WHERE duplicate_groups.status = 'pending'
+RETURNING id
+`
+
+type UpsertDuplicateGroupParams struct {
+	MemberKey         string
+	Tier              string
+	WorstBitErrorRate *float32
+	SweepID           pgtype.UUID
+}
+
+// Proposes a group, or refreshes one already pending. A group already dismissed
+// or merged is left exactly as it is: the WHERE on the update makes the conflict
+// a no-op, and the caller sees no row.
+func (q *Queries) UpsertDuplicateGroup(ctx context.Context, arg UpsertDuplicateGroupParams) (pgtype.UUID, error) {
+	row := q.db.QueryRow(ctx, upsertDuplicateGroup,
+		arg.MemberKey,
+		arg.Tier,
+		arg.WorstBitErrorRate,
+		arg.SweepID,
+	)
+	var id pgtype.UUID
+	err := row.Scan(&id)
+	return id, err
+}
@@ -297,6 +297,32 @@ type DiscoverTuning struct {
 	UpdatedAt        pgtype.Timestamptz
 }

+type DuplicateGroup struct {
+	ID                pgtype.UUID
+	MemberKey         string
+	Tier              string
+	WorstBitErrorRate *float32
+	Status            string
+	DetectedAt        pgtype.Timestamptz
+	LastSeenSweepID   pgtype.UUID
+	ResolvedAt        pgtype.Timestamptz
+}
+
+type DuplicateGroupMember struct {
+	GroupID pgtype.UUID
+	TrackID pgtype.UUID
+}
+
+type DuplicateSweep struct {
+	ID               pgtype.UUID
+	StartedAt        pgtype.Timestamptz
+	FinishedAt       pgtype.Timestamptz
+	Candidates       *int32
+	GroupsFound      *int32
+	OversizeClusters *int32
+	ErrorMessage     *string
+}
+
 type GeneralLike struct {
 	UserID  pgtype.UUID
 	TrackID pgtype.UUID