feat(admin): fingerprinting settings — on/off, length, match threshold, concurrency, sweep interval (M400 #3913)
test-go / test (push) Failing after 44s
test-web / test (push) Successful in 49s
test-go / integration (push) Failing after 2m42s
release / Build + push container image (push) Canceled after 0s
release / Verify release artifacts (tag releases only) (push) Canceled after 0s
release / Build signed APK (releases and dev) (push) Canceled after 4m8s

Rule 25: the fingerprinting knobs move out of source into a DB-backed
singleton (migration 0061), edited from a card on the Duplicates page and
shared live with the scanner, the backfill and the duplicate sweep through
one service instance, so a save needs no restart.

The length is the knob that can silently break the library: prints taken
at two lengths never match. Each track_fingerprints row now records the
length it was taken at, and every reader filters on the current one — the
backfill treats another length as stale, the gauge counts it pending, the
sweep never streams it. Equivalent to a version bump, except that setting
the length back makes rows not yet redone current again. The card warns
before a length change re-fingerprints the library.

Off stops every decode: the scan takes only the stream hash (a demux, and
what recognises a moved file) and stores nothing, dropping a changed file's
stale row; the backfill idles. A save also makes a sweep due, since a new
threshold or length changes what the same prints group into, and the sweep
interval gains slack so an hourly interval on an hourly tick doesn't skip
every other tick.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01SQ31KQpYbStyK5y58UmPLH
This commit is contained in:
2026-09-11 17:53:55 -04:00
co-authored by Claude Opus 5
parent c8bf9dc929
commit 077ae61235
38 changed files with 1679 additions and 201 deletions
+80 -3
View File
@@ -9,6 +9,9 @@ import (
"sort"
"strings"
"testing"
"time"
"github.com/jackc/pgx/v5/pgtype"
"git.fabledsword.com/bvandeusen/minstrel/internal/db/dbq"
syncpkg "git.fabledsword.com/bvandeusen/minstrel/internal/sync"
@@ -58,6 +61,7 @@ func TestDuplicateSweep_Integration(t *testing.T) {
}
if err := q.UpsertTrackFingerprint(ctx, dbq.UpsertTrackFingerprintParams{
TrackID: tr.ID, AudioStreamSha256: sum, Chromaprint: print, FingerprintVersion: fingerprintVersion,
ChromaprintLengthSec: defaultChromaprintLengthSec,
}); err != nil {
t.Fatalf("fingerprint %s: %v", name, err)
}
@@ -105,7 +109,7 @@ func TestDuplicateSweep_Integration(t *testing.T) {
}
// 1. A page size of one forces the keyset cursor across every candidate.
res, err := runDuplicateSweep(ctx, pool, logger, 1)
res, err := runDuplicateSweep(ctx, pool, logger, DefaultFingerprintSettings, 1)
if err != nil {
t.Fatalf("first sweep: %v", err)
}
@@ -133,7 +137,7 @@ func TestDuplicateSweep_Integration(t *testing.T) {
if _, err := pool.Exec(ctx, "UPDATE duplicate_groups SET status = 'dismissed' WHERE member_key = $1", acousticKey); err != nil {
t.Fatalf("dismiss: %v", err)
}
res, err = runDuplicateSweep(ctx, pool, logger, duplicateCandidatePage)
res, err = runDuplicateSweep(ctx, pool, logger, DefaultFingerprintSettings, duplicateCandidatePage)
if err != nil {
t.Fatalf("second sweep: %v", err)
}
@@ -150,7 +154,7 @@ func TestDuplicateSweep_Integration(t *testing.T) {
"DELETE FROM track_fingerprints f USING tracks t WHERE f.track_id = t.id AND t.file_path LIKE '%www-02.mp3'"); err != nil {
t.Fatalf("drop fingerprint: %v", err)
}
res, err = runDuplicateSweep(ctx, pool, logger, duplicateCandidatePage)
res, err = runDuplicateSweep(ctx, pool, logger, DefaultFingerprintSettings, duplicateCandidatePage)
if err != nil {
t.Fatalf("third sweep: %v", err)
}
@@ -170,4 +174,77 @@ func TestDuplicateSweep_Integration(t *testing.T) {
if !last.FinishedAt.Valid || last.ErrorMessage != nil {
t.Fatalf("latest sweep = %+v, want finished without error", last)
}
// 5. Prints taken at another length are never compared (#3913). Every print
// here was taken at the default length, so a sweep at 60s has nothing to read,
// rather than scoring 120s prints against each other as if they were 60s ones.
atOtherLength := DefaultFingerprintSettings
atOtherLength.ChromaprintLengthSec = 60
res, err = runDuplicateSweep(ctx, pool, logger, atOtherLength, duplicateCandidatePage)
if err != nil {
t.Fatalf("sweep at another length: %v", err)
}
if res.Candidates != 0 || res.Groups != 0 {
t.Fatalf("sweep at another length = %+v, want no candidates and no groups", res)
}
// 6. The sweep applies the threshold it is given. The recording's two copies
// disagree on about 5% of their bits: grouped at the default, not at 1%.
if _, err := pool.Exec(ctx, "DELETE FROM duplicate_groups"); err != nil {
t.Fatalf("clear groups: %v", err)
}
strict := DefaultFingerprintSettings
strict.AcousticMaxBitErrorRate = 0.01
res, err = runDuplicateSweep(ctx, pool, logger, strict, duplicateCandidatePage)
if err != nil {
t.Fatalf("strict sweep: %v", err)
}
if res.Groups != 0 {
t.Fatalf("sweep at a 1%% threshold = %+v, want the copies 5%% apart left ungrouped", res)
}
res, err = runDuplicateSweep(ctx, pool, logger, DefaultFingerprintSettings, duplicateCandidatePage)
if err != nil {
t.Fatalf("default sweep: %v", err)
}
if got := groups(); res.Groups != 1 || got[acousticKey] != (stored{"acoustic", "pending"}) {
t.Fatalf("sweep at the default threshold = %+v, groups %+v; want the recording's copies proposed", res, got)
}
}
func TestSweepIsDue(t *testing.T) {
now := time.Date(2026, 9, 11, 12, 0, 0, 0, time.UTC)
at := func(ago time.Duration) pgtype.Timestamptz {
return pgtype.Timestamptz{Time: now.Add(-ago), Valid: true}
}
never := pgtype.Timestamptz{}
hourly := DefaultFingerprintSettings
daily := DefaultFingerprintSettings
daily.SweepIntervalHours = 24
savedAgo := func(ago time.Duration) FingerprintSettings {
s := DefaultFingerprintSettings
s.UpdatedAt = now.Add(-ago)
return s
}
for _, tc := range []struct {
name string
latestPrint, lastSweep pgtype.Timestamptz
cfg FingerprintSettings
want bool
}{
{"nothing fingerprinted", never, never, hourly, false},
{"never swept", at(time.Minute), never, hourly, true},
{"new fingerprints since the last sweep", at(10 * time.Minute), at(2 * time.Hour), hourly, true},
{"nothing new since the last sweep", at(3 * time.Hour), at(2 * time.Hour), hourly, false},
// The sweep started a moment after the previous tick, so one tick later
// it is a moment under an hour old. Without the slack this is false.
{"one tick after an hourly sweep", at(time.Minute), at(time.Hour - 2*time.Second), hourly, true},
{"new fingerprints inside the interval", at(time.Minute), at(3 * time.Hour), daily, false},
{"new fingerprints past the interval", at(time.Minute), at(25 * time.Hour), daily, true},
{"settings saved since the last sweep", at(3 * time.Hour), at(2 * time.Hour), savedAgo(time.Hour), true},
{"settings saved before the last sweep", at(3 * time.Hour), at(2 * time.Hour), savedAgo(4 * time.Hour), false},
} {
if got := sweepIsDue(tc.latestPrint, tc.lastSweep, tc.cfg, now); got != tc.want {
t.Errorf("%s: sweepIsDue = %v, want %v", tc.name, got, tc.want)
}
}
}