feat(admin): fingerprinting settings — on/off, length, match threshold, concurrency, sweep interval (M400 #3913)
test-go / test (push) Failing after 44s
test-web / test (push) Successful in 49s
test-go / integration (push) Failing after 2m42s
release / Build + push container image (push) Canceled after 0s
release / Verify release artifacts (tag releases only) (push) Canceled after 0s
release / Build signed APK (releases and dev) (push) Canceled after 4m8s
test-go / test (push) Failing after 44s
test-web / test (push) Successful in 49s
test-go / integration (push) Failing after 2m42s
release / Build + push container image (push) Canceled after 0s
release / Verify release artifacts (tag releases only) (push) Canceled after 0s
release / Build signed APK (releases and dev) (push) Canceled after 4m8s
Rule 25: the fingerprinting knobs move out of source into a DB-backed singleton (migration 0061), edited from a card on the Duplicates page and shared live with the scanner, the backfill and the duplicate sweep through one service instance, so a save needs no restart. The length is the knob that can silently break the library: prints taken at two lengths never match. Each track_fingerprints row now records the length it was taken at, and every reader filters on the current one — the backfill treats another length as stale, the gauge counts it pending, the sweep never streams it. Equivalent to a version bump, except that setting the length back makes rows not yet redone current again. The card warns before a length change re-fingerprints the library. Off stops every decode: the scan takes only the stream hash (a demux, and what recognises a moved file) and stores nothing, dropping a changed file's stale row; the backfill idles. A save also makes a sweep due, since a new threshold or length changes what the same prints group into, and the sweep interval gains slack so an hourly interval on an hourly tick doesn't skip every other tick. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01SQ31KQpYbStyK5y58UmPLH
This commit is contained in:
@@ -7,6 +7,7 @@ import (
|
||||
"fmt"
|
||||
"io"
|
||||
"log/slog"
|
||||
"maps"
|
||||
"path/filepath"
|
||||
"sync"
|
||||
"testing"
|
||||
@@ -48,7 +49,7 @@ func TestFingerprintBackfill_Integration(t *testing.T) {
|
||||
} {
|
||||
if err := q.UpsertTrackFingerprint(ctx, dbq.UpsertTrackFingerprintParams{
|
||||
TrackID: seed.track.ID, AudioStreamSha256: sum, Chromaprint: []int32{1},
|
||||
FingerprintVersion: seed.version,
|
||||
FingerprintVersion: seed.version, ChromaprintLengthSec: defaultChromaprintLengthSec,
|
||||
}); err != nil {
|
||||
t.Fatalf("seed fingerprint: %v", err)
|
||||
}
|
||||
@@ -59,13 +60,24 @@ func TestFingerprintBackfill_Integration(t *testing.T) {
|
||||
|
||||
var mu sync.Mutex
|
||||
calls := map[string]int{}
|
||||
w := NewFingerprintBackfillWorker(pool, slog.New(slog.NewTextHandler(io.Discard, nil)))
|
||||
settings, err := NewFingerprintSettingsService(ctx, pool)
|
||||
if err != nil {
|
||||
t.Fatalf("fingerprint settings: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
if _, err := settings.Set(context.Background(), DefaultFingerprintSettings); err != nil {
|
||||
t.Errorf("restore fingerprint settings: %v", err)
|
||||
}
|
||||
})
|
||||
w := NewFingerprintBackfillWorker(pool, slog.New(slog.NewTextHandler(io.Discard, nil)), settings)
|
||||
// A batch of one forces the keyset cursor across several queries in a pass.
|
||||
w.batch = 1
|
||||
w.fingerprint = func(_ context.Context, path string) fingerprintResult {
|
||||
lengths := map[int32]int{}
|
||||
w.fingerprint = func(_ context.Context, path string, opts fingerprintOptions) fingerprintResult {
|
||||
name := filepath.Base(path)
|
||||
mu.Lock()
|
||||
calls[name]++
|
||||
lengths[opts.lengthSec]++
|
||||
mu.Unlock()
|
||||
switch name {
|
||||
case "stall.mp3":
|
||||
@@ -126,7 +138,7 @@ func TestFingerprintBackfill_Integration(t *testing.T) {
|
||||
}
|
||||
|
||||
// 4. The gauge counts what the passes wrote, and its buckets add up.
|
||||
cov, err := FingerprintCoverage(ctx, pool)
|
||||
cov, err := FingerprintCoverage(ctx, pool, settings.Get())
|
||||
if err != nil {
|
||||
t.Fatalf("coverage: %v", err)
|
||||
}
|
||||
@@ -138,6 +150,64 @@ func TestFingerprintBackfill_Integration(t *testing.T) {
|
||||
if cov.Fingerprinted+cov.Rejected+cov.Pending != cov.Total {
|
||||
t.Errorf("coverage buckets %+v do not sum to the total", cov)
|
||||
}
|
||||
|
||||
// 5. Changing the length (#3913) makes every stored row stale at once — the
|
||||
// gauge shows the whole library pending before the backfill has touched a
|
||||
// file — and the next pass re-derives each at the new length. The failure
|
||||
// this prevents is invisible from the UI: prints at two lengths that silently
|
||||
// never match.
|
||||
shorter := DefaultFingerprintSettings
|
||||
shorter.ChromaprintLengthSec = 60
|
||||
if _, err := settings.Set(ctx, shorter); err != nil {
|
||||
t.Fatalf("change length: %v", err)
|
||||
}
|
||||
cov, err = FingerprintCoverage(ctx, pool, settings.Get())
|
||||
if err != nil {
|
||||
t.Fatalf("coverage after length change: %v", err)
|
||||
}
|
||||
if cov.Total != 5 || cov.Pending != 5 {
|
||||
t.Fatalf("coverage after length change = %+v, want all 5 tracks pending", cov)
|
||||
}
|
||||
mu.Lock()
|
||||
clear(lengths)
|
||||
mu.Unlock()
|
||||
res, err = w.pass(ctx)
|
||||
if err != nil {
|
||||
t.Fatalf("new-length pass: %v", err)
|
||||
}
|
||||
if res.Processed != 5 {
|
||||
t.Fatalf("new-length pass = %+v, want all 5 present tracks re-derived", res)
|
||||
}
|
||||
mu.Lock()
|
||||
asked := maps.Clone(lengths)
|
||||
mu.Unlock()
|
||||
if len(asked) != 1 || asked[60] != 5 {
|
||||
t.Fatalf("new-length pass asked for lengths %v, want 60s for all 5", asked)
|
||||
}
|
||||
var atOldLength int
|
||||
if err := pool.QueryRow(ctx,
|
||||
"SELECT count(*) FROM track_fingerprints WHERE chromaprint_length_sec <> 60").Scan(&atOldLength); err != nil {
|
||||
t.Fatalf("count old-length rows: %v", err)
|
||||
}
|
||||
if atOldLength != 0 {
|
||||
t.Fatalf("%d fingerprints are still at the old length after a complete pass", atOldLength)
|
||||
}
|
||||
|
||||
// 6. Switched off, the backfill does nothing, even with work waiting.
|
||||
off := DefaultFingerprintSettings
|
||||
off.Enabled = false
|
||||
if _, err := settings.Set(ctx, off); err != nil {
|
||||
t.Fatalf("switch fingerprinting off: %v", err)
|
||||
}
|
||||
addTrack("later")
|
||||
res, err = w.pass(ctx)
|
||||
if err != nil {
|
||||
t.Fatalf("pass with fingerprinting off: %v", err)
|
||||
}
|
||||
if res.Processed != 0 || callCount("later.mp3") != 0 {
|
||||
t.Fatalf("pass with fingerprinting off = %+v (later.mp3 tried %d times), want nothing done",
|
||||
res, callCount("later.mp3"))
|
||||
}
|
||||
}
|
||||
|
||||
func TestBackfillFingerprintsResult_Add(t *testing.T) {
|
||||
|
||||
Reference in New Issue
Block a user