feat(admin): fingerprinting settings — on/off, length, match threshold, concurrency, sweep interval (M400 #3913)
test-go / test (push) Failing after 44s
test-web / test (push) Successful in 49s
test-go / integration (push) Failing after 2m42s
release / Build + push container image (push) Canceled after 0s
release / Verify release artifacts (tag releases only) (push) Canceled after 0s
release / Build signed APK (releases and dev) (push) Canceled after 4m8s
test-go / test (push) Failing after 44s
test-web / test (push) Successful in 49s
test-go / integration (push) Failing after 2m42s
release / Build + push container image (push) Canceled after 0s
release / Verify release artifacts (tag releases only) (push) Canceled after 0s
release / Build signed APK (releases and dev) (push) Canceled after 4m8s
Rule 25: the fingerprinting knobs move out of source into a DB-backed singleton (migration 0061), edited from a card on the Duplicates page and shared live with the scanner, the backfill and the duplicate sweep through one service instance, so a save needs no restart. The length is the knob that can silently break the library: prints taken at two lengths never match. Each track_fingerprints row now records the length it was taken at, and every reader filters on the current one — the backfill treats another length as stale, the gauge counts it pending, the sweep never streams it. Equivalent to a version bump, except that setting the length back makes rows not yet redone current again. The card warns before a length change re-fingerprints the library. Off stops every decode: the scan takes only the stream hash (a demux, and what recognises a moved file) and stores nothing, dropping a changed file's stale row; the backfill idles. A save also makes a sweep due, since a new threshold or length changes what the same prints group into, and the sweep interval gains slack so an hourly interval on an hourly tick doesn't skip every other tick. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01SQ31KQpYbStyK5y58UmPLH
This commit is contained in:
@@ -32,10 +32,17 @@ import (
|
||||
// carries a ~4 KB fingerprint, so a page is about 2 MB.
|
||||
const duplicateCandidatePage = 500
|
||||
|
||||
// duplicateSweepTick is how often the worker checks for anything new to sweep.
|
||||
// With nothing new, a tick is two cheap aggregate queries.
|
||||
// duplicateSweepTick is how often the worker checks whether a sweep is due. The
|
||||
// operator's sweep interval (#3913) is the least time between sweeps; the tick
|
||||
// only bounds how late past it one starts. With nothing due, a tick is two cheap
|
||||
// aggregate queries.
|
||||
const duplicateSweepTick = time.Hour
|
||||
|
||||
// sweepIntervalSlack absorbs the moment between a tick and the sweep it starts
|
||||
// stamping started_at. Without it a one-hour interval checked on a one-hour tick
|
||||
// would find the last sweep a moment under an hour old, and skip every other tick.
|
||||
const sweepIntervalSlack = 5 * time.Minute
|
||||
|
||||
// staleDuplicateSweepThreshold is the age past which an in-flight sweep is
|
||||
// assumed dead — a crash mid-sweep leaves finished_at NULL for ever — and another
|
||||
// may start. Twice the library scan's threshold, because a sweep compares
|
||||
@@ -58,13 +65,16 @@ type DuplicateSweepResult struct {
|
||||
Oversize int // acoustic clusters too large to propose
|
||||
}
|
||||
|
||||
// RunDuplicateSweep runs one sweep and records it in duplicate_sweeps.
|
||||
func RunDuplicateSweep(ctx context.Context, pool *pgxpool.Pool, logger *slog.Logger) (DuplicateSweepResult, error) {
|
||||
return runDuplicateSweep(ctx, pool, logger, duplicateCandidatePage)
|
||||
// RunDuplicateSweep runs one sweep and records it in duplicate_sweeps. cfg is a
|
||||
// snapshot: one sweep applies one threshold and one length throughout.
|
||||
func RunDuplicateSweep(
|
||||
ctx context.Context, pool *pgxpool.Pool, logger *slog.Logger, cfg FingerprintSettings,
|
||||
) (DuplicateSweepResult, error) {
|
||||
return runDuplicateSweep(ctx, pool, logger, cfg, duplicateCandidatePage)
|
||||
}
|
||||
|
||||
func runDuplicateSweep(
|
||||
ctx context.Context, pool *pgxpool.Pool, logger *slog.Logger, pageSize int32,
|
||||
ctx context.Context, pool *pgxpool.Pool, logger *slog.Logger, cfg FingerprintSettings, pageSize int32,
|
||||
) (DuplicateSweepResult, error) {
|
||||
q := dbq.New(pool)
|
||||
sweep, err := q.StartDuplicateSweep(ctx)
|
||||
@@ -72,7 +82,7 @@ func runDuplicateSweep(
|
||||
return DuplicateSweepResult{}, fmt.Errorf("start duplicate sweep: %w", err)
|
||||
}
|
||||
|
||||
res, runErr := sweepDuplicates(ctx, q, sweep.ID, pageSize)
|
||||
res, runErr := sweepDuplicates(ctx, q, sweep.ID, cfg, pageSize)
|
||||
|
||||
finishCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), duplicateSweepFinishTimeout)
|
||||
defer cancel()
|
||||
@@ -98,7 +108,7 @@ func runDuplicateSweep(
|
||||
}
|
||||
|
||||
func sweepDuplicates(
|
||||
ctx context.Context, q *dbq.Queries, sweepID pgtype.UUID, pageSize int32,
|
||||
ctx context.Context, q *dbq.Queries, sweepID pgtype.UUID, cfg FingerprintSettings, pageSize int32,
|
||||
) (DuplicateSweepResult, error) {
|
||||
var res DuplicateSweepResult
|
||||
|
||||
@@ -119,9 +129,12 @@ func sweepDuplicates(
|
||||
|
||||
// Acoustic tier, streamed in duration order. The first member of an exact
|
||||
// group the stream meets stands in for the whole group; the rest are skipped.
|
||||
grouper := newStreamGrouper(defaultAcousticMaxBitErrorRate)
|
||||
// Only prints at the current length are streamed: a print at another length
|
||||
// cannot be compared, and is waiting on the backfill to be re-derived.
|
||||
grouper := newStreamGrouper(cfg.AcousticMaxBitErrorRate)
|
||||
params := dbq.ListDuplicateCandidatesParams{
|
||||
CurrentVersion: fingerprintVersion,
|
||||
CurrentVersion: fingerprintVersion,
|
||||
ChromaprintLengthSec: cfg.ChromaprintLengthSec,
|
||||
// Durations are never negative, and the all-zero uuid sorts first: every
|
||||
// row is after this cursor. Valid must be true, or "> NULL" matches nothing.
|
||||
AfterDurationMs: -1,
|
||||
@@ -262,7 +275,9 @@ func formatUUIDs(ids []pgtype.UUID) []string {
|
||||
// running, reaping a sweep that has been in flight past
|
||||
// staleDuplicateSweepThreshold. Mirrors TryStartScan. The sweep runs on ctx, so
|
||||
// a caller answering an HTTP request must pass a context that outlives it.
|
||||
func TryStartDuplicateSweep(ctx context.Context, pool *pgxpool.Pool, logger *slog.Logger) (bool, error) {
|
||||
func TryStartDuplicateSweep(
|
||||
ctx context.Context, pool *pgxpool.Pool, logger *slog.Logger, cfg FingerprintSettings,
|
||||
) (bool, error) {
|
||||
q := dbq.New(pool)
|
||||
row, err := q.GetInFlightDuplicateSweep(ctx)
|
||||
switch {
|
||||
@@ -282,23 +297,28 @@ func TryStartDuplicateSweep(ctx context.Context, pool *pgxpool.Pool, logger *slo
|
||||
}
|
||||
|
||||
go func() {
|
||||
if _, err := RunDuplicateSweep(ctx, pool, logger); err != nil {
|
||||
if _, err := RunDuplicateSweep(ctx, pool, logger, cfg); err != nil {
|
||||
logger.Warn("duplicate sweep failed", "err", err)
|
||||
}
|
||||
}()
|
||||
return true, nil
|
||||
}
|
||||
|
||||
// DuplicateSweepWorker sweeps whenever fingerprints have changed.
|
||||
// DuplicateSweepWorker sweeps whenever its input has changed, at most once per
|
||||
// the operator's sweep interval.
|
||||
type DuplicateSweepWorker struct {
|
||||
pool *pgxpool.Pool
|
||||
logger *slog.Logger
|
||||
tick time.Duration
|
||||
pool *pgxpool.Pool
|
||||
logger *slog.Logger
|
||||
settings *FingerprintSettingsService
|
||||
tick time.Duration
|
||||
}
|
||||
|
||||
// NewDuplicateSweepWorker builds a worker with the production cadence.
|
||||
func NewDuplicateSweepWorker(pool *pgxpool.Pool, logger *slog.Logger) *DuplicateSweepWorker {
|
||||
return &DuplicateSweepWorker{pool: pool, logger: logger, tick: duplicateSweepTick}
|
||||
// NewDuplicateSweepWorker builds a worker with the production cadence. settings
|
||||
// is shared with the admin API; nil runs on defaults.
|
||||
func NewDuplicateSweepWorker(
|
||||
pool *pgxpool.Pool, logger *slog.Logger, settings *FingerprintSettingsService,
|
||||
) *DuplicateSweepWorker {
|
||||
return &DuplicateSweepWorker{pool: pool, logger: logger, settings: settings, tick: duplicateSweepTick}
|
||||
}
|
||||
|
||||
// Run blocks until ctx is cancelled, checking once at start and then each tick.
|
||||
@@ -323,7 +343,8 @@ func (w *DuplicateSweepWorker) tickOnce(ctx context.Context) {
|
||||
w.logger.Error("duplicate sweep: tick panicked", "panic", r)
|
||||
}
|
||||
}()
|
||||
due, err := duplicateSweepDue(ctx, dbq.New(w.pool))
|
||||
cfg := w.settings.Get()
|
||||
due, err := duplicateSweepDue(ctx, dbq.New(w.pool), cfg, time.Now())
|
||||
if err != nil {
|
||||
if ctx.Err() == nil {
|
||||
w.logger.Warn("duplicate sweep: due check failed", "err", err)
|
||||
@@ -333,28 +354,45 @@ func (w *DuplicateSweepWorker) tickOnce(ctx context.Context) {
|
||||
if !due {
|
||||
return
|
||||
}
|
||||
if _, err := TryStartDuplicateSweep(ctx, w.pool, w.logger); err != nil {
|
||||
if _, err := TryStartDuplicateSweep(ctx, w.pool, w.logger, cfg); err != nil {
|
||||
w.logger.Warn("duplicate sweep: start failed", "err", err)
|
||||
}
|
||||
}
|
||||
|
||||
// duplicateSweepDue reports whether any fingerprint was written after the latest
|
||||
// sweep started. Fingerprints are the sweep's only input, so nothing else can
|
||||
// change its answer; while the backfill is running this is true every tick.
|
||||
func duplicateSweepDue(ctx context.Context, q *dbq.Queries) (bool, error) {
|
||||
// duplicateSweepDue reads what sweepIsDue decides on.
|
||||
func duplicateSweepDue(ctx context.Context, q *dbq.Queries, cfg FingerprintSettings, now time.Time) (bool, error) {
|
||||
latest, err := q.GetLatestFingerprintComputedAt(ctx)
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("latest fingerprint: %w", err)
|
||||
}
|
||||
if !latest.Valid {
|
||||
return false, nil // nothing fingerprinted yet
|
||||
}
|
||||
var lastStart pgtype.Timestamptz
|
||||
last, err := q.GetLatestDuplicateSweep(ctx)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return true, nil
|
||||
}
|
||||
if err != nil {
|
||||
switch {
|
||||
case err == nil:
|
||||
lastStart = last.StartedAt
|
||||
case !errors.Is(err, pgx.ErrNoRows):
|
||||
return false, fmt.Errorf("latest duplicate sweep: %w", err)
|
||||
}
|
||||
return latest.Time.After(last.StartedAt.Time), nil
|
||||
return sweepIsDue(latest, lastStart, cfg, now), nil
|
||||
}
|
||||
|
||||
// sweepIsDue reports whether a sweep should start: something it reads has
|
||||
// changed since the last sweep started, and the operator's interval has passed.
|
||||
//
|
||||
// Two things can change its answer. Fingerprints are its input, so any written
|
||||
// after the last sweep began count; while the backfill runs that is true every
|
||||
// tick, which is what the interval is for. And a settings save counts, because a
|
||||
// new threshold or length changes what the same fingerprints group into.
|
||||
func sweepIsDue(latestFingerprint, lastSweepStart pgtype.Timestamptz, cfg FingerprintSettings, now time.Time) bool {
|
||||
if !latestFingerprint.Valid {
|
||||
return false // nothing fingerprinted yet
|
||||
}
|
||||
if !lastSweepStart.Valid {
|
||||
return true // never swept
|
||||
}
|
||||
interval := time.Duration(cfg.SweepIntervalHours) * time.Hour
|
||||
if now.Sub(lastSweepStart.Time) < interval-sweepIntervalSlack {
|
||||
return false
|
||||
}
|
||||
return latestFingerprint.Time.After(lastSweepStart.Time) || cfg.UpdatedAt.After(lastSweepStart.Time)
|
||||
}
|
||||
|
||||
@@ -9,6 +9,9 @@ import (
|
||||
"sort"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v5/pgtype"
|
||||
|
||||
"git.fabledsword.com/bvandeusen/minstrel/internal/db/dbq"
|
||||
syncpkg "git.fabledsword.com/bvandeusen/minstrel/internal/sync"
|
||||
@@ -58,6 +61,7 @@ func TestDuplicateSweep_Integration(t *testing.T) {
|
||||
}
|
||||
if err := q.UpsertTrackFingerprint(ctx, dbq.UpsertTrackFingerprintParams{
|
||||
TrackID: tr.ID, AudioStreamSha256: sum, Chromaprint: print, FingerprintVersion: fingerprintVersion,
|
||||
ChromaprintLengthSec: defaultChromaprintLengthSec,
|
||||
}); err != nil {
|
||||
t.Fatalf("fingerprint %s: %v", name, err)
|
||||
}
|
||||
@@ -105,7 +109,7 @@ func TestDuplicateSweep_Integration(t *testing.T) {
|
||||
}
|
||||
|
||||
// 1. A page size of one forces the keyset cursor across every candidate.
|
||||
res, err := runDuplicateSweep(ctx, pool, logger, 1)
|
||||
res, err := runDuplicateSweep(ctx, pool, logger, DefaultFingerprintSettings, 1)
|
||||
if err != nil {
|
||||
t.Fatalf("first sweep: %v", err)
|
||||
}
|
||||
@@ -133,7 +137,7 @@ func TestDuplicateSweep_Integration(t *testing.T) {
|
||||
if _, err := pool.Exec(ctx, "UPDATE duplicate_groups SET status = 'dismissed' WHERE member_key = $1", acousticKey); err != nil {
|
||||
t.Fatalf("dismiss: %v", err)
|
||||
}
|
||||
res, err = runDuplicateSweep(ctx, pool, logger, duplicateCandidatePage)
|
||||
res, err = runDuplicateSweep(ctx, pool, logger, DefaultFingerprintSettings, duplicateCandidatePage)
|
||||
if err != nil {
|
||||
t.Fatalf("second sweep: %v", err)
|
||||
}
|
||||
@@ -150,7 +154,7 @@ func TestDuplicateSweep_Integration(t *testing.T) {
|
||||
"DELETE FROM track_fingerprints f USING tracks t WHERE f.track_id = t.id AND t.file_path LIKE '%www-02.mp3'"); err != nil {
|
||||
t.Fatalf("drop fingerprint: %v", err)
|
||||
}
|
||||
res, err = runDuplicateSweep(ctx, pool, logger, duplicateCandidatePage)
|
||||
res, err = runDuplicateSweep(ctx, pool, logger, DefaultFingerprintSettings, duplicateCandidatePage)
|
||||
if err != nil {
|
||||
t.Fatalf("third sweep: %v", err)
|
||||
}
|
||||
@@ -170,4 +174,77 @@ func TestDuplicateSweep_Integration(t *testing.T) {
|
||||
if !last.FinishedAt.Valid || last.ErrorMessage != nil {
|
||||
t.Fatalf("latest sweep = %+v, want finished without error", last)
|
||||
}
|
||||
|
||||
// 5. Prints taken at another length are never compared (#3913). Every print
|
||||
// here was taken at the default length, so a sweep at 60s has nothing to read,
|
||||
// rather than scoring 120s prints against each other as if they were 60s ones.
|
||||
atOtherLength := DefaultFingerprintSettings
|
||||
atOtherLength.ChromaprintLengthSec = 60
|
||||
res, err = runDuplicateSweep(ctx, pool, logger, atOtherLength, duplicateCandidatePage)
|
||||
if err != nil {
|
||||
t.Fatalf("sweep at another length: %v", err)
|
||||
}
|
||||
if res.Candidates != 0 || res.Groups != 0 {
|
||||
t.Fatalf("sweep at another length = %+v, want no candidates and no groups", res)
|
||||
}
|
||||
|
||||
// 6. The sweep applies the threshold it is given. The recording's two copies
|
||||
// disagree on about 5% of their bits: grouped at the default, not at 1%.
|
||||
if _, err := pool.Exec(ctx, "DELETE FROM duplicate_groups"); err != nil {
|
||||
t.Fatalf("clear groups: %v", err)
|
||||
}
|
||||
strict := DefaultFingerprintSettings
|
||||
strict.AcousticMaxBitErrorRate = 0.01
|
||||
res, err = runDuplicateSweep(ctx, pool, logger, strict, duplicateCandidatePage)
|
||||
if err != nil {
|
||||
t.Fatalf("strict sweep: %v", err)
|
||||
}
|
||||
if res.Groups != 0 {
|
||||
t.Fatalf("sweep at a 1%% threshold = %+v, want the copies 5%% apart left ungrouped", res)
|
||||
}
|
||||
res, err = runDuplicateSweep(ctx, pool, logger, DefaultFingerprintSettings, duplicateCandidatePage)
|
||||
if err != nil {
|
||||
t.Fatalf("default sweep: %v", err)
|
||||
}
|
||||
if got := groups(); res.Groups != 1 || got[acousticKey] != (stored{"acoustic", "pending"}) {
|
||||
t.Fatalf("sweep at the default threshold = %+v, groups %+v; want the recording's copies proposed", res, got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSweepIsDue(t *testing.T) {
|
||||
now := time.Date(2026, 9, 11, 12, 0, 0, 0, time.UTC)
|
||||
at := func(ago time.Duration) pgtype.Timestamptz {
|
||||
return pgtype.Timestamptz{Time: now.Add(-ago), Valid: true}
|
||||
}
|
||||
never := pgtype.Timestamptz{}
|
||||
hourly := DefaultFingerprintSettings
|
||||
daily := DefaultFingerprintSettings
|
||||
daily.SweepIntervalHours = 24
|
||||
savedAgo := func(ago time.Duration) FingerprintSettings {
|
||||
s := DefaultFingerprintSettings
|
||||
s.UpdatedAt = now.Add(-ago)
|
||||
return s
|
||||
}
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
latestPrint, lastSweep pgtype.Timestamptz
|
||||
cfg FingerprintSettings
|
||||
want bool
|
||||
}{
|
||||
{"nothing fingerprinted", never, never, hourly, false},
|
||||
{"never swept", at(time.Minute), never, hourly, true},
|
||||
{"new fingerprints since the last sweep", at(10 * time.Minute), at(2 * time.Hour), hourly, true},
|
||||
{"nothing new since the last sweep", at(3 * time.Hour), at(2 * time.Hour), hourly, false},
|
||||
// The sweep started a moment after the previous tick, so one tick later
|
||||
// it is a moment under an hour old. Without the slack this is false.
|
||||
{"one tick after an hourly sweep", at(time.Minute), at(time.Hour - 2*time.Second), hourly, true},
|
||||
{"new fingerprints inside the interval", at(time.Minute), at(3 * time.Hour), daily, false},
|
||||
{"new fingerprints past the interval", at(time.Minute), at(25 * time.Hour), daily, true},
|
||||
{"settings saved since the last sweep", at(3 * time.Hour), at(2 * time.Hour), savedAgo(time.Hour), true},
|
||||
{"settings saved before the last sweep", at(3 * time.Hour), at(2 * time.Hour), savedAgo(4 * time.Hour), false},
|
||||
} {
|
||||
if got := sweepIsDue(tc.latestPrint, tc.lastSweep, tc.cfg, now); got != tc.want {
|
||||
t.Errorf("%s: sweepIsDue = %v, want %v", tc.name, got, tc.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -50,21 +50,28 @@ const fingerprintTimeout = 60 * time.Second
|
||||
const fingerprintWaitDelay = 5 * time.Second
|
||||
|
||||
// fingerprintVersion stamps how a track_fingerprints row was derived. Bump it
|
||||
// whenever the derivation changes — the hash arguments, fpcalc's flags or its
|
||||
// length — and the backfill re-derives every row below it. Fingerprints taken
|
||||
// by two methods are not comparable, and nothing else would reveal that the
|
||||
// library held a mix.
|
||||
// whenever the derivation changes — the hash arguments or fpcalc's flags — and
|
||||
// the backfill re-derives every row below it. Fingerprints taken by two methods
|
||||
// are not comparable, and nothing else would reveal that the library held a mix.
|
||||
//
|
||||
// The length is deliberately not part of it: it is an operator setting (#3913),
|
||||
// so each row records the length it was taken at and readers compare only rows
|
||||
// at the current one. See fingerprint_settings.go.
|
||||
const fingerprintVersion int16 = 1
|
||||
|
||||
// errFingerprintTimeout marks a tool that ran out of time. Distinct from a
|
||||
// failed exit because a stall is a fact about the mount, not about the file.
|
||||
var errFingerprintTimeout = errors.New("fingerprint tool timed out")
|
||||
|
||||
// defaultChromaprintLengthSec is how many seconds of audio fpcalc fingerprints.
|
||||
// 120 is fpcalc's own default. Fingerprints taken at different lengths are not
|
||||
// comparable, so changing this has to re-derive every stored one.
|
||||
// defaultChromaprintLengthSec is the shipped value of the length setting (#3913):
|
||||
// how many seconds of audio fpcalc fingerprints. 120 is fpcalc's own default.
|
||||
const defaultChromaprintLengthSec = 120
|
||||
|
||||
// errChromaprintSkipped marks a chromaprint not taken because fingerprinting is
|
||||
// switched off. Inconclusive rather than a verdict: nothing was learned about
|
||||
// the file.
|
||||
var errChromaprintSkipped = errors.New("chromaprint skipped: fingerprinting is off")
|
||||
|
||||
// fpcalcStderrTail caps how much of a failing tool's stderr reaches the log.
|
||||
const fpcalcStderrTail = 512
|
||||
|
||||
@@ -113,11 +120,24 @@ type fingerprintResult struct {
|
||||
printErr error
|
||||
}
|
||||
|
||||
// computeFingerprint derives both halves for the file at path.
|
||||
func computeFingerprint(ctx context.Context, path string) fingerprintResult {
|
||||
// fingerprintOptions is what the settings decide for one attempt. Captured once
|
||||
// per file, so the length a chromaprint was taken at is the length stored with it
|
||||
// even if the setting changes mid-attempt.
|
||||
type fingerprintOptions struct {
|
||||
lengthSec int32
|
||||
// chromaprint false takes the stream hash alone: a demux, no decode.
|
||||
chromaprint bool
|
||||
}
|
||||
|
||||
// computeFingerprint derives the halves opts asks for, for the file at path.
|
||||
func computeFingerprint(ctx context.Context, path string, opts fingerprintOptions) fingerprintResult {
|
||||
var r fingerprintResult
|
||||
r.streamSHA256, r.hashErr = computeAudioStreamSHA256(ctx, path)
|
||||
r.chromaprint, r.printErr = computeChromaprint(ctx, path, defaultChromaprintLengthSec)
|
||||
if !opts.chromaprint {
|
||||
r.printErr = errChromaprintSkipped
|
||||
return r
|
||||
}
|
||||
r.chromaprint, r.printErr = computeChromaprint(ctx, path, opts.lengthSec)
|
||||
return r
|
||||
}
|
||||
|
||||
@@ -130,11 +150,13 @@ func (r fingerprintResult) inconclusive() bool {
|
||||
}
|
||||
|
||||
// isInconclusive names the failures that are not a verdict on the file: a
|
||||
// stall, a cancelled scan, and a tool that is not installed. The last matters
|
||||
// outside the image — a dev binary run without fpcalc on PATH must not stamp
|
||||
// every track in the library as unfingerprintable.
|
||||
// stall, a cancelled scan, a tool that is not installed, and a chromaprint
|
||||
// skipped because fingerprinting is off. A missing tool matters outside the
|
||||
// image — a dev binary run without fpcalc on PATH must not stamp every track in
|
||||
// the library as unfingerprintable.
|
||||
func isInconclusive(err error) bool {
|
||||
return errors.Is(err, errFingerprintTimeout) ||
|
||||
errors.Is(err, errChromaprintSkipped) ||
|
||||
errors.Is(err, context.Canceled) ||
|
||||
errors.Is(err, context.DeadlineExceeded) ||
|
||||
errors.Is(err, exec.ErrNotFound)
|
||||
@@ -142,11 +164,11 @@ func isInconclusive(err error) bool {
|
||||
|
||||
// fingerprintFile runs the scanner's fingerprinter. A Scanner built without New
|
||||
// gets the real tools rather than a nil-func panic halfway through a scan.
|
||||
func (s *Scanner) fingerprintFile(ctx context.Context, path string) fingerprintResult {
|
||||
func (s *Scanner) fingerprintFile(ctx context.Context, path string, opts fingerprintOptions) fingerprintResult {
|
||||
if s.fingerprint == nil {
|
||||
return computeFingerprint(ctx, path)
|
||||
return computeFingerprint(ctx, path, opts)
|
||||
}
|
||||
return s.fingerprint(ctx, path)
|
||||
return s.fingerprint(ctx, path, opts)
|
||||
}
|
||||
|
||||
// fingerprintOutcome is what storeFingerprint did with one attempt.
|
||||
@@ -163,9 +185,11 @@ const (
|
||||
// the backfill (#3908) alike, so there is one rule for what gets written. It
|
||||
// never fails its caller: a missing fingerprint only keeps a track out of
|
||||
// duplicate detection, which is not worth dropping a scan or a pass over.
|
||||
//
|
||||
// lengthSec is the length fp's chromaprint was taken at, stored with it (#3913).
|
||||
func storeFingerprint(
|
||||
ctx context.Context, q *dbq.Queries, logger *slog.Logger,
|
||||
trackID pgtype.UUID, path string, fp fingerprintResult,
|
||||
trackID pgtype.UUID, path string, fp fingerprintResult, lengthSec int32,
|
||||
) fingerprintOutcome {
|
||||
if fp.hashErr != nil {
|
||||
logger.Warn("fingerprint: audio stream hash failed", "path", path, "err", fp.hashErr)
|
||||
@@ -186,10 +210,11 @@ func storeFingerprint(
|
||||
// is stamped at the current version so the backfill does not retry it on
|
||||
// every pass. It is retried when the file changes.
|
||||
if err := q.UpsertTrackFingerprint(ctx, dbq.UpsertTrackFingerprintParams{
|
||||
TrackID: trackID,
|
||||
AudioStreamSha256: fp.streamSHA256,
|
||||
Chromaprint: fp.chromaprint,
|
||||
FingerprintVersion: fingerprintVersion,
|
||||
TrackID: trackID,
|
||||
AudioStreamSha256: fp.streamSHA256,
|
||||
Chromaprint: fp.chromaprint,
|
||||
FingerprintVersion: fingerprintVersion,
|
||||
ChromaprintLengthSec: lengthSec,
|
||||
}); err != nil {
|
||||
logger.Warn("fingerprint: storing fingerprint failed", "path", path, "err", err)
|
||||
return outcomeStoreFailed
|
||||
@@ -211,8 +236,8 @@ func computeAudioStreamSHA256(ctx context.Context, path string) ([]byte, error)
|
||||
|
||||
// computeChromaprint returns the raw acoustic fingerprint of the first
|
||||
// lengthSec seconds of the file.
|
||||
func computeChromaprint(ctx context.Context, path string, lengthSec int) ([]int32, error) {
|
||||
out, err := runFingerprintTool(ctx, "fpcalc", fpcalcArgs(path, lengthSec))
|
||||
func computeChromaprint(ctx context.Context, path string, lengthSec int32) ([]int32, error) {
|
||||
out, err := runFingerprintTool(ctx, "fpcalc", fpcalcArgs(path, int(lengthSec)))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
@@ -40,10 +40,11 @@ const fingerprintBackfillTick = time.Hour
|
||||
// so tracks the scan adds mid-pass are not stuck behind one enormous page.
|
||||
const fingerprintBackfillBatch = 50
|
||||
|
||||
// fingerprintBackfillConcurrency is how many files are decoded at once. Two is
|
||||
// deliberately low: fpcalc and the stream hash compete with playback transcoding
|
||||
// for CPU and with streaming for the mount, and a backfill that makes playback
|
||||
// stutter is worse than one that takes longer. Operator-tunable in #3913.
|
||||
// fingerprintBackfillConcurrency is the shipped value of the concurrency setting
|
||||
// (#3913): how many files are decoded at once. Two is deliberately low: fpcalc
|
||||
// and the stream hash compete with playback transcoding for CPU and with
|
||||
// streaming for the mount, and a backfill that makes playback stutter is worse
|
||||
// than one that takes longer.
|
||||
const fingerprintBackfillConcurrency = 2
|
||||
|
||||
// BackfillFingerprintsResult tallies one pass.
|
||||
@@ -68,24 +69,27 @@ func (r *BackfillFingerprintsResult) add(o fingerprintOutcome) {
|
||||
|
||||
// FingerprintBackfillWorker fingerprints the tracks the scan never will.
|
||||
type FingerprintBackfillWorker struct {
|
||||
pool *pgxpool.Pool
|
||||
logger *slog.Logger
|
||||
tick time.Duration
|
||||
batch int32
|
||||
concurrency int
|
||||
pool *pgxpool.Pool
|
||||
logger *slog.Logger
|
||||
settings *FingerprintSettingsService
|
||||
tick time.Duration
|
||||
batch int32
|
||||
// fingerprint is a field for the same reason as Scanner.fingerprint: an
|
||||
// integration test pins which tracks a pass touches, not what the tools print.
|
||||
fingerprint func(ctx context.Context, path string) fingerprintResult
|
||||
fingerprint func(ctx context.Context, path string, opts fingerprintOptions) fingerprintResult
|
||||
}
|
||||
|
||||
// NewFingerprintBackfillWorker builds a worker with the production cadence.
|
||||
func NewFingerprintBackfillWorker(pool *pgxpool.Pool, logger *slog.Logger) *FingerprintBackfillWorker {
|
||||
// settings is shared with the scanner and the admin API; nil runs on defaults.
|
||||
func NewFingerprintBackfillWorker(
|
||||
pool *pgxpool.Pool, logger *slog.Logger, settings *FingerprintSettingsService,
|
||||
) *FingerprintBackfillWorker {
|
||||
return &FingerprintBackfillWorker{
|
||||
pool: pool,
|
||||
logger: logger,
|
||||
settings: settings,
|
||||
tick: fingerprintBackfillTick,
|
||||
batch: fingerprintBackfillBatch,
|
||||
concurrency: fingerprintBackfillConcurrency,
|
||||
fingerprint: computeFingerprint,
|
||||
}
|
||||
}
|
||||
@@ -129,6 +133,12 @@ func (w *FingerprintBackfillWorker) runOnce(ctx context.Context) {
|
||||
// cursor is what lets a pass end: an inconclusive attempt writes no row, so a
|
||||
// file that keeps timing out would otherwise be listed again immediately and
|
||||
// retried forever within the pass.
|
||||
//
|
||||
// Settings are read before every batch, so a save takes effect within a batch
|
||||
// rather than an hour (#3913): switching fingerprinting off ends the pass, a new
|
||||
// concurrency applies to the next batch, and a new length restarts the walk from
|
||||
// the top at that length, because every row written at the old one went stale
|
||||
// the moment it changed.
|
||||
func (w *FingerprintBackfillWorker) pass(ctx context.Context) (BackfillFingerprintsResult, error) {
|
||||
q := dbq.New(w.pool)
|
||||
var (
|
||||
@@ -137,15 +147,25 @@ func (w *FingerprintBackfillWorker) pass(ctx context.Context) (BackfillFingerpri
|
||||
)
|
||||
// The all-zero uuid sorts before every real id. Valid must be true: a NULL
|
||||
// cursor would make "id > NULL" match nothing and every pass a silent no-op.
|
||||
after := pgtype.UUID{Valid: true}
|
||||
start := pgtype.UUID{Valid: true}
|
||||
after := start
|
||||
lengthSec := w.settings.Get().ChromaprintLengthSec
|
||||
for {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return res, err
|
||||
}
|
||||
cfg := w.settings.Get()
|
||||
if !cfg.Enabled {
|
||||
return res, nil
|
||||
}
|
||||
if cfg.ChromaprintLengthSec != lengthSec {
|
||||
lengthSec, after = cfg.ChromaprintLengthSec, start
|
||||
}
|
||||
rows, err := q.ListTracksNeedingFingerprint(ctx, dbq.ListTracksNeedingFingerprintParams{
|
||||
CurrentVersion: fingerprintVersion,
|
||||
AfterID: after,
|
||||
BatchLimit: w.batch,
|
||||
CurrentVersion: fingerprintVersion,
|
||||
ChromaprintLengthSec: lengthSec,
|
||||
AfterID: after,
|
||||
BatchLimit: w.batch,
|
||||
})
|
||||
if err != nil {
|
||||
return res, fmt.Errorf("list tracks needing fingerprint: %w", err)
|
||||
@@ -154,7 +174,10 @@ func (w *FingerprintBackfillWorker) pass(ctx context.Context) (BackfillFingerpri
|
||||
return res, nil
|
||||
}
|
||||
|
||||
sem := make(chan struct{}, w.concurrency)
|
||||
opts := fingerprintOptions{lengthSec: lengthSec, chromaprint: true}
|
||||
// Validation keeps concurrency at one or more; the floor guards a zero
|
||||
// that would block the first send for ever.
|
||||
sem := make(chan struct{}, max(1, int(cfg.BackfillConcurrency)))
|
||||
var wg sync.WaitGroup
|
||||
for _, row := range rows {
|
||||
if ctx.Err() != nil {
|
||||
@@ -170,7 +193,7 @@ func (w *FingerprintBackfillWorker) pass(ctx context.Context) (BackfillFingerpri
|
||||
w.logger.Error("fingerprint backfill: track panicked", "path", path, "panic", r)
|
||||
}
|
||||
}()
|
||||
outcome := storeFingerprint(ctx, q, w.logger, trackID, path, w.fingerprintFile(ctx, path))
|
||||
outcome := storeFingerprint(ctx, q, w.logger, trackID, path, w.fingerprintFile(ctx, path, opts), lengthSec)
|
||||
mu.Lock()
|
||||
res.add(outcome)
|
||||
mu.Unlock()
|
||||
@@ -181,16 +204,21 @@ func (w *FingerprintBackfillWorker) pass(ctx context.Context) (BackfillFingerpri
|
||||
}
|
||||
}
|
||||
|
||||
func (w *FingerprintBackfillWorker) fingerprintFile(ctx context.Context, path string) fingerprintResult {
|
||||
func (w *FingerprintBackfillWorker) fingerprintFile(ctx context.Context, path string, opts fingerprintOptions) fingerprintResult {
|
||||
if w.fingerprint == nil {
|
||||
return computeFingerprint(ctx, path)
|
||||
return computeFingerprint(ctx, path, opts)
|
||||
}
|
||||
return w.fingerprint(ctx, path)
|
||||
return w.fingerprint(ctx, path, opts)
|
||||
}
|
||||
|
||||
// FingerprintCoverage reports how much of the library carries a current
|
||||
// fingerprint, for the admin gauge. It lives here, beside the backfill, so the
|
||||
// version it counts against is the one the backfill writes.
|
||||
func FingerprintCoverage(ctx context.Context, pool *pgxpool.Pool) (dbq.GetFingerprintCoverageRow, error) {
|
||||
return dbq.New(pool).GetFingerprintCoverage(ctx, fingerprintVersion)
|
||||
// version and length it counts against are the ones the backfill writes.
|
||||
func FingerprintCoverage(
|
||||
ctx context.Context, pool *pgxpool.Pool, cfg FingerprintSettings,
|
||||
) (dbq.GetFingerprintCoverageRow, error) {
|
||||
return dbq.New(pool).GetFingerprintCoverage(ctx, dbq.GetFingerprintCoverageParams{
|
||||
CurrentVersion: fingerprintVersion,
|
||||
ChromaprintLengthSec: cfg.ChromaprintLengthSec,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -7,6 +7,7 @@ import (
|
||||
"fmt"
|
||||
"io"
|
||||
"log/slog"
|
||||
"maps"
|
||||
"path/filepath"
|
||||
"sync"
|
||||
"testing"
|
||||
@@ -48,7 +49,7 @@ func TestFingerprintBackfill_Integration(t *testing.T) {
|
||||
} {
|
||||
if err := q.UpsertTrackFingerprint(ctx, dbq.UpsertTrackFingerprintParams{
|
||||
TrackID: seed.track.ID, AudioStreamSha256: sum, Chromaprint: []int32{1},
|
||||
FingerprintVersion: seed.version,
|
||||
FingerprintVersion: seed.version, ChromaprintLengthSec: defaultChromaprintLengthSec,
|
||||
}); err != nil {
|
||||
t.Fatalf("seed fingerprint: %v", err)
|
||||
}
|
||||
@@ -59,13 +60,24 @@ func TestFingerprintBackfill_Integration(t *testing.T) {
|
||||
|
||||
var mu sync.Mutex
|
||||
calls := map[string]int{}
|
||||
w := NewFingerprintBackfillWorker(pool, slog.New(slog.NewTextHandler(io.Discard, nil)))
|
||||
settings, err := NewFingerprintSettingsService(ctx, pool)
|
||||
if err != nil {
|
||||
t.Fatalf("fingerprint settings: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
if _, err := settings.Set(context.Background(), DefaultFingerprintSettings); err != nil {
|
||||
t.Errorf("restore fingerprint settings: %v", err)
|
||||
}
|
||||
})
|
||||
w := NewFingerprintBackfillWorker(pool, slog.New(slog.NewTextHandler(io.Discard, nil)), settings)
|
||||
// A batch of one forces the keyset cursor across several queries in a pass.
|
||||
w.batch = 1
|
||||
w.fingerprint = func(_ context.Context, path string) fingerprintResult {
|
||||
lengths := map[int32]int{}
|
||||
w.fingerprint = func(_ context.Context, path string, opts fingerprintOptions) fingerprintResult {
|
||||
name := filepath.Base(path)
|
||||
mu.Lock()
|
||||
calls[name]++
|
||||
lengths[opts.lengthSec]++
|
||||
mu.Unlock()
|
||||
switch name {
|
||||
case "stall.mp3":
|
||||
@@ -126,7 +138,7 @@ func TestFingerprintBackfill_Integration(t *testing.T) {
|
||||
}
|
||||
|
||||
// 4. The gauge counts what the passes wrote, and its buckets add up.
|
||||
cov, err := FingerprintCoverage(ctx, pool)
|
||||
cov, err := FingerprintCoverage(ctx, pool, settings.Get())
|
||||
if err != nil {
|
||||
t.Fatalf("coverage: %v", err)
|
||||
}
|
||||
@@ -138,6 +150,64 @@ func TestFingerprintBackfill_Integration(t *testing.T) {
|
||||
if cov.Fingerprinted+cov.Rejected+cov.Pending != cov.Total {
|
||||
t.Errorf("coverage buckets %+v do not sum to the total", cov)
|
||||
}
|
||||
|
||||
// 5. Changing the length (#3913) makes every stored row stale at once — the
|
||||
// gauge shows the whole library pending before the backfill has touched a
|
||||
// file — and the next pass re-derives each at the new length. The failure
|
||||
// this prevents is invisible from the UI: prints at two lengths that silently
|
||||
// never match.
|
||||
shorter := DefaultFingerprintSettings
|
||||
shorter.ChromaprintLengthSec = 60
|
||||
if _, err := settings.Set(ctx, shorter); err != nil {
|
||||
t.Fatalf("change length: %v", err)
|
||||
}
|
||||
cov, err = FingerprintCoverage(ctx, pool, settings.Get())
|
||||
if err != nil {
|
||||
t.Fatalf("coverage after length change: %v", err)
|
||||
}
|
||||
if cov.Total != 5 || cov.Pending != 5 {
|
||||
t.Fatalf("coverage after length change = %+v, want all 5 tracks pending", cov)
|
||||
}
|
||||
mu.Lock()
|
||||
clear(lengths)
|
||||
mu.Unlock()
|
||||
res, err = w.pass(ctx)
|
||||
if err != nil {
|
||||
t.Fatalf("new-length pass: %v", err)
|
||||
}
|
||||
if res.Processed != 5 {
|
||||
t.Fatalf("new-length pass = %+v, want all 5 present tracks re-derived", res)
|
||||
}
|
||||
mu.Lock()
|
||||
asked := maps.Clone(lengths)
|
||||
mu.Unlock()
|
||||
if len(asked) != 1 || asked[60] != 5 {
|
||||
t.Fatalf("new-length pass asked for lengths %v, want 60s for all 5", asked)
|
||||
}
|
||||
var atOldLength int
|
||||
if err := pool.QueryRow(ctx,
|
||||
"SELECT count(*) FROM track_fingerprints WHERE chromaprint_length_sec <> 60").Scan(&atOldLength); err != nil {
|
||||
t.Fatalf("count old-length rows: %v", err)
|
||||
}
|
||||
if atOldLength != 0 {
|
||||
t.Fatalf("%d fingerprints are still at the old length after a complete pass", atOldLength)
|
||||
}
|
||||
|
||||
// 6. Switched off, the backfill does nothing, even with work waiting.
|
||||
off := DefaultFingerprintSettings
|
||||
off.Enabled = false
|
||||
if _, err := settings.Set(ctx, off); err != nil {
|
||||
t.Fatalf("switch fingerprinting off: %v", err)
|
||||
}
|
||||
addTrack("later")
|
||||
res, err = w.pass(ctx)
|
||||
if err != nil {
|
||||
t.Fatalf("pass with fingerprinting off: %v", err)
|
||||
}
|
||||
if res.Processed != 0 || callCount("later.mp3") != 0 {
|
||||
t.Fatalf("pass with fingerprinting off = %+v (later.mp3 tried %d times), want nothing done",
|
||||
res, callCount("later.mp3"))
|
||||
}
|
||||
}
|
||||
|
||||
func TestBackfillFingerprintsResult_Add(t *testing.T) {
|
||||
|
||||
@@ -60,9 +60,24 @@ func TestScanner_FingerprintsOnlyNewOrChangedBytes_Integration(t *testing.T) {
|
||||
result := fingerprintResult{streamSHA256: sum, chromaprint: chroma}
|
||||
calls := map[string]int{}
|
||||
|
||||
scanner := New(pool, logger, []string{root})
|
||||
scanner.fingerprint = func(_ context.Context, path string) fingerprintResult {
|
||||
settings, err := NewFingerprintSettingsService(ctx, pool)
|
||||
if err != nil {
|
||||
t.Fatalf("fingerprint settings: %v", err)
|
||||
}
|
||||
if _, err := settings.Set(ctx, DefaultFingerprintSettings); err != nil {
|
||||
t.Fatalf("reset fingerprint settings: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
if _, err := settings.Set(context.Background(), DefaultFingerprintSettings); err != nil {
|
||||
t.Errorf("restore fingerprint settings: %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
var lastOpts fingerprintOptions
|
||||
scanner := New(pool, logger, []string{root}, settings)
|
||||
scanner.fingerprint = func(_ context.Context, path string, opts fingerprintOptions) fingerprintResult {
|
||||
calls[path]++
|
||||
lastOpts = opts
|
||||
return result
|
||||
}
|
||||
|
||||
@@ -78,14 +93,15 @@ func TestScanner_FingerprintsOnlyNewOrChangedBytes_Integration(t *testing.T) {
|
||||
sha []byte
|
||||
chroma []int32
|
||||
version int16
|
||||
length int32
|
||||
}
|
||||
stored := func(path string) (row, bool) {
|
||||
t.Helper()
|
||||
var r row
|
||||
err := pool.QueryRow(ctx, `
|
||||
SELECT f.audio_stream_sha256, f.chromaprint, f.fingerprint_version
|
||||
SELECT f.audio_stream_sha256, f.chromaprint, f.fingerprint_version, f.chromaprint_length_sec
|
||||
FROM track_fingerprints f JOIN tracks t ON t.id = f.track_id
|
||||
WHERE t.file_path = $1`, path).Scan(&r.sha, &r.chroma, &r.version)
|
||||
WHERE t.file_path = $1`, path).Scan(&r.sha, &r.chroma, &r.version, &r.length)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return row{}, false
|
||||
}
|
||||
@@ -113,8 +129,13 @@ func TestScanner_FingerprintsOnlyNewOrChangedBytes_Integration(t *testing.T) {
|
||||
if !ok {
|
||||
t.Fatal("first scan stored no fingerprint")
|
||||
}
|
||||
if !bytes.Equal(got.sha, sum) || !slices.Equal(got.chroma, chroma) || got.version != fingerprintVersion {
|
||||
t.Fatalf("stored %+v, want sha %x chromaprint %v version %d", got, sum, chroma, fingerprintVersion)
|
||||
if !bytes.Equal(got.sha, sum) || !slices.Equal(got.chroma, chroma) || got.version != fingerprintVersion ||
|
||||
got.length != defaultChromaprintLengthSec {
|
||||
t.Fatalf("stored %+v, want sha %x chromaprint %v version %d length %d",
|
||||
got, sum, chroma, fingerprintVersion, defaultChromaprintLengthSec)
|
||||
}
|
||||
if !lastOpts.chromaprint || lastOpts.lengthSec != defaultChromaprintLengthSec {
|
||||
t.Fatalf("first scan asked for %+v, want a chromaprint at the default length", lastOpts)
|
||||
}
|
||||
|
||||
// 2. A tag-repair pass re-reads every unchanged file and must not
|
||||
@@ -171,4 +192,39 @@ func TestScanner_FingerprintsOnlyNewOrChangedBytes_Integration(t *testing.T) {
|
||||
if got.sha != nil || got.chroma != nil || got.version != fingerprintVersion {
|
||||
t.Fatalf("rejected file stored %+v, want both halves NULL at version %d", got, fingerprintVersion)
|
||||
}
|
||||
|
||||
// 6. With fingerprinting off (#3913) the scan decodes nothing. It still asks
|
||||
// for the stream hash — a demux, and what recognises a moved file — but stores
|
||||
// no row, and a changed file's old row goes: it describes bytes that are gone.
|
||||
result = fingerprintResult{streamSHA256: sum, chromaprint: chroma}
|
||||
off := DefaultFingerprintSettings
|
||||
off.Enabled = false
|
||||
if _, err := settings.Set(ctx, off); err != nil {
|
||||
t.Fatalf("switch fingerprinting off: %v", err)
|
||||
}
|
||||
touch(a, 4*time.Hour)
|
||||
scan("fingerprinting-off scan")
|
||||
if calls[a] != 5 || lastOpts.chromaprint {
|
||||
t.Fatalf("fingerprinting-off scan: calls = %v, last options %+v; want a fifth call asking for no chromaprint",
|
||||
calls, lastOpts)
|
||||
}
|
||||
if _, ok := stored(a); ok {
|
||||
t.Fatal("with fingerprinting off, a changed file kept the previous bytes' fingerprint")
|
||||
}
|
||||
if _, ok := stored(b); !ok {
|
||||
t.Fatal("with fingerprinting off, an unchanged file lost its fingerprint")
|
||||
}
|
||||
|
||||
// 7. The length setting reaches the scan, and is stored with the row.
|
||||
longer := DefaultFingerprintSettings
|
||||
longer.ChromaprintLengthSec = 90
|
||||
if _, err := settings.Set(ctx, longer); err != nil {
|
||||
t.Fatalf("change length: %v", err)
|
||||
}
|
||||
touch(a, 5*time.Hour)
|
||||
scan("new-length scan")
|
||||
got, ok = stored(a)
|
||||
if !ok || got.length != 90 || lastOpts.lengthSec != 90 || !lastOpts.chromaprint {
|
||||
t.Fatalf("new-length scan stored %+v (present %v) after asking for %+v; want a chromaprint at 90s", got, ok, lastOpts)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,163 @@
|
||||
package library
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
|
||||
"git.fabledsword.com/bvandeusen/minstrel/internal/db/dbq"
|
||||
)
|
||||
|
||||
// Fingerprinting settings (M400 #3913).
|
||||
//
|
||||
// Rule 25: what an operator might tune lives in the database and changes without
|
||||
// a restart. One service instance is shared by the scanner, the backfill, the
|
||||
// duplicate sweep and the admin API, so a save reaches all of them at once.
|
||||
//
|
||||
// The length is the dangerous knob: chromaprints taken at two lengths cannot be
|
||||
// compared. Rather than bumping fingerprintVersion when it changes, every row
|
||||
// records the length it was taken at (migration 0061) and every reader filters
|
||||
// on the current one. A row at another length is stale to the backfill, pending
|
||||
// to the gauge and invisible to the sweep, which is the effect a version bump
|
||||
// would have, with one difference worth having: setting the length back makes
|
||||
// rows not yet re-derived current again, instead of redoing the library twice.
|
||||
|
||||
// Bounds for each setting, mirrored by migration 0061's CHECKs.
|
||||
const (
|
||||
// Below 30s a print has too few items to align at the matcher's larger
|
||||
// offsets and still overlap by minOverlapItems. Above 600s each print passes
|
||||
// 20 KB and a candidate page of them grows past what one sweep should hold.
|
||||
minChromaprintLengthSec = 30
|
||||
maxChromaprintLengthSec = 600
|
||||
// Unrelated audio sits near 0.5; a ceiling of 0.35 keeps a loosened
|
||||
// threshold well clear of proposing noise.
|
||||
minAcousticMaxBitErrorRate = 0.01
|
||||
maxAcousticMaxBitErrorRate = 0.35
|
||||
minBackfillConcurrency = 1
|
||||
maxBackfillConcurrency = 8
|
||||
minSweepIntervalHours = 1
|
||||
maxSweepIntervalHours = 168
|
||||
|
||||
defaultSweepIntervalHours = 1
|
||||
)
|
||||
|
||||
// FingerprintSettings mirrors the fingerprint_settings row.
|
||||
type FingerprintSettings struct {
|
||||
// Enabled off stops every decode: the scan takes only the stream hash (a
|
||||
// demux, and what recognises a moved file) and stores nothing, and the
|
||||
// backfill idles.
|
||||
Enabled bool
|
||||
ChromaprintLengthSec int32
|
||||
AcousticMaxBitErrorRate float64
|
||||
BackfillConcurrency int32
|
||||
SweepIntervalHours int32
|
||||
// UpdatedAt is when the settings were last saved. Set by the database;
|
||||
// ignored by Set.
|
||||
UpdatedAt time.Time
|
||||
}
|
||||
|
||||
// DefaultFingerprintSettings mirrors migration 0061's column defaults, so a
|
||||
// database that cannot be read still fingerprints the way a fresh install does.
|
||||
var DefaultFingerprintSettings = FingerprintSettings{
|
||||
Enabled: true,
|
||||
ChromaprintLengthSec: defaultChromaprintLengthSec,
|
||||
AcousticMaxBitErrorRate: defaultAcousticMaxBitErrorRate,
|
||||
BackfillConcurrency: fingerprintBackfillConcurrency,
|
||||
SweepIntervalHours: defaultSweepIntervalHours,
|
||||
}
|
||||
|
||||
// ErrFingerprintSettingOutOfRange is returned by Set for a value migration
|
||||
// 0061's CHECKs would reject, so the API answers 400 naming the field rather
|
||||
// than surfacing a constraint violation.
|
||||
var ErrFingerprintSettingOutOfRange = errors.New("fingerprint setting out of range")
|
||||
|
||||
// FingerprintSettingsService caches the settings and owns their persistence.
|
||||
// Cached because the scanner reads them for every file it fingerprints.
|
||||
type FingerprintSettingsService struct {
|
||||
pool *pgxpool.Pool
|
||||
|
||||
mu sync.RWMutex
|
||||
cur FingerprintSettings
|
||||
}
|
||||
|
||||
// NewFingerprintSettingsService loads once and caches. It always returns a
|
||||
// usable service, holding the defaults when the load fails; the error says so.
|
||||
func NewFingerprintSettingsService(ctx context.Context, pool *pgxpool.Pool) (*FingerprintSettingsService, error) {
|
||||
s := &FingerprintSettingsService{pool: pool, cur: DefaultFingerprintSettings}
|
||||
row, err := dbq.New(pool).GetFingerprintSettings(ctx)
|
||||
if err != nil {
|
||||
return s, fmt.Errorf("fingerprint settings: load: %w", err)
|
||||
}
|
||||
s.cur = fingerprintSettingsFromRow(row)
|
||||
return s, nil
|
||||
}
|
||||
|
||||
// Get returns the cached settings. A nil service answers with the defaults, so
|
||||
// a Scanner or worker built without one fingerprints as a fresh install would.
|
||||
func (s *FingerprintSettingsService) Get() FingerprintSettings {
|
||||
if s == nil {
|
||||
return DefaultFingerprintSettings
|
||||
}
|
||||
s.mu.RLock()
|
||||
defer s.mu.RUnlock()
|
||||
return s.cur
|
||||
}
|
||||
|
||||
// Set validates, persists and re-caches.
|
||||
func (s *FingerprintSettingsService) Set(ctx context.Context, in FingerprintSettings) (FingerprintSettings, error) {
|
||||
if err := validateFingerprintSettings(in); err != nil {
|
||||
return FingerprintSettings{}, err
|
||||
}
|
||||
if s == nil {
|
||||
return FingerprintSettings{}, errors.New("fingerprint settings: no settings service")
|
||||
}
|
||||
row, err := dbq.New(s.pool).UpdateFingerprintSettings(ctx, dbq.UpdateFingerprintSettingsParams{
|
||||
Enabled: in.Enabled,
|
||||
ChromaprintLengthSec: in.ChromaprintLengthSec,
|
||||
AcousticMaxBitErrorRate: in.AcousticMaxBitErrorRate,
|
||||
BackfillConcurrency: in.BackfillConcurrency,
|
||||
SweepIntervalHours: in.SweepIntervalHours,
|
||||
})
|
||||
if err != nil {
|
||||
return FingerprintSettings{}, fmt.Errorf("fingerprint settings: save: %w", err)
|
||||
}
|
||||
out := fingerprintSettingsFromRow(row)
|
||||
s.mu.Lock()
|
||||
s.cur = out
|
||||
s.mu.Unlock()
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func validateFingerprintSettings(in FingerprintSettings) error {
|
||||
switch {
|
||||
case in.ChromaprintLengthSec < minChromaprintLengthSec || in.ChromaprintLengthSec > maxChromaprintLengthSec:
|
||||
return fmt.Errorf("%w: chromaprint_length_sec must be %d-%d",
|
||||
ErrFingerprintSettingOutOfRange, minChromaprintLengthSec, maxChromaprintLengthSec)
|
||||
// Written as a negated range so NaN, which fails every comparison, is refused.
|
||||
case !(in.AcousticMaxBitErrorRate >= minAcousticMaxBitErrorRate && in.AcousticMaxBitErrorRate <= maxAcousticMaxBitErrorRate):
|
||||
return fmt.Errorf("%w: acoustic_max_bit_error_rate must be %.2f-%.2f",
|
||||
ErrFingerprintSettingOutOfRange, minAcousticMaxBitErrorRate, maxAcousticMaxBitErrorRate)
|
||||
case in.BackfillConcurrency < minBackfillConcurrency || in.BackfillConcurrency > maxBackfillConcurrency:
|
||||
return fmt.Errorf("%w: backfill_concurrency must be %d-%d",
|
||||
ErrFingerprintSettingOutOfRange, minBackfillConcurrency, maxBackfillConcurrency)
|
||||
case in.SweepIntervalHours < minSweepIntervalHours || in.SweepIntervalHours > maxSweepIntervalHours:
|
||||
return fmt.Errorf("%w: sweep_interval_hours must be %d-%d",
|
||||
ErrFingerprintSettingOutOfRange, minSweepIntervalHours, maxSweepIntervalHours)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func fingerprintSettingsFromRow(row dbq.FingerprintSetting) FingerprintSettings {
|
||||
return FingerprintSettings{
|
||||
Enabled: row.Enabled,
|
||||
ChromaprintLengthSec: row.ChromaprintLengthSec,
|
||||
AcousticMaxBitErrorRate: row.AcousticMaxBitErrorRate,
|
||||
BackfillConcurrency: row.BackfillConcurrency,
|
||||
SweepIntervalHours: row.SweepIntervalHours,
|
||||
UpdatedAt: row.UpdatedAt.Time,
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,146 @@
|
||||
package library
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"math"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestValidateFingerprintSettings(t *testing.T) {
|
||||
with := func(edit func(*FingerprintSettings)) FingerprintSettings {
|
||||
s := DefaultFingerprintSettings
|
||||
edit(&s)
|
||||
return s
|
||||
}
|
||||
valid := map[string]FingerprintSettings{
|
||||
"defaults": DefaultFingerprintSettings,
|
||||
"shortest length": with(func(s *FingerprintSettings) { s.ChromaprintLengthSec = minChromaprintLengthSec }),
|
||||
"longest length": with(func(s *FingerprintSettings) { s.ChromaprintLengthSec = maxChromaprintLengthSec }),
|
||||
"strictest match": with(func(s *FingerprintSettings) { s.AcousticMaxBitErrorRate = minAcousticMaxBitErrorRate }),
|
||||
"loosest match": with(func(s *FingerprintSettings) { s.AcousticMaxBitErrorRate = maxAcousticMaxBitErrorRate }),
|
||||
"fewest at once": with(func(s *FingerprintSettings) { s.BackfillConcurrency = minBackfillConcurrency }),
|
||||
"most at once": with(func(s *FingerprintSettings) { s.BackfillConcurrency = maxBackfillConcurrency }),
|
||||
"shortest interval": with(func(s *FingerprintSettings) { s.SweepIntervalHours = minSweepIntervalHours }),
|
||||
"longest interval": with(func(s *FingerprintSettings) { s.SweepIntervalHours = maxSweepIntervalHours }),
|
||||
"switched off": with(func(s *FingerprintSettings) { s.Enabled = false }),
|
||||
}
|
||||
for name, s := range valid {
|
||||
if err := validateFingerprintSettings(s); err != nil {
|
||||
t.Errorf("%s: rejected: %v", name, err)
|
||||
}
|
||||
}
|
||||
invalid := map[string]FingerprintSettings{
|
||||
"length too short": with(func(s *FingerprintSettings) { s.ChromaprintLengthSec = minChromaprintLengthSec - 1 }),
|
||||
"length too long": with(func(s *FingerprintSettings) { s.ChromaprintLengthSec = maxChromaprintLengthSec + 1 }),
|
||||
"match too strict": with(func(s *FingerprintSettings) { s.AcousticMaxBitErrorRate = minAcousticMaxBitErrorRate - 0.001 }),
|
||||
"match too loose": with(func(s *FingerprintSettings) { s.AcousticMaxBitErrorRate = maxAcousticMaxBitErrorRate + 0.001 }),
|
||||
"match not a number": with(func(s *FingerprintSettings) { s.AcousticMaxBitErrorRate = math.NaN() }),
|
||||
"none at once": with(func(s *FingerprintSettings) { s.BackfillConcurrency = 0 }),
|
||||
"too many at once": with(func(s *FingerprintSettings) { s.BackfillConcurrency = maxBackfillConcurrency + 1 }),
|
||||
"no interval": with(func(s *FingerprintSettings) { s.SweepIntervalHours = 0 }),
|
||||
"interval too long": with(func(s *FingerprintSettings) { s.SweepIntervalHours = maxSweepIntervalHours + 1 }),
|
||||
}
|
||||
for name, s := range invalid {
|
||||
if err := validateFingerprintSettings(s); !errors.Is(err, ErrFingerprintSettingOutOfRange) {
|
||||
t.Errorf("%s: err = %v, want ErrFingerprintSettingOutOfRange", name, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestFingerprintSettingsService_NilServesDefaults(t *testing.T) {
|
||||
var s *FingerprintSettingsService
|
||||
if got := s.Get(); got != DefaultFingerprintSettings {
|
||||
t.Fatalf("nil service Get = %+v, want the defaults", got)
|
||||
}
|
||||
// Validation still runs first, so a bad value is named rather than hidden
|
||||
// behind the missing service.
|
||||
bad := DefaultFingerprintSettings
|
||||
bad.BackfillConcurrency = 0
|
||||
if _, err := s.Set(context.Background(), bad); !errors.Is(err, ErrFingerprintSettingOutOfRange) {
|
||||
t.Fatalf("nil service Set of a bad value: err = %v, want ErrFingerprintSettingOutOfRange", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFingerprintSettingsService_Integration(t *testing.T) {
|
||||
pool := newPool(t)
|
||||
ctx := context.Background()
|
||||
reload := func(step string) FingerprintSettings {
|
||||
t.Helper()
|
||||
fresh, err := NewFingerprintSettingsService(ctx, pool)
|
||||
if err != nil {
|
||||
t.Fatalf("%s: load: %v", step, err)
|
||||
}
|
||||
return fresh.Get()
|
||||
}
|
||||
withoutTime := func(s FingerprintSettings) FingerprintSettings {
|
||||
s.UpdatedAt = time.Time{}
|
||||
return s
|
||||
}
|
||||
|
||||
svc, err := NewFingerprintSettingsService(ctx, pool)
|
||||
if err != nil {
|
||||
t.Fatalf("load: %v", err)
|
||||
}
|
||||
// ResetDB puts every column back to its migration default, so this pins the
|
||||
// Go defaults to migration 0061's. Were they to drift, a database that could
|
||||
// not be read would fingerprint differently from one that could.
|
||||
loaded := svc.Get()
|
||||
if loaded.UpdatedAt.IsZero() {
|
||||
t.Fatal("loaded settings carry no updated_at")
|
||||
}
|
||||
if withoutTime(loaded) != DefaultFingerprintSettings {
|
||||
t.Fatalf("stored defaults = %+v, want the Go defaults %+v", withoutTime(loaded), DefaultFingerprintSettings)
|
||||
}
|
||||
|
||||
// Every bound the service accepts, the table accepts too. A CHECK tighter
|
||||
// than validate would turn a value the card allows into a 500.
|
||||
lowest := FingerprintSettings{
|
||||
Enabled: false,
|
||||
ChromaprintLengthSec: minChromaprintLengthSec,
|
||||
AcousticMaxBitErrorRate: minAcousticMaxBitErrorRate,
|
||||
BackfillConcurrency: minBackfillConcurrency,
|
||||
SweepIntervalHours: minSweepIntervalHours,
|
||||
}
|
||||
highest := FingerprintSettings{
|
||||
Enabled: true,
|
||||
ChromaprintLengthSec: maxChromaprintLengthSec,
|
||||
AcousticMaxBitErrorRate: maxAcousticMaxBitErrorRate,
|
||||
BackfillConcurrency: maxBackfillConcurrency,
|
||||
SweepIntervalHours: maxSweepIntervalHours,
|
||||
}
|
||||
for _, step := range []struct {
|
||||
name string
|
||||
want FingerprintSettings
|
||||
}{{"lowest", lowest}, {"highest", highest}} {
|
||||
saved, err := svc.Set(ctx, step.want)
|
||||
if err != nil {
|
||||
t.Fatalf("%s: save: %v", step.name, err)
|
||||
}
|
||||
// A save must move updated_at forward: it is what makes a sweep due.
|
||||
if !saved.UpdatedAt.After(loaded.UpdatedAt) {
|
||||
t.Fatalf("%s: updated_at %v did not move past %v", step.name, saved.UpdatedAt, loaded.UpdatedAt)
|
||||
}
|
||||
if withoutTime(saved) != step.want || withoutTime(svc.Get()) != step.want {
|
||||
t.Fatalf("%s: saved %+v, cached %+v, want %+v", step.name, saved, svc.Get(), step.want)
|
||||
}
|
||||
if got := withoutTime(reload(step.name)); got != step.want {
|
||||
t.Fatalf("%s: table holds %+v, want %+v", step.name, got, step.want)
|
||||
}
|
||||
}
|
||||
|
||||
// An out-of-range save changes nothing, in the cache or the table.
|
||||
before := svc.Get()
|
||||
bad := highest
|
||||
bad.ChromaprintLengthSec = maxChromaprintLengthSec + 1
|
||||
if _, err := svc.Set(ctx, bad); !errors.Is(err, ErrFingerprintSettingOutOfRange) {
|
||||
t.Fatalf("out-of-range save: err = %v, want ErrFingerprintSettingOutOfRange", err)
|
||||
}
|
||||
if svc.Get() != before {
|
||||
t.Fatalf("out-of-range save changed the cache to %+v", svc.Get())
|
||||
}
|
||||
if got := reload("after out-of-range save"); got != before {
|
||||
t.Fatalf("out-of-range save changed the table to %+v", got)
|
||||
}
|
||||
}
|
||||
@@ -79,11 +79,14 @@ type Scanner struct {
|
||||
// integration test can substitute a deterministic one: CI has no real audio
|
||||
// to fingerprint, and what the test pins is WHEN the scan fingerprints, not
|
||||
// what the tools print. Call it through fingerprintFile.
|
||||
fingerprint func(ctx context.Context, path string) fingerprintResult
|
||||
fingerprint func(ctx context.Context, path string, opts fingerprintOptions) fingerprintResult
|
||||
// settings is the operator's fingerprinting policy (#3913), shared with the
|
||||
// workers and the admin API. Nil fingerprints with the defaults.
|
||||
settings *FingerprintSettingsService
|
||||
}
|
||||
|
||||
func New(pool *pgxpool.Pool, logger *slog.Logger, paths []string) *Scanner {
|
||||
return &Scanner{pool: pool, logger: logger, paths: paths, fingerprint: computeFingerprint}
|
||||
func New(pool *pgxpool.Pool, logger *slog.Logger, paths []string, settings *FingerprintSettingsService) *Scanner {
|
||||
return &Scanner{pool: pool, logger: logger, paths: paths, fingerprint: computeFingerprint, settings: settings}
|
||||
}
|
||||
|
||||
// Scan walks every configured root and upserts any audio file whose mtime is
|
||||
@@ -313,10 +316,18 @@ func (s *Scanner) scanFile(
|
||||
//
|
||||
// Computed before move adoption so adoption can match on the audio hash
|
||||
// (#3914); stored after the upsert, once the row id is known.
|
||||
//
|
||||
// With fingerprinting switched off (#3913) the scan decodes nothing, but it
|
||||
// still takes the stream hash: a demux rather than a decode, and the only
|
||||
// thing that recognises a moved file with no MBID.
|
||||
var fp fingerprintResult
|
||||
fpCfg := s.settings.Get()
|
||||
fingerprinted := !unchanged
|
||||
if fingerprinted {
|
||||
fp = s.fingerprintFile(ctx, path)
|
||||
fp = s.fingerprintFile(ctx, path, fingerprintOptions{
|
||||
lengthSec: fpCfg.ChromaprintLengthSec,
|
||||
chromaprint: fpCfg.Enabled,
|
||||
})
|
||||
}
|
||||
|
||||
// A path we've never seen might not be a new track — it might be one that
|
||||
@@ -384,7 +395,13 @@ func (s *Scanner) scanFile(
|
||||
s.logger.Warn("library scan: LogChange track upsert failed", "track_id", track.ID, "err", err)
|
||||
}
|
||||
if fingerprinted {
|
||||
storeFingerprint(ctx, q, s.logger, track.ID, path, fp)
|
||||
if fpCfg.Enabled {
|
||||
storeFingerprint(ctx, q, s.logger, track.ID, path, fp, fpCfg.ChromaprintLengthSec)
|
||||
} else if err := q.DeleteTrackFingerprint(ctx, track.ID); err != nil {
|
||||
// Off, nothing is stored — but a row describing the previous bytes
|
||||
// must not outlive them, or the sweep would compare audio that is gone.
|
||||
s.logger.Warn("fingerprint: clearing stale fingerprint failed", "path", path, "err", err)
|
||||
}
|
||||
}
|
||||
|
||||
if knownTrack {
|
||||
|
||||
@@ -81,7 +81,7 @@ func TestScanner_Integration(t *testing.T) {
|
||||
"TIT2": "Solo", "TPE1": "The Artist Y", "TALB": "Y Album", "TRCK": "1",
|
||||
})
|
||||
|
||||
scanner := New(pool, logger, []string{root})
|
||||
scanner := New(pool, logger, []string{root}, nil)
|
||||
stats, err := scanner.Scan(ctx, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("first scan: %v", err)
|
||||
@@ -259,7 +259,7 @@ func TestScanner_AdoptsMovedFile_Integration(t *testing.T) {
|
||||
})
|
||||
}
|
||||
|
||||
scanner := New(pool, logger, []string{root})
|
||||
scanner := New(pool, logger, []string{root}, nil)
|
||||
if _, err := scanner.Scan(ctx, nil); err != nil {
|
||||
t.Fatalf("first scan: %v", err)
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user