feat(admin): fingerprinting settings — on/off, length, match threshold, concurrency, sweep interval (M400 #3913)
test-go / test (push) Failing after 44s
test-web / test (push) Successful in 49s
test-go / integration (push) Failing after 2m42s
release / Build + push container image (push) Canceled after 0s
release / Verify release artifacts (tag releases only) (push) Canceled after 0s
release / Build signed APK (releases and dev) (push) Canceled after 4m8s

Rule 25: the fingerprinting knobs move out of source into a DB-backed
singleton (migration 0061), edited from a card on the Duplicates page and
shared live with the scanner, the backfill and the duplicate sweep through
one service instance, so a save needs no restart.

The length is the knob that can silently break the library: prints taken
at two lengths never match. Each track_fingerprints row now records the
length it was taken at, and every reader filters on the current one — the
backfill treats another length as stale, the gauge counts it pending, the
sweep never streams it. Equivalent to a version bump, except that setting
the length back makes rows not yet redone current again. The card warns
before a length change re-fingerprints the library.

Off stops every decode: the scan takes only the stream hash (a demux, and
what recognises a moved file) and stores nothing, dropping a changed file's
stale row; the backfill idles. A save also makes a sweep due, since a new
threshold or length changes what the same prints group into, and the sweep
interval gains slack so an hourly interval on an hourly tick doesn't skip
every other tick.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01SQ31KQpYbStyK5y58UmPLH
This commit is contained in:
2026-09-11 17:53:55 -04:00
co-authored by Claude Opus 5
parent c8bf9dc929
commit 077ae61235
38 changed files with 1679 additions and 201 deletions
+48 -23
View File
@@ -50,21 +50,28 @@ const fingerprintTimeout = 60 * time.Second
const fingerprintWaitDelay = 5 * time.Second
// fingerprintVersion stamps how a track_fingerprints row was derived. Bump it
// whenever the derivation changes — the hash arguments, fpcalc's flags or its
// length — and the backfill re-derives every row below it. Fingerprints taken
// by two methods are not comparable, and nothing else would reveal that the
// library held a mix.
// whenever the derivation changes — the hash arguments or fpcalc's flags — and
// the backfill re-derives every row below it. Fingerprints taken by two methods
// are not comparable, and nothing else would reveal that the library held a mix.
//
// The length is deliberately not part of it: it is an operator setting (#3913),
// so each row records the length it was taken at and readers compare only rows
// at the current one. See fingerprint_settings.go.
const fingerprintVersion int16 = 1
// errFingerprintTimeout marks a tool that ran out of time. Distinct from a
// failed exit because a stall is a fact about the mount, not about the file.
var errFingerprintTimeout = errors.New("fingerprint tool timed out")
// defaultChromaprintLengthSec is how many seconds of audio fpcalc fingerprints.
// 120 is fpcalc's own default. Fingerprints taken at different lengths are not
// comparable, so changing this has to re-derive every stored one.
// defaultChromaprintLengthSec is the shipped value of the length setting (#3913):
// how many seconds of audio fpcalc fingerprints. 120 is fpcalc's own default.
const defaultChromaprintLengthSec = 120
// errChromaprintSkipped marks a chromaprint not taken because fingerprinting is
// switched off. Inconclusive rather than a verdict: nothing was learned about
// the file.
var errChromaprintSkipped = errors.New("chromaprint skipped: fingerprinting is off")
// fpcalcStderrTail caps how much of a failing tool's stderr reaches the log.
const fpcalcStderrTail = 512
@@ -113,11 +120,24 @@ type fingerprintResult struct {
printErr error
}
// computeFingerprint derives both halves for the file at path.
func computeFingerprint(ctx context.Context, path string) fingerprintResult {
// fingerprintOptions is what the settings decide for one attempt. Captured once
// per file, so the length a chromaprint was taken at is the length stored with it
// even if the setting changes mid-attempt.
type fingerprintOptions struct {
lengthSec int32
// chromaprint false takes the stream hash alone: a demux, no decode.
chromaprint bool
}
// computeFingerprint derives the halves opts asks for, for the file at path.
func computeFingerprint(ctx context.Context, path string, opts fingerprintOptions) fingerprintResult {
var r fingerprintResult
r.streamSHA256, r.hashErr = computeAudioStreamSHA256(ctx, path)
r.chromaprint, r.printErr = computeChromaprint(ctx, path, defaultChromaprintLengthSec)
if !opts.chromaprint {
r.printErr = errChromaprintSkipped
return r
}
r.chromaprint, r.printErr = computeChromaprint(ctx, path, opts.lengthSec)
return r
}
@@ -130,11 +150,13 @@ func (r fingerprintResult) inconclusive() bool {
}
// isInconclusive names the failures that are not a verdict on the file: a
// stall, a cancelled scan, and a tool that is not installed. The last matters
// outside the image — a dev binary run without fpcalc on PATH must not stamp
// every track in the library as unfingerprintable.
// stall, a cancelled scan, a tool that is not installed, and a chromaprint
// skipped because fingerprinting is off. A missing tool matters outside the
// image — a dev binary run without fpcalc on PATH must not stamp every track in
// the library as unfingerprintable.
func isInconclusive(err error) bool {
return errors.Is(err, errFingerprintTimeout) ||
errors.Is(err, errChromaprintSkipped) ||
errors.Is(err, context.Canceled) ||
errors.Is(err, context.DeadlineExceeded) ||
errors.Is(err, exec.ErrNotFound)
@@ -142,11 +164,11 @@ func isInconclusive(err error) bool {
// fingerprintFile runs the scanner's fingerprinter. A Scanner built without New
// gets the real tools rather than a nil-func panic halfway through a scan.
func (s *Scanner) fingerprintFile(ctx context.Context, path string) fingerprintResult {
func (s *Scanner) fingerprintFile(ctx context.Context, path string, opts fingerprintOptions) fingerprintResult {
if s.fingerprint == nil {
return computeFingerprint(ctx, path)
return computeFingerprint(ctx, path, opts)
}
return s.fingerprint(ctx, path)
return s.fingerprint(ctx, path, opts)
}
// fingerprintOutcome is what storeFingerprint did with one attempt.
@@ -163,9 +185,11 @@ const (
// the backfill (#3908) alike, so there is one rule for what gets written. It
// never fails its caller: a missing fingerprint only keeps a track out of
// duplicate detection, which is not worth dropping a scan or a pass over.
//
// lengthSec is the length fp's chromaprint was taken at, stored with it (#3913).
func storeFingerprint(
ctx context.Context, q *dbq.Queries, logger *slog.Logger,
trackID pgtype.UUID, path string, fp fingerprintResult,
trackID pgtype.UUID, path string, fp fingerprintResult, lengthSec int32,
) fingerprintOutcome {
if fp.hashErr != nil {
logger.Warn("fingerprint: audio stream hash failed", "path", path, "err", fp.hashErr)
@@ -186,10 +210,11 @@ func storeFingerprint(
// is stamped at the current version so the backfill does not retry it on
// every pass. It is retried when the file changes.
if err := q.UpsertTrackFingerprint(ctx, dbq.UpsertTrackFingerprintParams{
TrackID: trackID,
AudioStreamSha256: fp.streamSHA256,
Chromaprint: fp.chromaprint,
FingerprintVersion: fingerprintVersion,
TrackID: trackID,
AudioStreamSha256: fp.streamSHA256,
Chromaprint: fp.chromaprint,
FingerprintVersion: fingerprintVersion,
ChromaprintLengthSec: lengthSec,
}); err != nil {
logger.Warn("fingerprint: storing fingerprint failed", "path", path, "err", err)
return outcomeStoreFailed
@@ -211,8 +236,8 @@ func computeAudioStreamSHA256(ctx context.Context, path string) ([]byte, error)
// computeChromaprint returns the raw acoustic fingerprint of the first
// lengthSec seconds of the file.
func computeChromaprint(ctx context.Context, path string, lengthSec int) ([]int32, error) {
out, err := runFingerprintTool(ctx, "fpcalc", fpcalcArgs(path, lengthSec))
func computeChromaprint(ctx context.Context, path string, lengthSec int32) ([]int32, error) {
out, err := runFingerprintTool(ctx, "fpcalc", fpcalcArgs(path, int(lengthSec)))
if err != nil {
return nil, err
}