feat(admin): fingerprinting settings — on/off, length, match threshold, concurrency, sweep interval (M400 #3913)
test-go / test (push) Failing after 44s
test-web / test (push) Successful in 49s
test-go / integration (push) Failing after 2m42s
release / Build + push container image (push) Canceled after 0s
release / Verify release artifacts (tag releases only) (push) Canceled after 0s
release / Build signed APK (releases and dev) (push) Canceled after 4m8s
test-go / test (push) Failing after 44s
test-web / test (push) Successful in 49s
test-go / integration (push) Failing after 2m42s
release / Build + push container image (push) Canceled after 0s
release / Verify release artifacts (tag releases only) (push) Canceled after 0s
release / Build signed APK (releases and dev) (push) Canceled after 4m8s
Rule 25: the fingerprinting knobs move out of source into a DB-backed singleton (migration 0061), edited from a card on the Duplicates page and shared live with the scanner, the backfill and the duplicate sweep through one service instance, so a save needs no restart. The length is the knob that can silently break the library: prints taken at two lengths never match. Each track_fingerprints row now records the length it was taken at, and every reader filters on the current one — the backfill treats another length as stale, the gauge counts it pending, the sweep never streams it. Equivalent to a version bump, except that setting the length back makes rows not yet redone current again. The card warns before a length change re-fingerprints the library. Off stops every decode: the scan takes only the stream hash (a demux, and what recognises a moved file) and stores nothing, dropping a changed file's stale row; the backfill idles. A save also makes a sweep due, since a new threshold or length changes what the same prints group into, and the sweep interval gains slack so an hourly interval on an hourly tick doesn't skip every other tick. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01SQ31KQpYbStyK5y58UmPLH
This commit is contained in:
@@ -50,21 +50,28 @@ const fingerprintTimeout = 60 * time.Second
|
||||
const fingerprintWaitDelay = 5 * time.Second
|
||||
|
||||
// fingerprintVersion stamps how a track_fingerprints row was derived. Bump it
|
||||
// whenever the derivation changes — the hash arguments, fpcalc's flags or its
|
||||
// length — and the backfill re-derives every row below it. Fingerprints taken
|
||||
// by two methods are not comparable, and nothing else would reveal that the
|
||||
// library held a mix.
|
||||
// whenever the derivation changes — the hash arguments or fpcalc's flags — and
|
||||
// the backfill re-derives every row below it. Fingerprints taken by two methods
|
||||
// are not comparable, and nothing else would reveal that the library held a mix.
|
||||
//
|
||||
// The length is deliberately not part of it: it is an operator setting (#3913),
|
||||
// so each row records the length it was taken at and readers compare only rows
|
||||
// at the current one. See fingerprint_settings.go.
|
||||
const fingerprintVersion int16 = 1
|
||||
|
||||
// errFingerprintTimeout marks a tool that ran out of time. Distinct from a
|
||||
// failed exit because a stall is a fact about the mount, not about the file.
|
||||
var errFingerprintTimeout = errors.New("fingerprint tool timed out")
|
||||
|
||||
// defaultChromaprintLengthSec is how many seconds of audio fpcalc fingerprints.
|
||||
// 120 is fpcalc's own default. Fingerprints taken at different lengths are not
|
||||
// comparable, so changing this has to re-derive every stored one.
|
||||
// defaultChromaprintLengthSec is the shipped value of the length setting (#3913):
|
||||
// how many seconds of audio fpcalc fingerprints. 120 is fpcalc's own default.
|
||||
const defaultChromaprintLengthSec = 120
|
||||
|
||||
// errChromaprintSkipped marks a chromaprint not taken because fingerprinting is
|
||||
// switched off. Inconclusive rather than a verdict: nothing was learned about
|
||||
// the file.
|
||||
var errChromaprintSkipped = errors.New("chromaprint skipped: fingerprinting is off")
|
||||
|
||||
// fpcalcStderrTail caps how much of a failing tool's stderr reaches the log.
|
||||
const fpcalcStderrTail = 512
|
||||
|
||||
@@ -113,11 +120,24 @@ type fingerprintResult struct {
|
||||
printErr error
|
||||
}
|
||||
|
||||
// computeFingerprint derives both halves for the file at path.
|
||||
func computeFingerprint(ctx context.Context, path string) fingerprintResult {
|
||||
// fingerprintOptions is what the settings decide for one attempt. Captured once
|
||||
// per file, so the length a chromaprint was taken at is the length stored with it
|
||||
// even if the setting changes mid-attempt.
|
||||
type fingerprintOptions struct {
|
||||
lengthSec int32
|
||||
// chromaprint false takes the stream hash alone: a demux, no decode.
|
||||
chromaprint bool
|
||||
}
|
||||
|
||||
// computeFingerprint derives the halves opts asks for, for the file at path.
|
||||
func computeFingerprint(ctx context.Context, path string, opts fingerprintOptions) fingerprintResult {
|
||||
var r fingerprintResult
|
||||
r.streamSHA256, r.hashErr = computeAudioStreamSHA256(ctx, path)
|
||||
r.chromaprint, r.printErr = computeChromaprint(ctx, path, defaultChromaprintLengthSec)
|
||||
if !opts.chromaprint {
|
||||
r.printErr = errChromaprintSkipped
|
||||
return r
|
||||
}
|
||||
r.chromaprint, r.printErr = computeChromaprint(ctx, path, opts.lengthSec)
|
||||
return r
|
||||
}
|
||||
|
||||
@@ -130,11 +150,13 @@ func (r fingerprintResult) inconclusive() bool {
|
||||
}
|
||||
|
||||
// isInconclusive names the failures that are not a verdict on the file: a
|
||||
// stall, a cancelled scan, and a tool that is not installed. The last matters
|
||||
// outside the image — a dev binary run without fpcalc on PATH must not stamp
|
||||
// every track in the library as unfingerprintable.
|
||||
// stall, a cancelled scan, a tool that is not installed, and a chromaprint
|
||||
// skipped because fingerprinting is off. A missing tool matters outside the
|
||||
// image — a dev binary run without fpcalc on PATH must not stamp every track in
|
||||
// the library as unfingerprintable.
|
||||
func isInconclusive(err error) bool {
|
||||
return errors.Is(err, errFingerprintTimeout) ||
|
||||
errors.Is(err, errChromaprintSkipped) ||
|
||||
errors.Is(err, context.Canceled) ||
|
||||
errors.Is(err, context.DeadlineExceeded) ||
|
||||
errors.Is(err, exec.ErrNotFound)
|
||||
@@ -142,11 +164,11 @@ func isInconclusive(err error) bool {
|
||||
|
||||
// fingerprintFile runs the scanner's fingerprinter. A Scanner built without New
|
||||
// gets the real tools rather than a nil-func panic halfway through a scan.
|
||||
func (s *Scanner) fingerprintFile(ctx context.Context, path string) fingerprintResult {
|
||||
func (s *Scanner) fingerprintFile(ctx context.Context, path string, opts fingerprintOptions) fingerprintResult {
|
||||
if s.fingerprint == nil {
|
||||
return computeFingerprint(ctx, path)
|
||||
return computeFingerprint(ctx, path, opts)
|
||||
}
|
||||
return s.fingerprint(ctx, path)
|
||||
return s.fingerprint(ctx, path, opts)
|
||||
}
|
||||
|
||||
// fingerprintOutcome is what storeFingerprint did with one attempt.
|
||||
@@ -163,9 +185,11 @@ const (
|
||||
// the backfill (#3908) alike, so there is one rule for what gets written. It
|
||||
// never fails its caller: a missing fingerprint only keeps a track out of
|
||||
// duplicate detection, which is not worth dropping a scan or a pass over.
|
||||
//
|
||||
// lengthSec is the length fp's chromaprint was taken at, stored with it (#3913).
|
||||
func storeFingerprint(
|
||||
ctx context.Context, q *dbq.Queries, logger *slog.Logger,
|
||||
trackID pgtype.UUID, path string, fp fingerprintResult,
|
||||
trackID pgtype.UUID, path string, fp fingerprintResult, lengthSec int32,
|
||||
) fingerprintOutcome {
|
||||
if fp.hashErr != nil {
|
||||
logger.Warn("fingerprint: audio stream hash failed", "path", path, "err", fp.hashErr)
|
||||
@@ -186,10 +210,11 @@ func storeFingerprint(
|
||||
// is stamped at the current version so the backfill does not retry it on
|
||||
// every pass. It is retried when the file changes.
|
||||
if err := q.UpsertTrackFingerprint(ctx, dbq.UpsertTrackFingerprintParams{
|
||||
TrackID: trackID,
|
||||
AudioStreamSha256: fp.streamSHA256,
|
||||
Chromaprint: fp.chromaprint,
|
||||
FingerprintVersion: fingerprintVersion,
|
||||
TrackID: trackID,
|
||||
AudioStreamSha256: fp.streamSHA256,
|
||||
Chromaprint: fp.chromaprint,
|
||||
FingerprintVersion: fingerprintVersion,
|
||||
ChromaprintLengthSec: lengthSec,
|
||||
}); err != nil {
|
||||
logger.Warn("fingerprint: storing fingerprint failed", "path", path, "err", err)
|
||||
return outcomeStoreFailed
|
||||
@@ -211,8 +236,8 @@ func computeAudioStreamSHA256(ctx context.Context, path string) ([]byte, error)
|
||||
|
||||
// computeChromaprint returns the raw acoustic fingerprint of the first
|
||||
// lengthSec seconds of the file.
|
||||
func computeChromaprint(ctx context.Context, path string, lengthSec int) ([]int32, error) {
|
||||
out, err := runFingerprintTool(ctx, "fpcalc", fpcalcArgs(path, lengthSec))
|
||||
func computeChromaprint(ctx context.Context, path string, lengthSec int32) ([]int32, error) {
|
||||
out, err := runFingerprintTool(ctx, "fpcalc", fpcalcArgs(path, int(lengthSec)))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user