feat(admin): fingerprinting settings — on/off, length, match threshold, concurrency, sweep interval (M400 #3913)
test-go / test (push) Failing after 44s
test-web / test (push) Successful in 49s
test-go / integration (push) Failing after 2m42s
release / Build + push container image (push) Canceled after 0s
release / Verify release artifacts (tag releases only) (push) Canceled after 0s
release / Build signed APK (releases and dev) (push) Canceled after 4m8s
test-go / test (push) Failing after 44s
test-web / test (push) Successful in 49s
test-go / integration (push) Failing after 2m42s
release / Build + push container image (push) Canceled after 0s
release / Verify release artifacts (tag releases only) (push) Canceled after 0s
release / Build signed APK (releases and dev) (push) Canceled after 4m8s
Rule 25: the fingerprinting knobs move out of source into a DB-backed singleton (migration 0061), edited from a card on the Duplicates page and shared live with the scanner, the backfill and the duplicate sweep through one service instance, so a save needs no restart. The length is the knob that can silently break the library: prints taken at two lengths never match. Each track_fingerprints row now records the length it was taken at, and every reader filters on the current one — the backfill treats another length as stale, the gauge counts it pending, the sweep never streams it. Equivalent to a version bump, except that setting the length back makes rows not yet redone current again. The card warns before a length change re-fingerprints the library. Off stops every decode: the scan takes only the stream hash (a demux, and what recognises a moved file) and stores nothing, dropping a changed file's stale row; the backfill idles. A save also makes a sweep due, since a new threshold or length changes what the same prints group into, and the sweep interval gains slack so an hourly interval on an hourly tick doesn't skip every other tick. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01SQ31KQpYbStyK5y58UmPLH
This commit is contained in:
@@ -40,10 +40,11 @@ const fingerprintBackfillTick = time.Hour
|
||||
// so tracks the scan adds mid-pass are not stuck behind one enormous page.
|
||||
const fingerprintBackfillBatch = 50
|
||||
|
||||
// fingerprintBackfillConcurrency is how many files are decoded at once. Two is
|
||||
// deliberately low: fpcalc and the stream hash compete with playback transcoding
|
||||
// for CPU and with streaming for the mount, and a backfill that makes playback
|
||||
// stutter is worse than one that takes longer. Operator-tunable in #3913.
|
||||
// fingerprintBackfillConcurrency is the shipped value of the concurrency setting
|
||||
// (#3913): how many files are decoded at once. Two is deliberately low: fpcalc
|
||||
// and the stream hash compete with playback transcoding for CPU and with
|
||||
// streaming for the mount, and a backfill that makes playback stutter is worse
|
||||
// than one that takes longer.
|
||||
const fingerprintBackfillConcurrency = 2
|
||||
|
||||
// BackfillFingerprintsResult tallies one pass.
|
||||
@@ -68,24 +69,27 @@ func (r *BackfillFingerprintsResult) add(o fingerprintOutcome) {
|
||||
|
||||
// FingerprintBackfillWorker fingerprints the tracks the scan never will.
|
||||
type FingerprintBackfillWorker struct {
|
||||
pool *pgxpool.Pool
|
||||
logger *slog.Logger
|
||||
tick time.Duration
|
||||
batch int32
|
||||
concurrency int
|
||||
pool *pgxpool.Pool
|
||||
logger *slog.Logger
|
||||
settings *FingerprintSettingsService
|
||||
tick time.Duration
|
||||
batch int32
|
||||
// fingerprint is a field for the same reason as Scanner.fingerprint: an
|
||||
// integration test pins which tracks a pass touches, not what the tools print.
|
||||
fingerprint func(ctx context.Context, path string) fingerprintResult
|
||||
fingerprint func(ctx context.Context, path string, opts fingerprintOptions) fingerprintResult
|
||||
}
|
||||
|
||||
// NewFingerprintBackfillWorker builds a worker with the production cadence.
|
||||
func NewFingerprintBackfillWorker(pool *pgxpool.Pool, logger *slog.Logger) *FingerprintBackfillWorker {
|
||||
// settings is shared with the scanner and the admin API; nil runs on defaults.
|
||||
func NewFingerprintBackfillWorker(
|
||||
pool *pgxpool.Pool, logger *slog.Logger, settings *FingerprintSettingsService,
|
||||
) *FingerprintBackfillWorker {
|
||||
return &FingerprintBackfillWorker{
|
||||
pool: pool,
|
||||
logger: logger,
|
||||
settings: settings,
|
||||
tick: fingerprintBackfillTick,
|
||||
batch: fingerprintBackfillBatch,
|
||||
concurrency: fingerprintBackfillConcurrency,
|
||||
fingerprint: computeFingerprint,
|
||||
}
|
||||
}
|
||||
@@ -129,6 +133,12 @@ func (w *FingerprintBackfillWorker) runOnce(ctx context.Context) {
|
||||
// cursor is what lets a pass end: an inconclusive attempt writes no row, so a
|
||||
// file that keeps timing out would otherwise be listed again immediately and
|
||||
// retried forever within the pass.
|
||||
//
|
||||
// Settings are read before every batch, so a save takes effect within a batch
|
||||
// rather than an hour (#3913): switching fingerprinting off ends the pass, a new
|
||||
// concurrency applies to the next batch, and a new length restarts the walk from
|
||||
// the top at that length, because every row written at the old one went stale
|
||||
// the moment it changed.
|
||||
func (w *FingerprintBackfillWorker) pass(ctx context.Context) (BackfillFingerprintsResult, error) {
|
||||
q := dbq.New(w.pool)
|
||||
var (
|
||||
@@ -137,15 +147,25 @@ func (w *FingerprintBackfillWorker) pass(ctx context.Context) (BackfillFingerpri
|
||||
)
|
||||
// The all-zero uuid sorts before every real id. Valid must be true: a NULL
|
||||
// cursor would make "id > NULL" match nothing and every pass a silent no-op.
|
||||
after := pgtype.UUID{Valid: true}
|
||||
start := pgtype.UUID{Valid: true}
|
||||
after := start
|
||||
lengthSec := w.settings.Get().ChromaprintLengthSec
|
||||
for {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return res, err
|
||||
}
|
||||
cfg := w.settings.Get()
|
||||
if !cfg.Enabled {
|
||||
return res, nil
|
||||
}
|
||||
if cfg.ChromaprintLengthSec != lengthSec {
|
||||
lengthSec, after = cfg.ChromaprintLengthSec, start
|
||||
}
|
||||
rows, err := q.ListTracksNeedingFingerprint(ctx, dbq.ListTracksNeedingFingerprintParams{
|
||||
CurrentVersion: fingerprintVersion,
|
||||
AfterID: after,
|
||||
BatchLimit: w.batch,
|
||||
CurrentVersion: fingerprintVersion,
|
||||
ChromaprintLengthSec: lengthSec,
|
||||
AfterID: after,
|
||||
BatchLimit: w.batch,
|
||||
})
|
||||
if err != nil {
|
||||
return res, fmt.Errorf("list tracks needing fingerprint: %w", err)
|
||||
@@ -154,7 +174,10 @@ func (w *FingerprintBackfillWorker) pass(ctx context.Context) (BackfillFingerpri
|
||||
return res, nil
|
||||
}
|
||||
|
||||
sem := make(chan struct{}, w.concurrency)
|
||||
opts := fingerprintOptions{lengthSec: lengthSec, chromaprint: true}
|
||||
// Validation keeps concurrency at one or more; the floor guards a zero
|
||||
// that would block the first send for ever.
|
||||
sem := make(chan struct{}, max(1, int(cfg.BackfillConcurrency)))
|
||||
var wg sync.WaitGroup
|
||||
for _, row := range rows {
|
||||
if ctx.Err() != nil {
|
||||
@@ -170,7 +193,7 @@ func (w *FingerprintBackfillWorker) pass(ctx context.Context) (BackfillFingerpri
|
||||
w.logger.Error("fingerprint backfill: track panicked", "path", path, "panic", r)
|
||||
}
|
||||
}()
|
||||
outcome := storeFingerprint(ctx, q, w.logger, trackID, path, w.fingerprintFile(ctx, path))
|
||||
outcome := storeFingerprint(ctx, q, w.logger, trackID, path, w.fingerprintFile(ctx, path, opts), lengthSec)
|
||||
mu.Lock()
|
||||
res.add(outcome)
|
||||
mu.Unlock()
|
||||
@@ -181,16 +204,21 @@ func (w *FingerprintBackfillWorker) pass(ctx context.Context) (BackfillFingerpri
|
||||
}
|
||||
}
|
||||
|
||||
func (w *FingerprintBackfillWorker) fingerprintFile(ctx context.Context, path string) fingerprintResult {
|
||||
func (w *FingerprintBackfillWorker) fingerprintFile(ctx context.Context, path string, opts fingerprintOptions) fingerprintResult {
|
||||
if w.fingerprint == nil {
|
||||
return computeFingerprint(ctx, path)
|
||||
return computeFingerprint(ctx, path, opts)
|
||||
}
|
||||
return w.fingerprint(ctx, path)
|
||||
return w.fingerprint(ctx, path, opts)
|
||||
}
|
||||
|
||||
// FingerprintCoverage reports how much of the library carries a current
|
||||
// fingerprint, for the admin gauge. It lives here, beside the backfill, so the
|
||||
// version it counts against is the one the backfill writes.
|
||||
func FingerprintCoverage(ctx context.Context, pool *pgxpool.Pool) (dbq.GetFingerprintCoverageRow, error) {
|
||||
return dbq.New(pool).GetFingerprintCoverage(ctx, fingerprintVersion)
|
||||
// version and length it counts against are the ones the backfill writes.
|
||||
func FingerprintCoverage(
|
||||
ctx context.Context, pool *pgxpool.Pool, cfg FingerprintSettings,
|
||||
) (dbq.GetFingerprintCoverageRow, error) {
|
||||
return dbq.New(pool).GetFingerprintCoverage(ctx, dbq.GetFingerprintCoverageParams{
|
||||
CurrentVersion: fingerprintVersion,
|
||||
ChromaprintLengthSec: cfg.ChromaprintLengthSec,
|
||||
})
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user