feat(admin): fingerprinting settings — on/off, length, match threshold, concurrency, sweep interval (M400 #3913)
test-go / test (push) Failing after 44s
test-web / test (push) Successful in 49s
test-go / integration (push) Failing after 2m42s
release / Build + push container image (push) Canceled after 0s
release / Verify release artifacts (tag releases only) (push) Canceled after 0s
release / Build signed APK (releases and dev) (push) Canceled after 4m8s

Rule 25: the fingerprinting knobs move out of source into a DB-backed
singleton (migration 0061), edited from a card on the Duplicates page and
shared live with the scanner, the backfill and the duplicate sweep through
one service instance, so a save needs no restart.

The length is the knob that can silently break the library: prints taken
at two lengths never match. Each track_fingerprints row now records the
length it was taken at, and every reader filters on the current one — the
backfill treats another length as stale, the gauge counts it pending, the
sweep never streams it. Equivalent to a version bump, except that setting
the length back makes rows not yet redone current again. The card warns
before a length change re-fingerprints the library.

Off stops every decode: the scan takes only the stream hash (a demux, and
what recognises a moved file) and stores nothing, dropping a changed file's
stale row; the backfill idles. A save also makes a sweep due, since a new
threshold or length changes what the same prints group into, and the sweep
interval gains slack so an hourly interval on an hourly tick doesn't skip
every other tick.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01SQ31KQpYbStyK5y58UmPLH
This commit is contained in:
2026-09-11 17:53:55 -04:00
co-authored by Claude Opus 5
parent c8bf9dc929
commit 077ae61235
38 changed files with 1679 additions and 201 deletions
+12 -3
View File
@@ -122,7 +122,15 @@ func run() error {
} }
defer pool.Close() defer pool.Close()
scanner := library.New(pool, logger, cfg.Library.ScanPaths) // Fingerprinting settings (M400 #3913): one instance, shared by the scanner,
// the fingerprint backfill, the duplicate sweep and the admin API, so a save
// reaches all of them without a restart. A load failure is logged, not fatal:
// the service falls back to the shipped defaults.
fpSettings, fpErr := library.NewFingerprintSettingsService(ctx, pool)
if fpErr != nil {
logger.Warn("fingerprint settings: using defaults", "err", fpErr)
}
scanner := library.New(pool, logger, cfg.Library.ScanPaths, fpSettings)
contact := cfg.Library.ContactEmail contact := cfg.Library.ContactEmail
if contact == "" { if contact == "" {
@@ -218,12 +226,12 @@ func run() error {
// will — everything imported before fingerprinting existed, and rows derived // will — everything imported before fingerprinting existed, and rows derived
// by an older method. A worker of its own rather than a scan stage; see // by an older method. A worker of its own rather than a scan stage; see
// internal/library/fingerprint_backfill.go for why. // internal/library/fingerprint_backfill.go for why.
go library.NewFingerprintBackfillWorker(pool, logger.With("component", "fingerprint_backfill")).Run(ctx) go library.NewFingerprintBackfillWorker(pool, logger.With("component", "fingerprint_backfill"), fpSettings).Run(ctx)
// Duplicate sweep (M400 #3910): proposes groups of tracks holding one // Duplicate sweep (M400 #3910): proposes groups of tracks holding one
// recording, from the fingerprints above. Sweeps only when fingerprints have // recording, from the fingerprints above. Sweeps only when fingerprints have
// changed since the last sweep. // changed since the last sweep.
go library.NewDuplicateSweepWorker(pool, logger.With("component", "duplicate_sweep")).Run(ctx) go library.NewDuplicateSweepWorker(pool, logger.With("component", "duplicate_sweep"), fpSettings).Run(ctx)
// Start the tag-enrichment worker (#1490). Reconciles the compiled-in // Start the tag-enrichment worker (#1490). Reconciles the compiled-in
// tag providers with tag_provider_settings, bumps the sources version if // tag providers with tag_provider_settings, bumps the sources version if
@@ -368,6 +376,7 @@ func run() error {
srv.PlaylistScheduler = playlistScheduler srv.PlaylistScheduler = playlistScheduler
srv.RecSettings = recSettings srv.RecSettings = recSettings
srv.TagSettings = tagSettings srv.TagSettings = tagSettings
srv.FingerprintSettings = fpSettings
srv.StreamSecret = cfg.StreamSecret srv.StreamSecret = cfg.StreamSecret
httpServer := &http.Server{ httpServer := &http.Server{
Addr: cfg.Server.Address, Addr: cfg.Server.Address,
+24 -8
View File
@@ -1,6 +1,7 @@
package api package api
import ( import (
"context"
"net/http" "net/http"
"git.fabledsword.com/bvandeusen/minstrel/internal/apierror" "git.fabledsword.com/bvandeusen/minstrel/internal/apierror"
@@ -40,12 +41,32 @@ func (h *handlers) handleGetLibraryCoverage(w http.ResponseWriter, r *http.Reque
// fingerprintCoverageResp is the wire shape for GET /api/admin/library/fingerprints. // fingerprintCoverageResp is the wire shape for GET /api/admin/library/fingerprints.
// fingerprinted + rejected + pending = total. Missing tracks are not counted: // fingerprinted + rejected + pending = total. Missing tracks are not counted:
// there is no file to fingerprint. // there is no file to fingerprint. Enabled travels with the counts because with
// fingerprinting off (#3913) pending never shrinks, and a gauge that implies
// progress would be promising work nothing is doing.
type fingerprintCoverageResp struct { type fingerprintCoverageResp struct {
Total int64 `json:"total"` Total int64 `json:"total"`
Fingerprinted int64 `json:"fingerprinted"` Fingerprinted int64 `json:"fingerprinted"`
Rejected int64 `json:"rejected"` Rejected int64 `json:"rejected"`
Pending int64 `json:"pending"` Pending int64 `json:"pending"`
Enabled bool `json:"enabled"`
}
// fingerprintCoverage reads the gauge against the current settings: a print at
// another length counts as pending, because the backfill will re-derive it.
func (h *handlers) fingerprintCoverage(ctx context.Context) (fingerprintCoverageResp, error) {
cfg := h.fingerprintSettings.Get()
row, err := library.FingerprintCoverage(ctx, h.pool, cfg)
if err != nil {
return fingerprintCoverageResp{}, err
}
return fingerprintCoverageResp{
Total: row.Total,
Fingerprinted: row.Fingerprinted,
Rejected: row.Rejected,
Pending: row.Pending,
Enabled: cfg.Enabled,
}, nil
} }
// handleGetFingerprintCoverage implements GET /api/admin/library/fingerprints: // handleGetFingerprintCoverage implements GET /api/admin/library/fingerprints:
@@ -53,15 +74,10 @@ type fingerprintCoverageResp struct {
// worker spanning many passes, with no scan run to attach a tally to, so its // worker spanning many passes, with no scan run to attach a tally to, so its
// progress is read live here. Always 200; zeros on an empty library. // progress is read live here. Always 200; zeros on an empty library.
func (h *handlers) handleGetFingerprintCoverage(w http.ResponseWriter, r *http.Request) { func (h *handlers) handleGetFingerprintCoverage(w http.ResponseWriter, r *http.Request) {
row, err := library.FingerprintCoverage(r.Context(), h.pool) cov, err := h.fingerprintCoverage(r.Context())
if err != nil { if err != nil {
writeErrWithLog(w, h.logger, "admin: get fingerprint coverage", apierror.InternalMsg("lookup failed", err)) writeErrWithLog(w, h.logger, "admin: get fingerprint coverage", apierror.InternalMsg("lookup failed", err))
return return
} }
writeJSON(w, http.StatusOK, fingerprintCoverageResp{ writeJSON(w, http.StatusOK, cov)
Total: row.Total,
Fingerprinted: row.Fingerprinted,
Rejected: row.Rejected,
Pending: row.Pending,
})
} }
+10 -10
View File
@@ -95,7 +95,7 @@ func (h *handlers) handleListDuplicates(w http.ResponseWriter, r *http.Request)
return return
} }
cov, err := library.FingerprintCoverage(ctx, h.pool) cov, err := h.fingerprintCoverage(ctx)
if err != nil { if err != nil {
h.logger.Error("admin: fingerprint coverage", "err", err) h.logger.Error("admin: fingerprint coverage", "err", err)
writeAdminJSONErr(w, http.StatusInternalServerError, "server_error") writeAdminJSONErr(w, http.StatusInternalServerError, "server_error")
@@ -117,14 +117,12 @@ func (h *handlers) handleListDuplicates(w http.ResponseWriter, r *http.Request)
} }
writeJSON(w, http.StatusOK, adminDuplicatesResponse{ writeJSON(w, http.StatusOK, adminDuplicatesResponse{
Sweep: sweep, Sweep: sweep,
Fingerprints: fingerprintCoverageResp{ Fingerprints: cov,
Total: cov.Total, Fingerprinted: cov.Fingerprinted, Rejected: cov.Rejected, Pending: cov.Pending, Total: total,
}, Limit: limit,
Total: total, Offset: offset,
Limit: limit, Groups: foldDuplicateGroups(rows),
Offset: offset,
Groups: foldDuplicateGroups(rows),
}) })
} }
@@ -194,8 +192,10 @@ func foldDuplicateGroups(rows []dbq.ListPendingDuplicateGroupMembersRow) []dupli
// The sweep outlives the request, so it runs on a background context, as // The sweep outlives the request, so it runs on a background context, as
// handleTriggerScan's scan does. // handleTriggerScan's scan does.
func (h *handlers) handleRunDuplicateSweep(w http.ResponseWriter, _ *http.Request) { func (h *handlers) handleRunDuplicateSweep(w http.ResponseWriter, _ *http.Request) {
// Runs whatever the sweep interval says: the interval paces the automatic
// sweep, and an operator pressing the button has already decided.
started, err := library.TryStartDuplicateSweep( started, err := library.TryStartDuplicateSweep(
context.Background(), h.pool, h.logger.With("source", "manual"), context.Background(), h.pool, h.logger.With("source", "manual"), h.fingerprintSettings.Get(),
) )
if err != nil { if err != nil {
h.logger.Error("admin: start duplicate sweep", "err", err) h.logger.Error("admin: start duplicate sweep", "err", err)
@@ -0,0 +1,67 @@
package api
import (
"encoding/json"
"errors"
"net/http"
"git.fabledsword.com/bvandeusen/minstrel/internal/apierror"
"git.fabledsword.com/bvandeusen/minstrel/internal/library"
)
// fingerprintSettingsBody is the wire shape for GET and PUT
// /api/admin/library/fingerprint-settings (M400 #3913). The threshold travels as
// the bit-error rate the matcher uses; the card presents it as a match percentage.
type fingerprintSettingsBody struct {
Enabled bool `json:"enabled"`
ChromaprintLengthSec int32 `json:"chromaprint_length_sec"`
AcousticMaxBitErrorRate float64 `json:"acoustic_max_bit_error_rate"`
BackfillConcurrency int32 `json:"backfill_concurrency"`
SweepIntervalHours int32 `json:"sweep_interval_hours"`
}
func fingerprintSettingsBodyOf(s library.FingerprintSettings) fingerprintSettingsBody {
return fingerprintSettingsBody{
Enabled: s.Enabled,
ChromaprintLengthSec: s.ChromaprintLengthSec,
AcousticMaxBitErrorRate: s.AcousticMaxBitErrorRate,
BackfillConcurrency: s.BackfillConcurrency,
SweepIntervalHours: s.SweepIntervalHours,
}
}
// handleGetFingerprintSettings implements GET /api/admin/library/fingerprint-settings.
func (h *handlers) handleGetFingerprintSettings(w http.ResponseWriter, _ *http.Request) {
writeJSON(w, http.StatusOK, fingerprintSettingsBodyOf(h.fingerprintSettings.Get()))
}
// handleUpdateFingerprintSettings implements PUT /api/admin/library/fingerprint-settings.
//
// A whole-row write. A body that leaves a field out decodes it as zero, which no
// field accepts, so a partial save is refused rather than zeroing what it omitted.
// The saved settings reach the scanner and both workers at once: they share the
// service instance.
func (h *handlers) handleUpdateFingerprintSettings(w http.ResponseWriter, r *http.Request) {
var req fingerprintSettingsBody
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
writeErr(w, apierror.BadRequest("invalid_body", "malformed JSON"))
return
}
saved, err := h.fingerprintSettings.Set(r.Context(), library.FingerprintSettings{
Enabled: req.Enabled,
ChromaprintLengthSec: req.ChromaprintLengthSec,
AcousticMaxBitErrorRate: req.AcousticMaxBitErrorRate,
BackfillConcurrency: req.BackfillConcurrency,
SweepIntervalHours: req.SweepIntervalHours,
})
if err != nil {
// Validation mirrors migration 0061's CHECKs and names the field.
if errors.Is(err, library.ErrFingerprintSettingOutOfRange) {
writeErr(w, apierror.BadRequest("invalid_setting", err.Error()))
return
}
writeErrWithLog(w, h.logger, "admin fingerprint settings: update failed", apierror.Internal(err))
return
}
writeJSON(w, http.StatusOK, fingerprintSettingsBodyOf(saved))
}
@@ -0,0 +1,55 @@
package api
import (
"encoding/json"
"io"
"log/slog"
"net/http"
"net/http/httptest"
"strings"
"testing"
"git.fabledsword.com/bvandeusen/minstrel/internal/library"
)
func TestGetFingerprintSettings_ServesDefaultsWithoutAService(t *testing.T) {
h := &handlers{logger: slog.New(slog.NewTextHandler(io.Discard, nil))}
rec := httptest.NewRecorder()
h.handleGetFingerprintSettings(rec, httptest.NewRequest(http.MethodGet, "/api/admin/library/fingerprint-settings", nil))
if rec.Code != http.StatusOK {
t.Fatalf("status = %d, want 200", rec.Code)
}
var got fingerprintSettingsBody
if err := json.NewDecoder(rec.Body).Decode(&got); err != nil {
t.Fatalf("decode: %v", err)
}
if want := fingerprintSettingsBodyOf(library.DefaultFingerprintSettings); got != want {
t.Fatalf("body = %+v, want the defaults %+v", got, want)
}
}
func TestUpdateFingerprintSettings_Rejects(t *testing.T) {
h := &handlers{logger: slog.New(slog.NewTextHandler(io.Discard, nil))}
for name, tc := range map[string]struct {
body string
code string
mentions string
}{
"a value out of range, naming the field": {
body: `{"enabled":true,"chromaprint_length_sec":5,"acoustic_max_bit_error_rate":0.15,"backfill_concurrency":2,"sweep_interval_hours":1}`,
code: "invalid_setting",
mentions: "chromaprint_length_sec",
},
// A partial body would otherwise zero every field it left out.
"a body missing fields": {body: `{"enabled":false}`, code: "invalid_setting"},
"malformed JSON": {body: `{"enabled":`, code: "invalid_body"},
} {
rec := httptest.NewRecorder()
h.handleUpdateFingerprintSettings(rec, httptest.NewRequest(
http.MethodPut, "/api/admin/library/fingerprint-settings", strings.NewReader(tc.body)))
body := rec.Body.String()
if rec.Code != http.StatusBadRequest || !strings.Contains(body, `"`+tc.code+`"`) || !strings.Contains(body, tc.mentions) {
t.Errorf("%s: status %d body %s; want 400 %s mentioning %q", name, rec.Code, body, tc.code, tc.mentions)
}
}
}
+28 -21
View File
@@ -33,30 +33,31 @@ import (
// Mount attaches /api/* handlers to r. Public endpoints (login) are outside // Mount attaches /api/* handlers to r. Public endpoints (login) are outside
// RequireUser; everything else is gated by the middleware. The events writer // RequireUser; everything else is gated by the middleware. The events writer
// is shared with the Subsonic mount so /rest/scrobble feeds the same store. // is shared with the Subsonic mount so /rest/scrobble feeds the same store.
func Mount(r chi.Router, pool *pgxpool.Pool, logger *slog.Logger, events *playevents.Writer, recCfg config.RecommendationConfig, recSettings *recsettings.Service, lidarrCfg *lidarrconfig.Service, lidarrReqs *lidarrrequests.Service, lidarrQuar *lidarrquarantine.Service, tracksSvc *tracks.Service, playlistsSvc *playlists.Service, coverEnricher *coverart.Enricher, coverSettings *coverart.SettingsService, tagSettings *tags.SettingsService, scanner *library.Scanner, scanCfg library.RunScanConfig, dataDir string, sender mailer.Sender, bus *eventbus.Bus, playlistScheduler *playlists.Scheduler, streamSecret []byte, netSettings *netsettings.Service, reacqSettings *reacquisition.SettingsService) { func Mount(r chi.Router, pool *pgxpool.Pool, logger *slog.Logger, events *playevents.Writer, recCfg config.RecommendationConfig, recSettings *recsettings.Service, lidarrCfg *lidarrconfig.Service, lidarrReqs *lidarrrequests.Service, lidarrQuar *lidarrquarantine.Service, tracksSvc *tracks.Service, playlistsSvc *playlists.Service, coverEnricher *coverart.Enricher, coverSettings *coverart.SettingsService, tagSettings *tags.SettingsService, scanner *library.Scanner, scanCfg library.RunScanConfig, dataDir string, sender mailer.Sender, bus *eventbus.Bus, playlistScheduler *playlists.Scheduler, streamSecret []byte, netSettings *netsettings.Service, reacqSettings *reacquisition.SettingsService, fpSettings *library.FingerprintSettingsService) {
rng := rand.New(rand.NewSource(rand.Int63())) rng := rand.New(rand.NewSource(rand.Int63()))
h := &handlers{ h := &handlers{
pool: pool, logger: logger, events: events, recCfg: recCfg, pool: pool, logger: logger, events: events, recCfg: recCfg,
recSettings: recSettings, recSettings: recSettings,
rng: rng.Float64, rng: rng.Float64,
lidarrCfg: lidarrCfg, lidarrCfg: lidarrCfg,
lidarrRequests: lidarrReqs, lidarrRequests: lidarrReqs,
lidarrQuarantine: lidarrQuar, lidarrQuarantine: lidarrQuar,
tracks: tracksSvc, tracks: tracksSvc,
playlists: playlistsSvc, playlists: playlistsSvc,
coverart: coverEnricher, coverart: coverEnricher,
coverSettings: coverSettings, coverSettings: coverSettings,
tagSettings: tagSettings, tagSettings: tagSettings,
scanner: scanner, scanner: scanner,
scanCfg: scanCfg, scanCfg: scanCfg,
dataDir: dataDir, dataDir: dataDir,
mailer: sender, mailer: sender,
eventbus: bus, eventbus: bus,
playlistScheduler: playlistScheduler, playlistScheduler: playlistScheduler,
streamSecret: streamSecret, streamSecret: streamSecret,
netSettings: netSettings, netSettings: netSettings,
reacqSettings: reacqSettings, reacqSettings: reacqSettings,
librarySize: recommendation.NewLibrarySize(nil), fingerprintSettings: fpSettings,
librarySize: recommendation.NewLibrarySize(nil),
} }
r.Route("/api", func(api chi.Router) { r.Route("/api", func(api chi.Router) {
@@ -216,6 +217,8 @@ func Mount(r chi.Router, pool *pgxpool.Pool, logger *slog.Logger, events *playev
admin.Get("/library/coverage", h.handleGetLibraryCoverage) admin.Get("/library/coverage", h.handleGetLibraryCoverage)
admin.Get("/library/fingerprints", h.handleGetFingerprintCoverage) admin.Get("/library/fingerprints", h.handleGetFingerprintCoverage)
admin.Get("/library/fingerprint-settings", h.handleGetFingerprintSettings)
admin.Put("/library/fingerprint-settings", h.handleUpdateFingerprintSettings)
// Duplicates report (#3912): proposals from the duplicate sweep, a // Duplicates report (#3912): proposals from the duplicate sweep, a
// trigger to sweep now, dismissal, and the merge (#3911), which deletes // trigger to sweep now, dismissal, and the merge (#3911), which deletes
// the removed copies' files after moving their history onto the kept one. // the removed copies' files after moving their history onto the kept one.
@@ -306,6 +309,10 @@ type handlers struct {
// missing files (milestone #290) — grace window, backoff, attempt caps. // missing files (milestone #290) — grace window, backoff, attempt caps.
// Cached in the service, so the admin card reads it without a query. // Cached in the service, so the admin card reads it without a query.
reacqSettings *reacquisition.SettingsService reacqSettings *reacquisition.SettingsService
// fingerprintSettings is the fingerprinting policy (M400 #3913), the same
// instance the scanner and the fingerprint workers read, so a save from the
// admin card reaches them without a restart. Nil serves the defaults.
fingerprintSettings *library.FingerprintSettingsService
// netSettings caches the trusted reverse-proxy depth read by the auth // netSettings caches the trusted reverse-proxy depth read by the auth
// middleware on every request and edited from the admin network card. // middleware on every request and edited from the admin network card.
netSettings *netsettings.Service netSettings *netsettings.Service
+12 -6
View File
@@ -211,16 +211,21 @@ SELECT t.id, t.duration_ms, f.audio_stream_sha256, f.chromaprint
WHERE t.missing_since IS NULL WHERE t.missing_since IS NULL
AND f.fingerprint_version >= $1 AND f.fingerprint_version >= $1
AND f.chromaprint IS NOT NULL AND f.chromaprint IS NOT NULL
AND (t.duration_ms, t.id) > ($2::integer, $3::uuid) -- Only chromaprints taken at the current length: prints at two lengths are not
-- comparable, and after a length change the backfill is still re-deriving the
-- rest (#3913).
AND f.chromaprint_length_sec = $2
AND (t.duration_ms, t.id) > ($3::integer, $4::uuid)
ORDER BY t.duration_ms, t.id ORDER BY t.duration_ms, t.id
LIMIT $4 LIMIT $5
` `
type ListDuplicateCandidatesParams struct { type ListDuplicateCandidatesParams struct {
CurrentVersion int16 CurrentVersion int16
AfterDurationMs int32 ChromaprintLengthSec int32
AfterID pgtype.UUID AfterDurationMs int32
PageLimit int32 AfterID pgtype.UUID
PageLimit int32
} }
type ListDuplicateCandidatesRow struct { type ListDuplicateCandidatesRow struct {
@@ -237,6 +242,7 @@ type ListDuplicateCandidatesRow struct {
func (q *Queries) ListDuplicateCandidates(ctx context.Context, arg ListDuplicateCandidatesParams) ([]ListDuplicateCandidatesRow, error) { func (q *Queries) ListDuplicateCandidates(ctx context.Context, arg ListDuplicateCandidatesParams) ([]ListDuplicateCandidatesRow, error) {
rows, err := q.db.Query(ctx, listDuplicateCandidates, rows, err := q.db.Query(ctx, listDuplicateCandidates,
arg.CurrentVersion, arg.CurrentVersion,
arg.ChromaprintLengthSec,
arg.AfterDurationMs, arg.AfterDurationMs,
arg.AfterID, arg.AfterID,
arg.PageLimit, arg.PageLimit,
@@ -0,0 +1,72 @@
// Code generated by sqlc. DO NOT EDIT.
// versions:
// sqlc v1.31.1
// source: fingerprint_settings.sql
package dbq
import (
"context"
)
const getFingerprintSettings = `-- name: GetFingerprintSettings :one
SELECT id, enabled, chromaprint_length_sec, acoustic_max_bit_error_rate, backfill_concurrency, sweep_interval_hours, updated_at FROM fingerprint_settings WHERE id = true
`
func (q *Queries) GetFingerprintSettings(ctx context.Context) (FingerprintSetting, error) {
row := q.db.QueryRow(ctx, getFingerprintSettings)
var i FingerprintSetting
err := row.Scan(
&i.ID,
&i.Enabled,
&i.ChromaprintLengthSec,
&i.AcousticMaxBitErrorRate,
&i.BackfillConcurrency,
&i.SweepIntervalHours,
&i.UpdatedAt,
)
return i, err
}
const updateFingerprintSettings = `-- name: UpdateFingerprintSettings :one
UPDATE fingerprint_settings
SET enabled = $1,
chromaprint_length_sec = $2,
acoustic_max_bit_error_rate = $3,
backfill_concurrency = $4,
sweep_interval_hours = $5,
updated_at = now()
WHERE id = true
RETURNING id, enabled, chromaprint_length_sec, acoustic_max_bit_error_rate, backfill_concurrency, sweep_interval_hours, updated_at
`
type UpdateFingerprintSettingsParams struct {
Enabled bool
ChromaprintLengthSec int32
AcousticMaxBitErrorRate float64
BackfillConcurrency int32
SweepIntervalHours int32
}
// Whole-row write from the admin card; migration 0061's CHECKs are the backstop
// behind the service's own validation.
func (q *Queries) UpdateFingerprintSettings(ctx context.Context, arg UpdateFingerprintSettingsParams) (FingerprintSetting, error) {
row := q.db.QueryRow(ctx, updateFingerprintSettings,
arg.Enabled,
arg.ChromaprintLengthSec,
arg.AcousticMaxBitErrorRate,
arg.BackfillConcurrency,
arg.SweepIntervalHours,
)
var i FingerprintSetting
err := row.Scan(
&i.ID,
&i.Enabled,
&i.ChromaprintLengthSec,
&i.AcousticMaxBitErrorRate,
&i.BackfillConcurrency,
&i.SweepIntervalHours,
&i.UpdatedAt,
)
return i, err
}
+47 -23
View File
@@ -27,20 +27,29 @@ const getFingerprintCoverage = `-- name: GetFingerprintCoverage :one
SELECT count(*)::bigint AS total, SELECT count(*)::bigint AS total,
count(*) FILTER ( count(*) FILTER (
WHERE f.fingerprint_version >= $1 WHERE f.fingerprint_version >= $1
AND f.chromaprint_length_sec = $2
AND f.audio_stream_sha256 IS NOT NULL AND f.chromaprint IS NOT NULL AND f.audio_stream_sha256 IS NOT NULL AND f.chromaprint IS NOT NULL
)::bigint AS fingerprinted, )::bigint AS fingerprinted,
count(*) FILTER ( count(*) FILTER (
WHERE f.fingerprint_version >= $1 WHERE f.fingerprint_version >= $1
AND f.chromaprint_length_sec = $2
AND (f.audio_stream_sha256 IS NULL OR f.chromaprint IS NULL) AND (f.audio_stream_sha256 IS NULL OR f.chromaprint IS NULL)
)::bigint AS rejected, )::bigint AS rejected,
count(*) FILTER ( count(*) FILTER (
WHERE f.track_id IS NULL OR f.fingerprint_version < $1 WHERE f.track_id IS NULL
OR f.fingerprint_version < $1
OR f.chromaprint_length_sec <> $2
)::bigint AS pending )::bigint AS pending
FROM tracks t FROM tracks t
LEFT JOIN track_fingerprints f ON f.track_id = t.id LEFT JOIN track_fingerprints f ON f.track_id = t.id
WHERE t.missing_since IS NULL WHERE t.missing_since IS NULL
` `
type GetFingerprintCoverageParams struct {
CurrentVersion int16
ChromaprintLengthSec int32
}
type GetFingerprintCoverageRow struct { type GetFingerprintCoverageRow struct {
Total int64 Total int64
Fingerprinted int64 Fingerprinted int64
@@ -49,11 +58,13 @@ type GetFingerprintCoverageRow struct {
} }
// The admin gauge for the backfill. fingerprinted + rejected + pending = total. // The admin gauge for the backfill. fingerprinted + rejected + pending = total.
// rejected is a row at the current version with a NULL half: a tool ran and // "Current" means derived by the current method AT the current length: a row at
// refused the file, which is settled rather than waiting. Missing tracks are // another length is pending, because the backfill will re-derive it. rejected is
// excluded, or the gauge could never reach the end. // a current row with a NULL half: a tool ran and refused the file, which is
func (q *Queries) GetFingerprintCoverage(ctx context.Context, currentVersion int16) (GetFingerprintCoverageRow, error) { // settled rather than waiting. Missing tracks are excluded, or the gauge could
row := q.db.QueryRow(ctx, getFingerprintCoverage, currentVersion) // never reach the end.
func (q *Queries) GetFingerprintCoverage(ctx context.Context, arg GetFingerprintCoverageParams) (GetFingerprintCoverageRow, error) {
row := q.db.QueryRow(ctx, getFingerprintCoverage, arg.CurrentVersion, arg.ChromaprintLengthSec)
var i GetFingerprintCoverageRow var i GetFingerprintCoverageRow
err := row.Scan( err := row.Scan(
&i.Total, &i.Total,
@@ -69,16 +80,21 @@ SELECT t.id, t.file_path
FROM tracks t FROM tracks t
LEFT JOIN track_fingerprints f ON f.track_id = t.id LEFT JOIN track_fingerprints f ON f.track_id = t.id
WHERE t.missing_since IS NULL WHERE t.missing_since IS NULL
AND (f.track_id IS NULL OR f.fingerprint_version < $1) -- A row taken at another length is as stale as one from an older method:
AND t.id > $2 -- chromaprints at two lengths cannot be compared (#3913).
AND (f.track_id IS NULL
OR f.fingerprint_version < $1
OR f.chromaprint_length_sec <> $2)
AND t.id > $3
ORDER BY t.id ORDER BY t.id
LIMIT $3 LIMIT $4
` `
type ListTracksNeedingFingerprintParams struct { type ListTracksNeedingFingerprintParams struct {
CurrentVersion int16 CurrentVersion int16
AfterID pgtype.UUID ChromaprintLengthSec int32
BatchLimit int32 AfterID pgtype.UUID
BatchLimit int32
} }
type ListTracksNeedingFingerprintRow struct { type ListTracksNeedingFingerprintRow struct {
@@ -93,7 +109,12 @@ type ListTracksNeedingFingerprintRow struct {
// and retried in a tight loop. Missing tracks are skipped — there is no file to // and retried in a tight loop. Missing tracks are skipped — there is no file to
// read. // read.
func (q *Queries) ListTracksNeedingFingerprint(ctx context.Context, arg ListTracksNeedingFingerprintParams) ([]ListTracksNeedingFingerprintRow, error) { func (q *Queries) ListTracksNeedingFingerprint(ctx context.Context, arg ListTracksNeedingFingerprintParams) ([]ListTracksNeedingFingerprintRow, error) {
rows, err := q.db.Query(ctx, listTracksNeedingFingerprint, arg.CurrentVersion, arg.AfterID, arg.BatchLimit) rows, err := q.db.Query(ctx, listTracksNeedingFingerprint,
arg.CurrentVersion,
arg.ChromaprintLengthSec,
arg.AfterID,
arg.BatchLimit,
)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@@ -114,23 +135,25 @@ func (q *Queries) ListTracksNeedingFingerprint(ctx context.Context, arg ListTrac
const upsertTrackFingerprint = `-- name: UpsertTrackFingerprint :exec const upsertTrackFingerprint = `-- name: UpsertTrackFingerprint :exec
INSERT INTO track_fingerprints ( INSERT INTO track_fingerprints (
track_id, audio_stream_sha256, chromaprint, fingerprint_version track_id, audio_stream_sha256, chromaprint, fingerprint_version, chromaprint_length_sec
) VALUES ( ) VALUES (
$1, $2, $3, $1, $2, $3,
$4 $4, $5
) )
ON CONFLICT (track_id) DO UPDATE SET ON CONFLICT (track_id) DO UPDATE SET
audio_stream_sha256 = EXCLUDED.audio_stream_sha256, audio_stream_sha256 = EXCLUDED.audio_stream_sha256,
chromaprint = EXCLUDED.chromaprint, chromaprint = EXCLUDED.chromaprint,
fingerprint_version = EXCLUDED.fingerprint_version, fingerprint_version = EXCLUDED.fingerprint_version,
computed_at = now() chromaprint_length_sec = EXCLUDED.chromaprint_length_sec,
computed_at = now()
` `
type UpsertTrackFingerprintParams struct { type UpsertTrackFingerprintParams struct {
TrackID pgtype.UUID TrackID pgtype.UUID
AudioStreamSha256 []byte AudioStreamSha256 []byte
Chromaprint []int32 Chromaprint []int32
FingerprintVersion int16 FingerprintVersion int16
ChromaprintLengthSec int32
} }
// Written whenever a track's fingerprint is derived: by the scan when a file is // Written whenever a track's fingerprint is derived: by the scan when a file is
@@ -143,6 +166,7 @@ func (q *Queries) UpsertTrackFingerprint(ctx context.Context, arg UpsertTrackFin
arg.AudioStreamSha256, arg.AudioStreamSha256,
arg.Chromaprint, arg.Chromaprint,
arg.FingerprintVersion, arg.FingerprintVersion,
arg.ChromaprintLengthSec,
) )
return err return err
} }
+16 -5
View File
@@ -323,6 +323,16 @@ type DuplicateSweep struct {
ErrorMessage *string ErrorMessage *string
} }
type FingerprintSetting struct {
ID bool
Enabled bool
ChromaprintLengthSec int32
AcousticMaxBitErrorRate float64
BackfillConcurrency int32
SweepIntervalHours int32
UpdatedAt pgtype.Timestamptz
}
type GeneralLike struct { type GeneralLike struct {
UserID pgtype.UUID UserID pgtype.UUID
TrackID pgtype.UUID TrackID pgtype.UUID
@@ -694,11 +704,12 @@ type Track struct {
} }
type TrackFingerprint struct { type TrackFingerprint struct {
TrackID pgtype.UUID TrackID pgtype.UUID
AudioStreamSha256 []byte AudioStreamSha256 []byte
Chromaprint []int32 Chromaprint []int32
FingerprintVersion int16 FingerprintVersion int16
ComputedAt pgtype.Timestamptz ComputedAt pgtype.Timestamptz
ChromaprintLengthSec int32
} }
type TrackSimilarity struct { type TrackSimilarity struct {
@@ -0,0 +1,2 @@
ALTER TABLE track_fingerprints DROP COLUMN chromaprint_length_sec;
DROP TABLE fingerprint_settings;
@@ -0,0 +1,50 @@
-- 0061_fingerprint_settings.up.sql — fingerprinting's knobs, in admin Settings
-- (Scribe #3913, milestone #400). Rule 25: anything an operator might tune is a
-- database row, changed without a restart. Singleton in the style of
-- reacquisition_settings (0056).
CREATE TABLE fingerprint_settings (
id boolean PRIMARY KEY DEFAULT true,
-- Fingerprinting new files, the backfill, and the duplicate sweep. Off stops
-- the decode work entirely — the reason to turn it off is a slow NAS, and
-- that is the operator's call. On by default: a library that cannot tell its
-- duplicates apart is what milestone #400 exists to end.
enabled boolean NOT NULL DEFAULT true,
-- Seconds of audio fpcalc fingerprints. Chromaprints taken at different
-- lengths cannot be compared, which is why track_fingerprints records the
-- length each row was taken at (below): change this and every chromaprint is
-- re-derived, and until then only rows at the new length are compared.
chromaprint_length_sec integer NOT NULL DEFAULT 120,
-- The most disagreement two aligned fingerprints may show and still be
-- proposed as one recording. Unrelated audio sits near 0.5, so the ceiling
-- stays well clear of it.
acoustic_max_bit_error_rate double precision NOT NULL DEFAULT 0.15,
-- Files the backfill decodes at once. Decoding competes with playback
-- transcoding for CPU and with streaming for the mount.
backfill_concurrency integer NOT NULL DEFAULT 2,
-- The least time between duplicate sweeps. A sweep still runs only when
-- fingerprints have changed since the last one.
sweep_interval_hours integer NOT NULL DEFAULT 1,
-- When the settings were last saved. A new threshold or length can change
-- what a sweep finds, so a save makes a sweep due.
updated_at timestamptz NOT NULL DEFAULT now(),
CONSTRAINT fingerprint_settings_singleton CHECK (id = true),
CONSTRAINT fingerprint_settings_length_range
CHECK (chromaprint_length_sec >= 30 AND chromaprint_length_sec <= 600),
CONSTRAINT fingerprint_settings_threshold_range
CHECK (acoustic_max_bit_error_rate >= 0.01 AND acoustic_max_bit_error_rate <= 0.35),
CONSTRAINT fingerprint_settings_concurrency_range
CHECK (backfill_concurrency >= 1 AND backfill_concurrency <= 8),
CONSTRAINT fingerprint_settings_sweep_interval_range
CHECK (sweep_interval_hours >= 1 AND sweep_interval_hours <= 168)
);
INSERT INTO fingerprint_settings (id) VALUES (true) ON CONFLICT (id) DO NOTHING;
-- Every row written so far was taken at fpcalc's default length.
ALTER TABLE track_fingerprints ADD COLUMN chromaprint_length_sec integer NOT NULL DEFAULT 120;
+4
View File
@@ -53,6 +53,10 @@ SELECT t.id, t.duration_ms, f.audio_stream_sha256, f.chromaprint
WHERE t.missing_since IS NULL WHERE t.missing_since IS NULL
AND f.fingerprint_version >= sqlc.arg(current_version) AND f.fingerprint_version >= sqlc.arg(current_version)
AND f.chromaprint IS NOT NULL AND f.chromaprint IS NOT NULL
-- Only chromaprints taken at the current length: prints at two lengths are not
-- comparable, and after a length change the backfill is still re-deriving the
-- rest (#3913).
AND f.chromaprint_length_sec = sqlc.arg(chromaprint_length_sec)
AND (t.duration_ms, t.id) > (sqlc.arg(after_duration_ms)::integer, sqlc.arg(after_id)::uuid) AND (t.duration_ms, t.id) > (sqlc.arg(after_duration_ms)::integer, sqlc.arg(after_id)::uuid)
ORDER BY t.duration_ms, t.id ORDER BY t.duration_ms, t.id
LIMIT sqlc.arg(page_limit); LIMIT sqlc.arg(page_limit);
@@ -0,0 +1,15 @@
-- name: GetFingerprintSettings :one
SELECT * FROM fingerprint_settings WHERE id = true;
-- name: UpdateFingerprintSettings :one
-- Whole-row write from the admin card; migration 0061's CHECKs are the backstop
-- behind the service's own validation.
UPDATE fingerprint_settings
SET enabled = sqlc.arg(enabled),
chromaprint_length_sec = sqlc.arg(chromaprint_length_sec),
acoustic_max_bit_error_rate = sqlc.arg(acoustic_max_bit_error_rate),
backfill_concurrency = sqlc.arg(backfill_concurrency),
sweep_interval_hours = sqlc.arg(sweep_interval_hours),
updated_at = now()
WHERE id = true
RETURNING *;
+22 -11
View File
@@ -4,16 +4,17 @@
-- older method. Replaces the row wholesale — a fingerprint of the old bytes has -- older method. Replaces the row wholesale — a fingerprint of the old bytes has
-- no standing once the file has changed. -- no standing once the file has changed.
INSERT INTO track_fingerprints ( INSERT INTO track_fingerprints (
track_id, audio_stream_sha256, chromaprint, fingerprint_version track_id, audio_stream_sha256, chromaprint, fingerprint_version, chromaprint_length_sec
) VALUES ( ) VALUES (
sqlc.arg(track_id), sqlc.narg(audio_stream_sha256), sqlc.narg(chromaprint), sqlc.arg(track_id), sqlc.narg(audio_stream_sha256), sqlc.narg(chromaprint),
sqlc.arg(fingerprint_version) sqlc.arg(fingerprint_version), sqlc.arg(chromaprint_length_sec)
) )
ON CONFLICT (track_id) DO UPDATE SET ON CONFLICT (track_id) DO UPDATE SET
audio_stream_sha256 = EXCLUDED.audio_stream_sha256, audio_stream_sha256 = EXCLUDED.audio_stream_sha256,
chromaprint = EXCLUDED.chromaprint, chromaprint = EXCLUDED.chromaprint,
fingerprint_version = EXCLUDED.fingerprint_version, fingerprint_version = EXCLUDED.fingerprint_version,
computed_at = now(); chromaprint_length_sec = EXCLUDED.chromaprint_length_sec,
computed_at = now();
-- name: DeleteTrackFingerprint :exec -- name: DeleteTrackFingerprint :exec
-- A file changed but could not be fingerprinted, for a reason unrelated to the -- A file changed but could not be fingerprinted, for a reason unrelated to the
@@ -32,27 +33,37 @@ SELECT t.id, t.file_path
FROM tracks t FROM tracks t
LEFT JOIN track_fingerprints f ON f.track_id = t.id LEFT JOIN track_fingerprints f ON f.track_id = t.id
WHERE t.missing_since IS NULL WHERE t.missing_since IS NULL
AND (f.track_id IS NULL OR f.fingerprint_version < sqlc.arg(current_version)) -- A row taken at another length is as stale as one from an older method:
-- chromaprints at two lengths cannot be compared (#3913).
AND (f.track_id IS NULL
OR f.fingerprint_version < sqlc.arg(current_version)
OR f.chromaprint_length_sec <> sqlc.arg(chromaprint_length_sec))
AND t.id > sqlc.arg(after_id) AND t.id > sqlc.arg(after_id)
ORDER BY t.id ORDER BY t.id
LIMIT sqlc.arg(batch_limit); LIMIT sqlc.arg(batch_limit);
-- name: GetFingerprintCoverage :one -- name: GetFingerprintCoverage :one
-- The admin gauge for the backfill. fingerprinted + rejected + pending = total. -- The admin gauge for the backfill. fingerprinted + rejected + pending = total.
-- rejected is a row at the current version with a NULL half: a tool ran and -- "Current" means derived by the current method AT the current length: a row at
-- refused the file, which is settled rather than waiting. Missing tracks are -- another length is pending, because the backfill will re-derive it. rejected is
-- excluded, or the gauge could never reach the end. -- a current row with a NULL half: a tool ran and refused the file, which is
-- settled rather than waiting. Missing tracks are excluded, or the gauge could
-- never reach the end.
SELECT count(*)::bigint AS total, SELECT count(*)::bigint AS total,
count(*) FILTER ( count(*) FILTER (
WHERE f.fingerprint_version >= sqlc.arg(current_version) WHERE f.fingerprint_version >= sqlc.arg(current_version)
AND f.chromaprint_length_sec = sqlc.arg(chromaprint_length_sec)
AND f.audio_stream_sha256 IS NOT NULL AND f.chromaprint IS NOT NULL AND f.audio_stream_sha256 IS NOT NULL AND f.chromaprint IS NOT NULL
)::bigint AS fingerprinted, )::bigint AS fingerprinted,
count(*) FILTER ( count(*) FILTER (
WHERE f.fingerprint_version >= sqlc.arg(current_version) WHERE f.fingerprint_version >= sqlc.arg(current_version)
AND f.chromaprint_length_sec = sqlc.arg(chromaprint_length_sec)
AND (f.audio_stream_sha256 IS NULL OR f.chromaprint IS NULL) AND (f.audio_stream_sha256 IS NULL OR f.chromaprint IS NULL)
)::bigint AS rejected, )::bigint AS rejected,
count(*) FILTER ( count(*) FILTER (
WHERE f.track_id IS NULL OR f.fingerprint_version < sqlc.arg(current_version) WHERE f.track_id IS NULL
OR f.fingerprint_version < sqlc.arg(current_version)
OR f.chromaprint_length_sec <> sqlc.arg(chromaprint_length_sec)
)::bigint AS pending )::bigint AS pending
FROM tracks t FROM tracks t
LEFT JOIN track_fingerprints f ON f.track_id = t.id LEFT JOIN track_fingerprints f ON f.track_id = t.id
+11
View File
@@ -130,4 +130,15 @@ func ResetDB(t *testing.T, pool *pgxpool.Pool) {
); err != nil { ); err != nil {
t.Fatalf("dbtest.ResetDB reset tag-sources version: %v", err) t.Fatalf("dbtest.ResetDB reset tag-sources version: %v", err)
} }
// Fingerprinting settings (M400 #3913), a singleton like the counters above.
// Every column goes back to its migration default rather than to literals
// written here, so a test can pin the Go defaults to the migration's.
if _, err := pool.Exec(ctx, `
UPDATE fingerprint_settings
SET enabled = DEFAULT, chromaprint_length_sec = DEFAULT,
acoustic_max_bit_error_rate = DEFAULT, backfill_concurrency = DEFAULT,
sweep_interval_hours = DEFAULT, updated_at = DEFAULT`,
); err != nil {
t.Fatalf("dbtest.ResetDB reset fingerprint settings: %v", err)
}
} }
+71 -33
View File
@@ -32,10 +32,17 @@ import (
// carries a ~4 KB fingerprint, so a page is about 2 MB. // carries a ~4 KB fingerprint, so a page is about 2 MB.
const duplicateCandidatePage = 500 const duplicateCandidatePage = 500
// duplicateSweepTick is how often the worker checks for anything new to sweep. // duplicateSweepTick is how often the worker checks whether a sweep is due. The
// With nothing new, a tick is two cheap aggregate queries. // operator's sweep interval (#3913) is the least time between sweeps; the tick
// only bounds how late past it one starts. With nothing due, a tick is two cheap
// aggregate queries.
const duplicateSweepTick = time.Hour const duplicateSweepTick = time.Hour
// sweepIntervalSlack absorbs the moment between a tick and the sweep it starts
// stamping started_at. Without it a one-hour interval checked on a one-hour tick
// would find the last sweep a moment under an hour old, and skip every other tick.
const sweepIntervalSlack = 5 * time.Minute
// staleDuplicateSweepThreshold is the age past which an in-flight sweep is // staleDuplicateSweepThreshold is the age past which an in-flight sweep is
// assumed dead — a crash mid-sweep leaves finished_at NULL for ever — and another // assumed dead — a crash mid-sweep leaves finished_at NULL for ever — and another
// may start. Twice the library scan's threshold, because a sweep compares // may start. Twice the library scan's threshold, because a sweep compares
@@ -58,13 +65,16 @@ type DuplicateSweepResult struct {
Oversize int // acoustic clusters too large to propose Oversize int // acoustic clusters too large to propose
} }
// RunDuplicateSweep runs one sweep and records it in duplicate_sweeps. // RunDuplicateSweep runs one sweep and records it in duplicate_sweeps. cfg is a
func RunDuplicateSweep(ctx context.Context, pool *pgxpool.Pool, logger *slog.Logger) (DuplicateSweepResult, error) { // snapshot: one sweep applies one threshold and one length throughout.
return runDuplicateSweep(ctx, pool, logger, duplicateCandidatePage) func RunDuplicateSweep(
ctx context.Context, pool *pgxpool.Pool, logger *slog.Logger, cfg FingerprintSettings,
) (DuplicateSweepResult, error) {
return runDuplicateSweep(ctx, pool, logger, cfg, duplicateCandidatePage)
} }
func runDuplicateSweep( func runDuplicateSweep(
ctx context.Context, pool *pgxpool.Pool, logger *slog.Logger, pageSize int32, ctx context.Context, pool *pgxpool.Pool, logger *slog.Logger, cfg FingerprintSettings, pageSize int32,
) (DuplicateSweepResult, error) { ) (DuplicateSweepResult, error) {
q := dbq.New(pool) q := dbq.New(pool)
sweep, err := q.StartDuplicateSweep(ctx) sweep, err := q.StartDuplicateSweep(ctx)
@@ -72,7 +82,7 @@ func runDuplicateSweep(
return DuplicateSweepResult{}, fmt.Errorf("start duplicate sweep: %w", err) return DuplicateSweepResult{}, fmt.Errorf("start duplicate sweep: %w", err)
} }
res, runErr := sweepDuplicates(ctx, q, sweep.ID, pageSize) res, runErr := sweepDuplicates(ctx, q, sweep.ID, cfg, pageSize)
finishCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), duplicateSweepFinishTimeout) finishCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), duplicateSweepFinishTimeout)
defer cancel() defer cancel()
@@ -98,7 +108,7 @@ func runDuplicateSweep(
} }
func sweepDuplicates( func sweepDuplicates(
ctx context.Context, q *dbq.Queries, sweepID pgtype.UUID, pageSize int32, ctx context.Context, q *dbq.Queries, sweepID pgtype.UUID, cfg FingerprintSettings, pageSize int32,
) (DuplicateSweepResult, error) { ) (DuplicateSweepResult, error) {
var res DuplicateSweepResult var res DuplicateSweepResult
@@ -119,9 +129,12 @@ func sweepDuplicates(
// Acoustic tier, streamed in duration order. The first member of an exact // Acoustic tier, streamed in duration order. The first member of an exact
// group the stream meets stands in for the whole group; the rest are skipped. // group the stream meets stands in for the whole group; the rest are skipped.
grouper := newStreamGrouper(defaultAcousticMaxBitErrorRate) // Only prints at the current length are streamed: a print at another length
// cannot be compared, and is waiting on the backfill to be re-derived.
grouper := newStreamGrouper(cfg.AcousticMaxBitErrorRate)
params := dbq.ListDuplicateCandidatesParams{ params := dbq.ListDuplicateCandidatesParams{
CurrentVersion: fingerprintVersion, CurrentVersion: fingerprintVersion,
ChromaprintLengthSec: cfg.ChromaprintLengthSec,
// Durations are never negative, and the all-zero uuid sorts first: every // Durations are never negative, and the all-zero uuid sorts first: every
// row is after this cursor. Valid must be true, or "> NULL" matches nothing. // row is after this cursor. Valid must be true, or "> NULL" matches nothing.
AfterDurationMs: -1, AfterDurationMs: -1,
@@ -262,7 +275,9 @@ func formatUUIDs(ids []pgtype.UUID) []string {
// running, reaping a sweep that has been in flight past // running, reaping a sweep that has been in flight past
// staleDuplicateSweepThreshold. Mirrors TryStartScan. The sweep runs on ctx, so // staleDuplicateSweepThreshold. Mirrors TryStartScan. The sweep runs on ctx, so
// a caller answering an HTTP request must pass a context that outlives it. // a caller answering an HTTP request must pass a context that outlives it.
func TryStartDuplicateSweep(ctx context.Context, pool *pgxpool.Pool, logger *slog.Logger) (bool, error) { func TryStartDuplicateSweep(
ctx context.Context, pool *pgxpool.Pool, logger *slog.Logger, cfg FingerprintSettings,
) (bool, error) {
q := dbq.New(pool) q := dbq.New(pool)
row, err := q.GetInFlightDuplicateSweep(ctx) row, err := q.GetInFlightDuplicateSweep(ctx)
switch { switch {
@@ -282,23 +297,28 @@ func TryStartDuplicateSweep(ctx context.Context, pool *pgxpool.Pool, logger *slo
} }
go func() { go func() {
if _, err := RunDuplicateSweep(ctx, pool, logger); err != nil { if _, err := RunDuplicateSweep(ctx, pool, logger, cfg); err != nil {
logger.Warn("duplicate sweep failed", "err", err) logger.Warn("duplicate sweep failed", "err", err)
} }
}() }()
return true, nil return true, nil
} }
// DuplicateSweepWorker sweeps whenever fingerprints have changed. // DuplicateSweepWorker sweeps whenever its input has changed, at most once per
// the operator's sweep interval.
type DuplicateSweepWorker struct { type DuplicateSweepWorker struct {
pool *pgxpool.Pool pool *pgxpool.Pool
logger *slog.Logger logger *slog.Logger
tick time.Duration settings *FingerprintSettingsService
tick time.Duration
} }
// NewDuplicateSweepWorker builds a worker with the production cadence. // NewDuplicateSweepWorker builds a worker with the production cadence. settings
func NewDuplicateSweepWorker(pool *pgxpool.Pool, logger *slog.Logger) *DuplicateSweepWorker { // is shared with the admin API; nil runs on defaults.
return &DuplicateSweepWorker{pool: pool, logger: logger, tick: duplicateSweepTick} func NewDuplicateSweepWorker(
pool *pgxpool.Pool, logger *slog.Logger, settings *FingerprintSettingsService,
) *DuplicateSweepWorker {
return &DuplicateSweepWorker{pool: pool, logger: logger, settings: settings, tick: duplicateSweepTick}
} }
// Run blocks until ctx is cancelled, checking once at start and then each tick. // Run blocks until ctx is cancelled, checking once at start and then each tick.
@@ -323,7 +343,8 @@ func (w *DuplicateSweepWorker) tickOnce(ctx context.Context) {
w.logger.Error("duplicate sweep: tick panicked", "panic", r) w.logger.Error("duplicate sweep: tick panicked", "panic", r)
} }
}() }()
due, err := duplicateSweepDue(ctx, dbq.New(w.pool)) cfg := w.settings.Get()
due, err := duplicateSweepDue(ctx, dbq.New(w.pool), cfg, time.Now())
if err != nil { if err != nil {
if ctx.Err() == nil { if ctx.Err() == nil {
w.logger.Warn("duplicate sweep: due check failed", "err", err) w.logger.Warn("duplicate sweep: due check failed", "err", err)
@@ -333,28 +354,45 @@ func (w *DuplicateSweepWorker) tickOnce(ctx context.Context) {
if !due { if !due {
return return
} }
if _, err := TryStartDuplicateSweep(ctx, w.pool, w.logger); err != nil { if _, err := TryStartDuplicateSweep(ctx, w.pool, w.logger, cfg); err != nil {
w.logger.Warn("duplicate sweep: start failed", "err", err) w.logger.Warn("duplicate sweep: start failed", "err", err)
} }
} }
// duplicateSweepDue reports whether any fingerprint was written after the latest // duplicateSweepDue reads what sweepIsDue decides on.
// sweep started. Fingerprints are the sweep's only input, so nothing else can func duplicateSweepDue(ctx context.Context, q *dbq.Queries, cfg FingerprintSettings, now time.Time) (bool, error) {
// change its answer; while the backfill is running this is true every tick.
func duplicateSweepDue(ctx context.Context, q *dbq.Queries) (bool, error) {
latest, err := q.GetLatestFingerprintComputedAt(ctx) latest, err := q.GetLatestFingerprintComputedAt(ctx)
if err != nil { if err != nil {
return false, fmt.Errorf("latest fingerprint: %w", err) return false, fmt.Errorf("latest fingerprint: %w", err)
} }
if !latest.Valid { var lastStart pgtype.Timestamptz
return false, nil // nothing fingerprinted yet
}
last, err := q.GetLatestDuplicateSweep(ctx) last, err := q.GetLatestDuplicateSweep(ctx)
if errors.Is(err, pgx.ErrNoRows) { switch {
return true, nil case err == nil:
} lastStart = last.StartedAt
if err != nil { case !errors.Is(err, pgx.ErrNoRows):
return false, fmt.Errorf("latest duplicate sweep: %w", err) return false, fmt.Errorf("latest duplicate sweep: %w", err)
} }
return latest.Time.After(last.StartedAt.Time), nil return sweepIsDue(latest, lastStart, cfg, now), nil
}
// sweepIsDue reports whether a sweep should start: something it reads has
// changed since the last sweep started, and the operator's interval has passed.
//
// Two things can change its answer. Fingerprints are its input, so any written
// after the last sweep began count; while the backfill runs that is true every
// tick, which is what the interval is for. And a settings save counts, because a
// new threshold or length changes what the same fingerprints group into.
func sweepIsDue(latestFingerprint, lastSweepStart pgtype.Timestamptz, cfg FingerprintSettings, now time.Time) bool {
if !latestFingerprint.Valid {
return false // nothing fingerprinted yet
}
if !lastSweepStart.Valid {
return true // never swept
}
interval := time.Duration(cfg.SweepIntervalHours) * time.Hour
if now.Sub(lastSweepStart.Time) < interval-sweepIntervalSlack {
return false
}
return latestFingerprint.Time.After(lastSweepStart.Time) || cfg.UpdatedAt.After(lastSweepStart.Time)
} }
+80 -3
View File
@@ -9,6 +9,9 @@ import (
"sort" "sort"
"strings" "strings"
"testing" "testing"
"time"
"github.com/jackc/pgx/v5/pgtype"
"git.fabledsword.com/bvandeusen/minstrel/internal/db/dbq" "git.fabledsword.com/bvandeusen/minstrel/internal/db/dbq"
syncpkg "git.fabledsword.com/bvandeusen/minstrel/internal/sync" syncpkg "git.fabledsword.com/bvandeusen/minstrel/internal/sync"
@@ -58,6 +61,7 @@ func TestDuplicateSweep_Integration(t *testing.T) {
} }
if err := q.UpsertTrackFingerprint(ctx, dbq.UpsertTrackFingerprintParams{ if err := q.UpsertTrackFingerprint(ctx, dbq.UpsertTrackFingerprintParams{
TrackID: tr.ID, AudioStreamSha256: sum, Chromaprint: print, FingerprintVersion: fingerprintVersion, TrackID: tr.ID, AudioStreamSha256: sum, Chromaprint: print, FingerprintVersion: fingerprintVersion,
ChromaprintLengthSec: defaultChromaprintLengthSec,
}); err != nil { }); err != nil {
t.Fatalf("fingerprint %s: %v", name, err) t.Fatalf("fingerprint %s: %v", name, err)
} }
@@ -105,7 +109,7 @@ func TestDuplicateSweep_Integration(t *testing.T) {
} }
// 1. A page size of one forces the keyset cursor across every candidate. // 1. A page size of one forces the keyset cursor across every candidate.
res, err := runDuplicateSweep(ctx, pool, logger, 1) res, err := runDuplicateSweep(ctx, pool, logger, DefaultFingerprintSettings, 1)
if err != nil { if err != nil {
t.Fatalf("first sweep: %v", err) t.Fatalf("first sweep: %v", err)
} }
@@ -133,7 +137,7 @@ func TestDuplicateSweep_Integration(t *testing.T) {
if _, err := pool.Exec(ctx, "UPDATE duplicate_groups SET status = 'dismissed' WHERE member_key = $1", acousticKey); err != nil { if _, err := pool.Exec(ctx, "UPDATE duplicate_groups SET status = 'dismissed' WHERE member_key = $1", acousticKey); err != nil {
t.Fatalf("dismiss: %v", err) t.Fatalf("dismiss: %v", err)
} }
res, err = runDuplicateSweep(ctx, pool, logger, duplicateCandidatePage) res, err = runDuplicateSweep(ctx, pool, logger, DefaultFingerprintSettings, duplicateCandidatePage)
if err != nil { if err != nil {
t.Fatalf("second sweep: %v", err) t.Fatalf("second sweep: %v", err)
} }
@@ -150,7 +154,7 @@ func TestDuplicateSweep_Integration(t *testing.T) {
"DELETE FROM track_fingerprints f USING tracks t WHERE f.track_id = t.id AND t.file_path LIKE '%www-02.mp3'"); err != nil { "DELETE FROM track_fingerprints f USING tracks t WHERE f.track_id = t.id AND t.file_path LIKE '%www-02.mp3'"); err != nil {
t.Fatalf("drop fingerprint: %v", err) t.Fatalf("drop fingerprint: %v", err)
} }
res, err = runDuplicateSweep(ctx, pool, logger, duplicateCandidatePage) res, err = runDuplicateSweep(ctx, pool, logger, DefaultFingerprintSettings, duplicateCandidatePage)
if err != nil { if err != nil {
t.Fatalf("third sweep: %v", err) t.Fatalf("third sweep: %v", err)
} }
@@ -170,4 +174,77 @@ func TestDuplicateSweep_Integration(t *testing.T) {
if !last.FinishedAt.Valid || last.ErrorMessage != nil { if !last.FinishedAt.Valid || last.ErrorMessage != nil {
t.Fatalf("latest sweep = %+v, want finished without error", last) t.Fatalf("latest sweep = %+v, want finished without error", last)
} }
// 5. Prints taken at another length are never compared (#3913). Every print
// here was taken at the default length, so a sweep at 60s has nothing to read,
// rather than scoring 120s prints against each other as if they were 60s ones.
atOtherLength := DefaultFingerprintSettings
atOtherLength.ChromaprintLengthSec = 60
res, err = runDuplicateSweep(ctx, pool, logger, atOtherLength, duplicateCandidatePage)
if err != nil {
t.Fatalf("sweep at another length: %v", err)
}
if res.Candidates != 0 || res.Groups != 0 {
t.Fatalf("sweep at another length = %+v, want no candidates and no groups", res)
}
// 6. The sweep applies the threshold it is given. The recording's two copies
// disagree on about 5% of their bits: grouped at the default, not at 1%.
if _, err := pool.Exec(ctx, "DELETE FROM duplicate_groups"); err != nil {
t.Fatalf("clear groups: %v", err)
}
strict := DefaultFingerprintSettings
strict.AcousticMaxBitErrorRate = 0.01
res, err = runDuplicateSweep(ctx, pool, logger, strict, duplicateCandidatePage)
if err != nil {
t.Fatalf("strict sweep: %v", err)
}
if res.Groups != 0 {
t.Fatalf("sweep at a 1%% threshold = %+v, want the copies 5%% apart left ungrouped", res)
}
res, err = runDuplicateSweep(ctx, pool, logger, DefaultFingerprintSettings, duplicateCandidatePage)
if err != nil {
t.Fatalf("default sweep: %v", err)
}
if got := groups(); res.Groups != 1 || got[acousticKey] != (stored{"acoustic", "pending"}) {
t.Fatalf("sweep at the default threshold = %+v, groups %+v; want the recording's copies proposed", res, got)
}
}
func TestSweepIsDue(t *testing.T) {
now := time.Date(2026, 9, 11, 12, 0, 0, 0, time.UTC)
at := func(ago time.Duration) pgtype.Timestamptz {
return pgtype.Timestamptz{Time: now.Add(-ago), Valid: true}
}
never := pgtype.Timestamptz{}
hourly := DefaultFingerprintSettings
daily := DefaultFingerprintSettings
daily.SweepIntervalHours = 24
savedAgo := func(ago time.Duration) FingerprintSettings {
s := DefaultFingerprintSettings
s.UpdatedAt = now.Add(-ago)
return s
}
for _, tc := range []struct {
name string
latestPrint, lastSweep pgtype.Timestamptz
cfg FingerprintSettings
want bool
}{
{"nothing fingerprinted", never, never, hourly, false},
{"never swept", at(time.Minute), never, hourly, true},
{"new fingerprints since the last sweep", at(10 * time.Minute), at(2 * time.Hour), hourly, true},
{"nothing new since the last sweep", at(3 * time.Hour), at(2 * time.Hour), hourly, false},
// The sweep started a moment after the previous tick, so one tick later
// it is a moment under an hour old. Without the slack this is false.
{"one tick after an hourly sweep", at(time.Minute), at(time.Hour - 2*time.Second), hourly, true},
{"new fingerprints inside the interval", at(time.Minute), at(3 * time.Hour), daily, false},
{"new fingerprints past the interval", at(time.Minute), at(25 * time.Hour), daily, true},
{"settings saved since the last sweep", at(3 * time.Hour), at(2 * time.Hour), savedAgo(time.Hour), true},
{"settings saved before the last sweep", at(3 * time.Hour), at(2 * time.Hour), savedAgo(4 * time.Hour), false},
} {
if got := sweepIsDue(tc.latestPrint, tc.lastSweep, tc.cfg, now); got != tc.want {
t.Errorf("%s: sweepIsDue = %v, want %v", tc.name, got, tc.want)
}
}
} }
+48 -23
View File
@@ -50,21 +50,28 @@ const fingerprintTimeout = 60 * time.Second
const fingerprintWaitDelay = 5 * time.Second const fingerprintWaitDelay = 5 * time.Second
// fingerprintVersion stamps how a track_fingerprints row was derived. Bump it // fingerprintVersion stamps how a track_fingerprints row was derived. Bump it
// whenever the derivation changes — the hash arguments, fpcalc's flags or its // whenever the derivation changes — the hash arguments or fpcalc's flags — and
// length — and the backfill re-derives every row below it. Fingerprints taken // the backfill re-derives every row below it. Fingerprints taken by two methods
// by two methods are not comparable, and nothing else would reveal that the // are not comparable, and nothing else would reveal that the library held a mix.
// library held a mix. //
// The length is deliberately not part of it: it is an operator setting (#3913),
// so each row records the length it was taken at and readers compare only rows
// at the current one. See fingerprint_settings.go.
const fingerprintVersion int16 = 1 const fingerprintVersion int16 = 1
// errFingerprintTimeout marks a tool that ran out of time. Distinct from a // errFingerprintTimeout marks a tool that ran out of time. Distinct from a
// failed exit because a stall is a fact about the mount, not about the file. // failed exit because a stall is a fact about the mount, not about the file.
var errFingerprintTimeout = errors.New("fingerprint tool timed out") var errFingerprintTimeout = errors.New("fingerprint tool timed out")
// defaultChromaprintLengthSec is how many seconds of audio fpcalc fingerprints. // defaultChromaprintLengthSec is the shipped value of the length setting (#3913):
// 120 is fpcalc's own default. Fingerprints taken at different lengths are not // how many seconds of audio fpcalc fingerprints. 120 is fpcalc's own default.
// comparable, so changing this has to re-derive every stored one.
const defaultChromaprintLengthSec = 120 const defaultChromaprintLengthSec = 120
// errChromaprintSkipped marks a chromaprint not taken because fingerprinting is
// switched off. Inconclusive rather than a verdict: nothing was learned about
// the file.
var errChromaprintSkipped = errors.New("chromaprint skipped: fingerprinting is off")
// fpcalcStderrTail caps how much of a failing tool's stderr reaches the log. // fpcalcStderrTail caps how much of a failing tool's stderr reaches the log.
const fpcalcStderrTail = 512 const fpcalcStderrTail = 512
@@ -113,11 +120,24 @@ type fingerprintResult struct {
printErr error printErr error
} }
// computeFingerprint derives both halves for the file at path. // fingerprintOptions is what the settings decide for one attempt. Captured once
func computeFingerprint(ctx context.Context, path string) fingerprintResult { // per file, so the length a chromaprint was taken at is the length stored with it
// even if the setting changes mid-attempt.
type fingerprintOptions struct {
lengthSec int32
// chromaprint false takes the stream hash alone: a demux, no decode.
chromaprint bool
}
// computeFingerprint derives the halves opts asks for, for the file at path.
func computeFingerprint(ctx context.Context, path string, opts fingerprintOptions) fingerprintResult {
var r fingerprintResult var r fingerprintResult
r.streamSHA256, r.hashErr = computeAudioStreamSHA256(ctx, path) r.streamSHA256, r.hashErr = computeAudioStreamSHA256(ctx, path)
r.chromaprint, r.printErr = computeChromaprint(ctx, path, defaultChromaprintLengthSec) if !opts.chromaprint {
r.printErr = errChromaprintSkipped
return r
}
r.chromaprint, r.printErr = computeChromaprint(ctx, path, opts.lengthSec)
return r return r
} }
@@ -130,11 +150,13 @@ func (r fingerprintResult) inconclusive() bool {
} }
// isInconclusive names the failures that are not a verdict on the file: a // isInconclusive names the failures that are not a verdict on the file: a
// stall, a cancelled scan, and a tool that is not installed. The last matters // stall, a cancelled scan, a tool that is not installed, and a chromaprint
// outside the image — a dev binary run without fpcalc on PATH must not stamp // skipped because fingerprinting is off. A missing tool matters outside the
// every track in the library as unfingerprintable. // image — a dev binary run without fpcalc on PATH must not stamp every track in
// the library as unfingerprintable.
func isInconclusive(err error) bool { func isInconclusive(err error) bool {
return errors.Is(err, errFingerprintTimeout) || return errors.Is(err, errFingerprintTimeout) ||
errors.Is(err, errChromaprintSkipped) ||
errors.Is(err, context.Canceled) || errors.Is(err, context.Canceled) ||
errors.Is(err, context.DeadlineExceeded) || errors.Is(err, context.DeadlineExceeded) ||
errors.Is(err, exec.ErrNotFound) errors.Is(err, exec.ErrNotFound)
@@ -142,11 +164,11 @@ func isInconclusive(err error) bool {
// fingerprintFile runs the scanner's fingerprinter. A Scanner built without New // fingerprintFile runs the scanner's fingerprinter. A Scanner built without New
// gets the real tools rather than a nil-func panic halfway through a scan. // gets the real tools rather than a nil-func panic halfway through a scan.
func (s *Scanner) fingerprintFile(ctx context.Context, path string) fingerprintResult { func (s *Scanner) fingerprintFile(ctx context.Context, path string, opts fingerprintOptions) fingerprintResult {
if s.fingerprint == nil { if s.fingerprint == nil {
return computeFingerprint(ctx, path) return computeFingerprint(ctx, path, opts)
} }
return s.fingerprint(ctx, path) return s.fingerprint(ctx, path, opts)
} }
// fingerprintOutcome is what storeFingerprint did with one attempt. // fingerprintOutcome is what storeFingerprint did with one attempt.
@@ -163,9 +185,11 @@ const (
// the backfill (#3908) alike, so there is one rule for what gets written. It // the backfill (#3908) alike, so there is one rule for what gets written. It
// never fails its caller: a missing fingerprint only keeps a track out of // never fails its caller: a missing fingerprint only keeps a track out of
// duplicate detection, which is not worth dropping a scan or a pass over. // duplicate detection, which is not worth dropping a scan or a pass over.
//
// lengthSec is the length fp's chromaprint was taken at, stored with it (#3913).
func storeFingerprint( func storeFingerprint(
ctx context.Context, q *dbq.Queries, logger *slog.Logger, ctx context.Context, q *dbq.Queries, logger *slog.Logger,
trackID pgtype.UUID, path string, fp fingerprintResult, trackID pgtype.UUID, path string, fp fingerprintResult, lengthSec int32,
) fingerprintOutcome { ) fingerprintOutcome {
if fp.hashErr != nil { if fp.hashErr != nil {
logger.Warn("fingerprint: audio stream hash failed", "path", path, "err", fp.hashErr) logger.Warn("fingerprint: audio stream hash failed", "path", path, "err", fp.hashErr)
@@ -186,10 +210,11 @@ func storeFingerprint(
// is stamped at the current version so the backfill does not retry it on // is stamped at the current version so the backfill does not retry it on
// every pass. It is retried when the file changes. // every pass. It is retried when the file changes.
if err := q.UpsertTrackFingerprint(ctx, dbq.UpsertTrackFingerprintParams{ if err := q.UpsertTrackFingerprint(ctx, dbq.UpsertTrackFingerprintParams{
TrackID: trackID, TrackID: trackID,
AudioStreamSha256: fp.streamSHA256, AudioStreamSha256: fp.streamSHA256,
Chromaprint: fp.chromaprint, Chromaprint: fp.chromaprint,
FingerprintVersion: fingerprintVersion, FingerprintVersion: fingerprintVersion,
ChromaprintLengthSec: lengthSec,
}); err != nil { }); err != nil {
logger.Warn("fingerprint: storing fingerprint failed", "path", path, "err", err) logger.Warn("fingerprint: storing fingerprint failed", "path", path, "err", err)
return outcomeStoreFailed return outcomeStoreFailed
@@ -211,8 +236,8 @@ func computeAudioStreamSHA256(ctx context.Context, path string) ([]byte, error)
// computeChromaprint returns the raw acoustic fingerprint of the first // computeChromaprint returns the raw acoustic fingerprint of the first
// lengthSec seconds of the file. // lengthSec seconds of the file.
func computeChromaprint(ctx context.Context, path string, lengthSec int) ([]int32, error) { func computeChromaprint(ctx context.Context, path string, lengthSec int32) ([]int32, error) {
out, err := runFingerprintTool(ctx, "fpcalc", fpcalcArgs(path, lengthSec)) out, err := runFingerprintTool(ctx, "fpcalc", fpcalcArgs(path, int(lengthSec)))
if err != nil { if err != nil {
return nil, err return nil, err
} }
+52 -24
View File
@@ -40,10 +40,11 @@ const fingerprintBackfillTick = time.Hour
// so tracks the scan adds mid-pass are not stuck behind one enormous page. // so tracks the scan adds mid-pass are not stuck behind one enormous page.
const fingerprintBackfillBatch = 50 const fingerprintBackfillBatch = 50
// fingerprintBackfillConcurrency is how many files are decoded at once. Two is // fingerprintBackfillConcurrency is the shipped value of the concurrency setting
// deliberately low: fpcalc and the stream hash compete with playback transcoding // (#3913): how many files are decoded at once. Two is deliberately low: fpcalc
// for CPU and with streaming for the mount, and a backfill that makes playback // and the stream hash compete with playback transcoding for CPU and with
// stutter is worse than one that takes longer. Operator-tunable in #3913. // streaming for the mount, and a backfill that makes playback stutter is worse
// than one that takes longer.
const fingerprintBackfillConcurrency = 2 const fingerprintBackfillConcurrency = 2
// BackfillFingerprintsResult tallies one pass. // BackfillFingerprintsResult tallies one pass.
@@ -68,24 +69,27 @@ func (r *BackfillFingerprintsResult) add(o fingerprintOutcome) {
// FingerprintBackfillWorker fingerprints the tracks the scan never will. // FingerprintBackfillWorker fingerprints the tracks the scan never will.
type FingerprintBackfillWorker struct { type FingerprintBackfillWorker struct {
pool *pgxpool.Pool pool *pgxpool.Pool
logger *slog.Logger logger *slog.Logger
tick time.Duration settings *FingerprintSettingsService
batch int32 tick time.Duration
concurrency int batch int32
// fingerprint is a field for the same reason as Scanner.fingerprint: an // fingerprint is a field for the same reason as Scanner.fingerprint: an
// integration test pins which tracks a pass touches, not what the tools print. // integration test pins which tracks a pass touches, not what the tools print.
fingerprint func(ctx context.Context, path string) fingerprintResult fingerprint func(ctx context.Context, path string, opts fingerprintOptions) fingerprintResult
} }
// NewFingerprintBackfillWorker builds a worker with the production cadence. // NewFingerprintBackfillWorker builds a worker with the production cadence.
func NewFingerprintBackfillWorker(pool *pgxpool.Pool, logger *slog.Logger) *FingerprintBackfillWorker { // settings is shared with the scanner and the admin API; nil runs on defaults.
func NewFingerprintBackfillWorker(
pool *pgxpool.Pool, logger *slog.Logger, settings *FingerprintSettingsService,
) *FingerprintBackfillWorker {
return &FingerprintBackfillWorker{ return &FingerprintBackfillWorker{
pool: pool, pool: pool,
logger: logger, logger: logger,
settings: settings,
tick: fingerprintBackfillTick, tick: fingerprintBackfillTick,
batch: fingerprintBackfillBatch, batch: fingerprintBackfillBatch,
concurrency: fingerprintBackfillConcurrency,
fingerprint: computeFingerprint, fingerprint: computeFingerprint,
} }
} }
@@ -129,6 +133,12 @@ func (w *FingerprintBackfillWorker) runOnce(ctx context.Context) {
// cursor is what lets a pass end: an inconclusive attempt writes no row, so a // cursor is what lets a pass end: an inconclusive attempt writes no row, so a
// file that keeps timing out would otherwise be listed again immediately and // file that keeps timing out would otherwise be listed again immediately and
// retried forever within the pass. // retried forever within the pass.
//
// Settings are read before every batch, so a save takes effect within a batch
// rather than an hour (#3913): switching fingerprinting off ends the pass, a new
// concurrency applies to the next batch, and a new length restarts the walk from
// the top at that length, because every row written at the old one went stale
// the moment it changed.
func (w *FingerprintBackfillWorker) pass(ctx context.Context) (BackfillFingerprintsResult, error) { func (w *FingerprintBackfillWorker) pass(ctx context.Context) (BackfillFingerprintsResult, error) {
q := dbq.New(w.pool) q := dbq.New(w.pool)
var ( var (
@@ -137,15 +147,25 @@ func (w *FingerprintBackfillWorker) pass(ctx context.Context) (BackfillFingerpri
) )
// The all-zero uuid sorts before every real id. Valid must be true: a NULL // The all-zero uuid sorts before every real id. Valid must be true: a NULL
// cursor would make "id > NULL" match nothing and every pass a silent no-op. // cursor would make "id > NULL" match nothing and every pass a silent no-op.
after := pgtype.UUID{Valid: true} start := pgtype.UUID{Valid: true}
after := start
lengthSec := w.settings.Get().ChromaprintLengthSec
for { for {
if err := ctx.Err(); err != nil { if err := ctx.Err(); err != nil {
return res, err return res, err
} }
cfg := w.settings.Get()
if !cfg.Enabled {
return res, nil
}
if cfg.ChromaprintLengthSec != lengthSec {
lengthSec, after = cfg.ChromaprintLengthSec, start
}
rows, err := q.ListTracksNeedingFingerprint(ctx, dbq.ListTracksNeedingFingerprintParams{ rows, err := q.ListTracksNeedingFingerprint(ctx, dbq.ListTracksNeedingFingerprintParams{
CurrentVersion: fingerprintVersion, CurrentVersion: fingerprintVersion,
AfterID: after, ChromaprintLengthSec: lengthSec,
BatchLimit: w.batch, AfterID: after,
BatchLimit: w.batch,
}) })
if err != nil { if err != nil {
return res, fmt.Errorf("list tracks needing fingerprint: %w", err) return res, fmt.Errorf("list tracks needing fingerprint: %w", err)
@@ -154,7 +174,10 @@ func (w *FingerprintBackfillWorker) pass(ctx context.Context) (BackfillFingerpri
return res, nil return res, nil
} }
sem := make(chan struct{}, w.concurrency) opts := fingerprintOptions{lengthSec: lengthSec, chromaprint: true}
// Validation keeps concurrency at one or more; the floor guards a zero
// that would block the first send for ever.
sem := make(chan struct{}, max(1, int(cfg.BackfillConcurrency)))
var wg sync.WaitGroup var wg sync.WaitGroup
for _, row := range rows { for _, row := range rows {
if ctx.Err() != nil { if ctx.Err() != nil {
@@ -170,7 +193,7 @@ func (w *FingerprintBackfillWorker) pass(ctx context.Context) (BackfillFingerpri
w.logger.Error("fingerprint backfill: track panicked", "path", path, "panic", r) w.logger.Error("fingerprint backfill: track panicked", "path", path, "panic", r)
} }
}() }()
outcome := storeFingerprint(ctx, q, w.logger, trackID, path, w.fingerprintFile(ctx, path)) outcome := storeFingerprint(ctx, q, w.logger, trackID, path, w.fingerprintFile(ctx, path, opts), lengthSec)
mu.Lock() mu.Lock()
res.add(outcome) res.add(outcome)
mu.Unlock() mu.Unlock()
@@ -181,16 +204,21 @@ func (w *FingerprintBackfillWorker) pass(ctx context.Context) (BackfillFingerpri
} }
} }
func (w *FingerprintBackfillWorker) fingerprintFile(ctx context.Context, path string) fingerprintResult { func (w *FingerprintBackfillWorker) fingerprintFile(ctx context.Context, path string, opts fingerprintOptions) fingerprintResult {
if w.fingerprint == nil { if w.fingerprint == nil {
return computeFingerprint(ctx, path) return computeFingerprint(ctx, path, opts)
} }
return w.fingerprint(ctx, path) return w.fingerprint(ctx, path, opts)
} }
// FingerprintCoverage reports how much of the library carries a current // FingerprintCoverage reports how much of the library carries a current
// fingerprint, for the admin gauge. It lives here, beside the backfill, so the // fingerprint, for the admin gauge. It lives here, beside the backfill, so the
// version it counts against is the one the backfill writes. // version and length it counts against are the ones the backfill writes.
func FingerprintCoverage(ctx context.Context, pool *pgxpool.Pool) (dbq.GetFingerprintCoverageRow, error) { func FingerprintCoverage(
return dbq.New(pool).GetFingerprintCoverage(ctx, fingerprintVersion) ctx context.Context, pool *pgxpool.Pool, cfg FingerprintSettings,
) (dbq.GetFingerprintCoverageRow, error) {
return dbq.New(pool).GetFingerprintCoverage(ctx, dbq.GetFingerprintCoverageParams{
CurrentVersion: fingerprintVersion,
ChromaprintLengthSec: cfg.ChromaprintLengthSec,
})
} }
+74 -4
View File
@@ -7,6 +7,7 @@ import (
"fmt" "fmt"
"io" "io"
"log/slog" "log/slog"
"maps"
"path/filepath" "path/filepath"
"sync" "sync"
"testing" "testing"
@@ -48,7 +49,7 @@ func TestFingerprintBackfill_Integration(t *testing.T) {
} { } {
if err := q.UpsertTrackFingerprint(ctx, dbq.UpsertTrackFingerprintParams{ if err := q.UpsertTrackFingerprint(ctx, dbq.UpsertTrackFingerprintParams{
TrackID: seed.track.ID, AudioStreamSha256: sum, Chromaprint: []int32{1}, TrackID: seed.track.ID, AudioStreamSha256: sum, Chromaprint: []int32{1},
FingerprintVersion: seed.version, FingerprintVersion: seed.version, ChromaprintLengthSec: defaultChromaprintLengthSec,
}); err != nil { }); err != nil {
t.Fatalf("seed fingerprint: %v", err) t.Fatalf("seed fingerprint: %v", err)
} }
@@ -59,13 +60,24 @@ func TestFingerprintBackfill_Integration(t *testing.T) {
var mu sync.Mutex var mu sync.Mutex
calls := map[string]int{} calls := map[string]int{}
w := NewFingerprintBackfillWorker(pool, slog.New(slog.NewTextHandler(io.Discard, nil))) settings, err := NewFingerprintSettingsService(ctx, pool)
if err != nil {
t.Fatalf("fingerprint settings: %v", err)
}
t.Cleanup(func() {
if _, err := settings.Set(context.Background(), DefaultFingerprintSettings); err != nil {
t.Errorf("restore fingerprint settings: %v", err)
}
})
w := NewFingerprintBackfillWorker(pool, slog.New(slog.NewTextHandler(io.Discard, nil)), settings)
// A batch of one forces the keyset cursor across several queries in a pass. // A batch of one forces the keyset cursor across several queries in a pass.
w.batch = 1 w.batch = 1
w.fingerprint = func(_ context.Context, path string) fingerprintResult { lengths := map[int32]int{}
w.fingerprint = func(_ context.Context, path string, opts fingerprintOptions) fingerprintResult {
name := filepath.Base(path) name := filepath.Base(path)
mu.Lock() mu.Lock()
calls[name]++ calls[name]++
lengths[opts.lengthSec]++
mu.Unlock() mu.Unlock()
switch name { switch name {
case "stall.mp3": case "stall.mp3":
@@ -126,7 +138,7 @@ func TestFingerprintBackfill_Integration(t *testing.T) {
} }
// 4. The gauge counts what the passes wrote, and its buckets add up. // 4. The gauge counts what the passes wrote, and its buckets add up.
cov, err := FingerprintCoverage(ctx, pool) cov, err := FingerprintCoverage(ctx, pool, settings.Get())
if err != nil { if err != nil {
t.Fatalf("coverage: %v", err) t.Fatalf("coverage: %v", err)
} }
@@ -138,6 +150,64 @@ func TestFingerprintBackfill_Integration(t *testing.T) {
if cov.Fingerprinted+cov.Rejected+cov.Pending != cov.Total { if cov.Fingerprinted+cov.Rejected+cov.Pending != cov.Total {
t.Errorf("coverage buckets %+v do not sum to the total", cov) t.Errorf("coverage buckets %+v do not sum to the total", cov)
} }
// 5. Changing the length (#3913) makes every stored row stale at once — the
// gauge shows the whole library pending before the backfill has touched a
// file — and the next pass re-derives each at the new length. The failure
// this prevents is invisible from the UI: prints at two lengths that silently
// never match.
shorter := DefaultFingerprintSettings
shorter.ChromaprintLengthSec = 60
if _, err := settings.Set(ctx, shorter); err != nil {
t.Fatalf("change length: %v", err)
}
cov, err = FingerprintCoverage(ctx, pool, settings.Get())
if err != nil {
t.Fatalf("coverage after length change: %v", err)
}
if cov.Total != 5 || cov.Pending != 5 {
t.Fatalf("coverage after length change = %+v, want all 5 tracks pending", cov)
}
mu.Lock()
clear(lengths)
mu.Unlock()
res, err = w.pass(ctx)
if err != nil {
t.Fatalf("new-length pass: %v", err)
}
if res.Processed != 5 {
t.Fatalf("new-length pass = %+v, want all 5 present tracks re-derived", res)
}
mu.Lock()
asked := maps.Clone(lengths)
mu.Unlock()
if len(asked) != 1 || asked[60] != 5 {
t.Fatalf("new-length pass asked for lengths %v, want 60s for all 5", asked)
}
var atOldLength int
if err := pool.QueryRow(ctx,
"SELECT count(*) FROM track_fingerprints WHERE chromaprint_length_sec <> 60").Scan(&atOldLength); err != nil {
t.Fatalf("count old-length rows: %v", err)
}
if atOldLength != 0 {
t.Fatalf("%d fingerprints are still at the old length after a complete pass", atOldLength)
}
// 6. Switched off, the backfill does nothing, even with work waiting.
off := DefaultFingerprintSettings
off.Enabled = false
if _, err := settings.Set(ctx, off); err != nil {
t.Fatalf("switch fingerprinting off: %v", err)
}
addTrack("later")
res, err = w.pass(ctx)
if err != nil {
t.Fatalf("pass with fingerprinting off: %v", err)
}
if res.Processed != 0 || callCount("later.mp3") != 0 {
t.Fatalf("pass with fingerprinting off = %+v (later.mp3 tried %d times), want nothing done",
res, callCount("later.mp3"))
}
} }
func TestBackfillFingerprintsResult_Add(t *testing.T) { func TestBackfillFingerprintsResult_Add(t *testing.T) {
+62 -6
View File
@@ -60,9 +60,24 @@ func TestScanner_FingerprintsOnlyNewOrChangedBytes_Integration(t *testing.T) {
result := fingerprintResult{streamSHA256: sum, chromaprint: chroma} result := fingerprintResult{streamSHA256: sum, chromaprint: chroma}
calls := map[string]int{} calls := map[string]int{}
scanner := New(pool, logger, []string{root}) settings, err := NewFingerprintSettingsService(ctx, pool)
scanner.fingerprint = func(_ context.Context, path string) fingerprintResult { if err != nil {
t.Fatalf("fingerprint settings: %v", err)
}
if _, err := settings.Set(ctx, DefaultFingerprintSettings); err != nil {
t.Fatalf("reset fingerprint settings: %v", err)
}
t.Cleanup(func() {
if _, err := settings.Set(context.Background(), DefaultFingerprintSettings); err != nil {
t.Errorf("restore fingerprint settings: %v", err)
}
})
var lastOpts fingerprintOptions
scanner := New(pool, logger, []string{root}, settings)
scanner.fingerprint = func(_ context.Context, path string, opts fingerprintOptions) fingerprintResult {
calls[path]++ calls[path]++
lastOpts = opts
return result return result
} }
@@ -78,14 +93,15 @@ func TestScanner_FingerprintsOnlyNewOrChangedBytes_Integration(t *testing.T) {
sha []byte sha []byte
chroma []int32 chroma []int32
version int16 version int16
length int32
} }
stored := func(path string) (row, bool) { stored := func(path string) (row, bool) {
t.Helper() t.Helper()
var r row var r row
err := pool.QueryRow(ctx, ` err := pool.QueryRow(ctx, `
SELECT f.audio_stream_sha256, f.chromaprint, f.fingerprint_version SELECT f.audio_stream_sha256, f.chromaprint, f.fingerprint_version, f.chromaprint_length_sec
FROM track_fingerprints f JOIN tracks t ON t.id = f.track_id FROM track_fingerprints f JOIN tracks t ON t.id = f.track_id
WHERE t.file_path = $1`, path).Scan(&r.sha, &r.chroma, &r.version) WHERE t.file_path = $1`, path).Scan(&r.sha, &r.chroma, &r.version, &r.length)
if errors.Is(err, pgx.ErrNoRows) { if errors.Is(err, pgx.ErrNoRows) {
return row{}, false return row{}, false
} }
@@ -113,8 +129,13 @@ func TestScanner_FingerprintsOnlyNewOrChangedBytes_Integration(t *testing.T) {
if !ok { if !ok {
t.Fatal("first scan stored no fingerprint") t.Fatal("first scan stored no fingerprint")
} }
if !bytes.Equal(got.sha, sum) || !slices.Equal(got.chroma, chroma) || got.version != fingerprintVersion { if !bytes.Equal(got.sha, sum) || !slices.Equal(got.chroma, chroma) || got.version != fingerprintVersion ||
t.Fatalf("stored %+v, want sha %x chromaprint %v version %d", got, sum, chroma, fingerprintVersion) got.length != defaultChromaprintLengthSec {
t.Fatalf("stored %+v, want sha %x chromaprint %v version %d length %d",
got, sum, chroma, fingerprintVersion, defaultChromaprintLengthSec)
}
if !lastOpts.chromaprint || lastOpts.lengthSec != defaultChromaprintLengthSec {
t.Fatalf("first scan asked for %+v, want a chromaprint at the default length", lastOpts)
} }
// 2. A tag-repair pass re-reads every unchanged file and must not // 2. A tag-repair pass re-reads every unchanged file and must not
@@ -171,4 +192,39 @@ func TestScanner_FingerprintsOnlyNewOrChangedBytes_Integration(t *testing.T) {
if got.sha != nil || got.chroma != nil || got.version != fingerprintVersion { if got.sha != nil || got.chroma != nil || got.version != fingerprintVersion {
t.Fatalf("rejected file stored %+v, want both halves NULL at version %d", got, fingerprintVersion) t.Fatalf("rejected file stored %+v, want both halves NULL at version %d", got, fingerprintVersion)
} }
// 6. With fingerprinting off (#3913) the scan decodes nothing. It still asks
// for the stream hash — a demux, and what recognises a moved file — but stores
// no row, and a changed file's old row goes: it describes bytes that are gone.
result = fingerprintResult{streamSHA256: sum, chromaprint: chroma}
off := DefaultFingerprintSettings
off.Enabled = false
if _, err := settings.Set(ctx, off); err != nil {
t.Fatalf("switch fingerprinting off: %v", err)
}
touch(a, 4*time.Hour)
scan("fingerprinting-off scan")
if calls[a] != 5 || lastOpts.chromaprint {
t.Fatalf("fingerprinting-off scan: calls = %v, last options %+v; want a fifth call asking for no chromaprint",
calls, lastOpts)
}
if _, ok := stored(a); ok {
t.Fatal("with fingerprinting off, a changed file kept the previous bytes' fingerprint")
}
if _, ok := stored(b); !ok {
t.Fatal("with fingerprinting off, an unchanged file lost its fingerprint")
}
// 7. The length setting reaches the scan, and is stored with the row.
longer := DefaultFingerprintSettings
longer.ChromaprintLengthSec = 90
if _, err := settings.Set(ctx, longer); err != nil {
t.Fatalf("change length: %v", err)
}
touch(a, 5*time.Hour)
scan("new-length scan")
got, ok = stored(a)
if !ok || got.length != 90 || lastOpts.lengthSec != 90 || !lastOpts.chromaprint {
t.Fatalf("new-length scan stored %+v (present %v) after asking for %+v; want a chromaprint at 90s", got, ok, lastOpts)
}
} }
+163
View File
@@ -0,0 +1,163 @@
package library
import (
"context"
"errors"
"fmt"
"sync"
"time"
"github.com/jackc/pgx/v5/pgxpool"
"git.fabledsword.com/bvandeusen/minstrel/internal/db/dbq"
)
// Fingerprinting settings (M400 #3913).
//
// Rule 25: what an operator might tune lives in the database and changes without
// a restart. One service instance is shared by the scanner, the backfill, the
// duplicate sweep and the admin API, so a save reaches all of them at once.
//
// The length is the dangerous knob: chromaprints taken at two lengths cannot be
// compared. Rather than bumping fingerprintVersion when it changes, every row
// records the length it was taken at (migration 0061) and every reader filters
// on the current one. A row at another length is stale to the backfill, pending
// to the gauge and invisible to the sweep, which is the effect a version bump
// would have, with one difference worth having: setting the length back makes
// rows not yet re-derived current again, instead of redoing the library twice.
// Bounds for each setting, mirrored by migration 0061's CHECKs.
const (
// Below 30s a print has too few items to align at the matcher's larger
// offsets and still overlap by minOverlapItems. Above 600s each print passes
// 20 KB and a candidate page of them grows past what one sweep should hold.
minChromaprintLengthSec = 30
maxChromaprintLengthSec = 600
// Unrelated audio sits near 0.5; a ceiling of 0.35 keeps a loosened
// threshold well clear of proposing noise.
minAcousticMaxBitErrorRate = 0.01
maxAcousticMaxBitErrorRate = 0.35
minBackfillConcurrency = 1
maxBackfillConcurrency = 8
minSweepIntervalHours = 1
maxSweepIntervalHours = 168
defaultSweepIntervalHours = 1
)
// FingerprintSettings mirrors the fingerprint_settings row.
type FingerprintSettings struct {
// Enabled off stops every decode: the scan takes only the stream hash (a
// demux, and what recognises a moved file) and stores nothing, and the
// backfill idles.
Enabled bool
ChromaprintLengthSec int32
AcousticMaxBitErrorRate float64
BackfillConcurrency int32
SweepIntervalHours int32
// UpdatedAt is when the settings were last saved. Set by the database;
// ignored by Set.
UpdatedAt time.Time
}
// DefaultFingerprintSettings mirrors migration 0061's column defaults, so a
// database that cannot be read still fingerprints the way a fresh install does.
var DefaultFingerprintSettings = FingerprintSettings{
Enabled: true,
ChromaprintLengthSec: defaultChromaprintLengthSec,
AcousticMaxBitErrorRate: defaultAcousticMaxBitErrorRate,
BackfillConcurrency: fingerprintBackfillConcurrency,
SweepIntervalHours: defaultSweepIntervalHours,
}
// ErrFingerprintSettingOutOfRange is returned by Set for a value migration
// 0061's CHECKs would reject, so the API answers 400 naming the field rather
// than surfacing a constraint violation.
var ErrFingerprintSettingOutOfRange = errors.New("fingerprint setting out of range")
// FingerprintSettingsService caches the settings and owns their persistence.
// Cached because the scanner reads them for every file it fingerprints.
type FingerprintSettingsService struct {
pool *pgxpool.Pool
mu sync.RWMutex
cur FingerprintSettings
}
// NewFingerprintSettingsService loads once and caches. It always returns a
// usable service, holding the defaults when the load fails; the error says so.
func NewFingerprintSettingsService(ctx context.Context, pool *pgxpool.Pool) (*FingerprintSettingsService, error) {
s := &FingerprintSettingsService{pool: pool, cur: DefaultFingerprintSettings}
row, err := dbq.New(pool).GetFingerprintSettings(ctx)
if err != nil {
return s, fmt.Errorf("fingerprint settings: load: %w", err)
}
s.cur = fingerprintSettingsFromRow(row)
return s, nil
}
// Get returns the cached settings. A nil service answers with the defaults, so
// a Scanner or worker built without one fingerprints as a fresh install would.
func (s *FingerprintSettingsService) Get() FingerprintSettings {
if s == nil {
return DefaultFingerprintSettings
}
s.mu.RLock()
defer s.mu.RUnlock()
return s.cur
}
// Set validates, persists and re-caches.
func (s *FingerprintSettingsService) Set(ctx context.Context, in FingerprintSettings) (FingerprintSettings, error) {
if err := validateFingerprintSettings(in); err != nil {
return FingerprintSettings{}, err
}
if s == nil {
return FingerprintSettings{}, errors.New("fingerprint settings: no settings service")
}
row, err := dbq.New(s.pool).UpdateFingerprintSettings(ctx, dbq.UpdateFingerprintSettingsParams{
Enabled: in.Enabled,
ChromaprintLengthSec: in.ChromaprintLengthSec,
AcousticMaxBitErrorRate: in.AcousticMaxBitErrorRate,
BackfillConcurrency: in.BackfillConcurrency,
SweepIntervalHours: in.SweepIntervalHours,
})
if err != nil {
return FingerprintSettings{}, fmt.Errorf("fingerprint settings: save: %w", err)
}
out := fingerprintSettingsFromRow(row)
s.mu.Lock()
s.cur = out
s.mu.Unlock()
return out, nil
}
func validateFingerprintSettings(in FingerprintSettings) error {
switch {
case in.ChromaprintLengthSec < minChromaprintLengthSec || in.ChromaprintLengthSec > maxChromaprintLengthSec:
return fmt.Errorf("%w: chromaprint_length_sec must be %d-%d",
ErrFingerprintSettingOutOfRange, minChromaprintLengthSec, maxChromaprintLengthSec)
// Written as a negated range so NaN, which fails every comparison, is refused.
case !(in.AcousticMaxBitErrorRate >= minAcousticMaxBitErrorRate && in.AcousticMaxBitErrorRate <= maxAcousticMaxBitErrorRate):
return fmt.Errorf("%w: acoustic_max_bit_error_rate must be %.2f-%.2f",
ErrFingerprintSettingOutOfRange, minAcousticMaxBitErrorRate, maxAcousticMaxBitErrorRate)
case in.BackfillConcurrency < minBackfillConcurrency || in.BackfillConcurrency > maxBackfillConcurrency:
return fmt.Errorf("%w: backfill_concurrency must be %d-%d",
ErrFingerprintSettingOutOfRange, minBackfillConcurrency, maxBackfillConcurrency)
case in.SweepIntervalHours < minSweepIntervalHours || in.SweepIntervalHours > maxSweepIntervalHours:
return fmt.Errorf("%w: sweep_interval_hours must be %d-%d",
ErrFingerprintSettingOutOfRange, minSweepIntervalHours, maxSweepIntervalHours)
}
return nil
}
func fingerprintSettingsFromRow(row dbq.FingerprintSetting) FingerprintSettings {
return FingerprintSettings{
Enabled: row.Enabled,
ChromaprintLengthSec: row.ChromaprintLengthSec,
AcousticMaxBitErrorRate: row.AcousticMaxBitErrorRate,
BackfillConcurrency: row.BackfillConcurrency,
SweepIntervalHours: row.SweepIntervalHours,
UpdatedAt: row.UpdatedAt.Time,
}
}
@@ -0,0 +1,146 @@
package library
import (
"context"
"errors"
"math"
"testing"
"time"
)
func TestValidateFingerprintSettings(t *testing.T) {
with := func(edit func(*FingerprintSettings)) FingerprintSettings {
s := DefaultFingerprintSettings
edit(&s)
return s
}
valid := map[string]FingerprintSettings{
"defaults": DefaultFingerprintSettings,
"shortest length": with(func(s *FingerprintSettings) { s.ChromaprintLengthSec = minChromaprintLengthSec }),
"longest length": with(func(s *FingerprintSettings) { s.ChromaprintLengthSec = maxChromaprintLengthSec }),
"strictest match": with(func(s *FingerprintSettings) { s.AcousticMaxBitErrorRate = minAcousticMaxBitErrorRate }),
"loosest match": with(func(s *FingerprintSettings) { s.AcousticMaxBitErrorRate = maxAcousticMaxBitErrorRate }),
"fewest at once": with(func(s *FingerprintSettings) { s.BackfillConcurrency = minBackfillConcurrency }),
"most at once": with(func(s *FingerprintSettings) { s.BackfillConcurrency = maxBackfillConcurrency }),
"shortest interval": with(func(s *FingerprintSettings) { s.SweepIntervalHours = minSweepIntervalHours }),
"longest interval": with(func(s *FingerprintSettings) { s.SweepIntervalHours = maxSweepIntervalHours }),
"switched off": with(func(s *FingerprintSettings) { s.Enabled = false }),
}
for name, s := range valid {
if err := validateFingerprintSettings(s); err != nil {
t.Errorf("%s: rejected: %v", name, err)
}
}
invalid := map[string]FingerprintSettings{
"length too short": with(func(s *FingerprintSettings) { s.ChromaprintLengthSec = minChromaprintLengthSec - 1 }),
"length too long": with(func(s *FingerprintSettings) { s.ChromaprintLengthSec = maxChromaprintLengthSec + 1 }),
"match too strict": with(func(s *FingerprintSettings) { s.AcousticMaxBitErrorRate = minAcousticMaxBitErrorRate - 0.001 }),
"match too loose": with(func(s *FingerprintSettings) { s.AcousticMaxBitErrorRate = maxAcousticMaxBitErrorRate + 0.001 }),
"match not a number": with(func(s *FingerprintSettings) { s.AcousticMaxBitErrorRate = math.NaN() }),
"none at once": with(func(s *FingerprintSettings) { s.BackfillConcurrency = 0 }),
"too many at once": with(func(s *FingerprintSettings) { s.BackfillConcurrency = maxBackfillConcurrency + 1 }),
"no interval": with(func(s *FingerprintSettings) { s.SweepIntervalHours = 0 }),
"interval too long": with(func(s *FingerprintSettings) { s.SweepIntervalHours = maxSweepIntervalHours + 1 }),
}
for name, s := range invalid {
if err := validateFingerprintSettings(s); !errors.Is(err, ErrFingerprintSettingOutOfRange) {
t.Errorf("%s: err = %v, want ErrFingerprintSettingOutOfRange", name, err)
}
}
}
func TestFingerprintSettingsService_NilServesDefaults(t *testing.T) {
var s *FingerprintSettingsService
if got := s.Get(); got != DefaultFingerprintSettings {
t.Fatalf("nil service Get = %+v, want the defaults", got)
}
// Validation still runs first, so a bad value is named rather than hidden
// behind the missing service.
bad := DefaultFingerprintSettings
bad.BackfillConcurrency = 0
if _, err := s.Set(context.Background(), bad); !errors.Is(err, ErrFingerprintSettingOutOfRange) {
t.Fatalf("nil service Set of a bad value: err = %v, want ErrFingerprintSettingOutOfRange", err)
}
}
func TestFingerprintSettingsService_Integration(t *testing.T) {
pool := newPool(t)
ctx := context.Background()
reload := func(step string) FingerprintSettings {
t.Helper()
fresh, err := NewFingerprintSettingsService(ctx, pool)
if err != nil {
t.Fatalf("%s: load: %v", step, err)
}
return fresh.Get()
}
withoutTime := func(s FingerprintSettings) FingerprintSettings {
s.UpdatedAt = time.Time{}
return s
}
svc, err := NewFingerprintSettingsService(ctx, pool)
if err != nil {
t.Fatalf("load: %v", err)
}
// ResetDB puts every column back to its migration default, so this pins the
// Go defaults to migration 0061's. Were they to drift, a database that could
// not be read would fingerprint differently from one that could.
loaded := svc.Get()
if loaded.UpdatedAt.IsZero() {
t.Fatal("loaded settings carry no updated_at")
}
if withoutTime(loaded) != DefaultFingerprintSettings {
t.Fatalf("stored defaults = %+v, want the Go defaults %+v", withoutTime(loaded), DefaultFingerprintSettings)
}
// Every bound the service accepts, the table accepts too. A CHECK tighter
// than validate would turn a value the card allows into a 500.
lowest := FingerprintSettings{
Enabled: false,
ChromaprintLengthSec: minChromaprintLengthSec,
AcousticMaxBitErrorRate: minAcousticMaxBitErrorRate,
BackfillConcurrency: minBackfillConcurrency,
SweepIntervalHours: minSweepIntervalHours,
}
highest := FingerprintSettings{
Enabled: true,
ChromaprintLengthSec: maxChromaprintLengthSec,
AcousticMaxBitErrorRate: maxAcousticMaxBitErrorRate,
BackfillConcurrency: maxBackfillConcurrency,
SweepIntervalHours: maxSweepIntervalHours,
}
for _, step := range []struct {
name string
want FingerprintSettings
}{{"lowest", lowest}, {"highest", highest}} {
saved, err := svc.Set(ctx, step.want)
if err != nil {
t.Fatalf("%s: save: %v", step.name, err)
}
// A save must move updated_at forward: it is what makes a sweep due.
if !saved.UpdatedAt.After(loaded.UpdatedAt) {
t.Fatalf("%s: updated_at %v did not move past %v", step.name, saved.UpdatedAt, loaded.UpdatedAt)
}
if withoutTime(saved) != step.want || withoutTime(svc.Get()) != step.want {
t.Fatalf("%s: saved %+v, cached %+v, want %+v", step.name, saved, svc.Get(), step.want)
}
if got := withoutTime(reload(step.name)); got != step.want {
t.Fatalf("%s: table holds %+v, want %+v", step.name, got, step.want)
}
}
// An out-of-range save changes nothing, in the cache or the table.
before := svc.Get()
bad := highest
bad.ChromaprintLengthSec = maxChromaprintLengthSec + 1
if _, err := svc.Set(ctx, bad); !errors.Is(err, ErrFingerprintSettingOutOfRange) {
t.Fatalf("out-of-range save: err = %v, want ErrFingerprintSettingOutOfRange", err)
}
if svc.Get() != before {
t.Fatalf("out-of-range save changed the cache to %+v", svc.Get())
}
if got := reload("after out-of-range save"); got != before {
t.Fatalf("out-of-range save changed the table to %+v", got)
}
}
+22 -5
View File
@@ -79,11 +79,14 @@ type Scanner struct {
// integration test can substitute a deterministic one: CI has no real audio // integration test can substitute a deterministic one: CI has no real audio
// to fingerprint, and what the test pins is WHEN the scan fingerprints, not // to fingerprint, and what the test pins is WHEN the scan fingerprints, not
// what the tools print. Call it through fingerprintFile. // what the tools print. Call it through fingerprintFile.
fingerprint func(ctx context.Context, path string) fingerprintResult fingerprint func(ctx context.Context, path string, opts fingerprintOptions) fingerprintResult
// settings is the operator's fingerprinting policy (#3913), shared with the
// workers and the admin API. Nil fingerprints with the defaults.
settings *FingerprintSettingsService
} }
func New(pool *pgxpool.Pool, logger *slog.Logger, paths []string) *Scanner { func New(pool *pgxpool.Pool, logger *slog.Logger, paths []string, settings *FingerprintSettingsService) *Scanner {
return &Scanner{pool: pool, logger: logger, paths: paths, fingerprint: computeFingerprint} return &Scanner{pool: pool, logger: logger, paths: paths, fingerprint: computeFingerprint, settings: settings}
} }
// Scan walks every configured root and upserts any audio file whose mtime is // Scan walks every configured root and upserts any audio file whose mtime is
@@ -313,10 +316,18 @@ func (s *Scanner) scanFile(
// //
// Computed before move adoption so adoption can match on the audio hash // Computed before move adoption so adoption can match on the audio hash
// (#3914); stored after the upsert, once the row id is known. // (#3914); stored after the upsert, once the row id is known.
//
// With fingerprinting switched off (#3913) the scan decodes nothing, but it
// still takes the stream hash: a demux rather than a decode, and the only
// thing that recognises a moved file with no MBID.
var fp fingerprintResult var fp fingerprintResult
fpCfg := s.settings.Get()
fingerprinted := !unchanged fingerprinted := !unchanged
if fingerprinted { if fingerprinted {
fp = s.fingerprintFile(ctx, path) fp = s.fingerprintFile(ctx, path, fingerprintOptions{
lengthSec: fpCfg.ChromaprintLengthSec,
chromaprint: fpCfg.Enabled,
})
} }
// A path we've never seen might not be a new track — it might be one that // A path we've never seen might not be a new track — it might be one that
@@ -384,7 +395,13 @@ func (s *Scanner) scanFile(
s.logger.Warn("library scan: LogChange track upsert failed", "track_id", track.ID, "err", err) s.logger.Warn("library scan: LogChange track upsert failed", "track_id", track.ID, "err", err)
} }
if fingerprinted { if fingerprinted {
storeFingerprint(ctx, q, s.logger, track.ID, path, fp) if fpCfg.Enabled {
storeFingerprint(ctx, q, s.logger, track.ID, path, fp, fpCfg.ChromaprintLengthSec)
} else if err := q.DeleteTrackFingerprint(ctx, track.ID); err != nil {
// Off, nothing is stored — but a row describing the previous bytes
// must not outlive them, or the sweep would compare audio that is gone.
s.logger.Warn("fingerprint: clearing stale fingerprint failed", "path", path, "err", err)
}
} }
if knownTrack { if knownTrack {
+2 -2
View File
@@ -81,7 +81,7 @@ func TestScanner_Integration(t *testing.T) {
"TIT2": "Solo", "TPE1": "The Artist Y", "TALB": "Y Album", "TRCK": "1", "TIT2": "Solo", "TPE1": "The Artist Y", "TALB": "Y Album", "TRCK": "1",
}) })
scanner := New(pool, logger, []string{root}) scanner := New(pool, logger, []string{root}, nil)
stats, err := scanner.Scan(ctx, nil) stats, err := scanner.Scan(ctx, nil)
if err != nil { if err != nil {
t.Fatalf("first scan: %v", err) t.Fatalf("first scan: %v", err)
@@ -259,7 +259,7 @@ func TestScanner_AdoptsMovedFile_Integration(t *testing.T) {
}) })
} }
scanner := New(pool, logger, []string{root}) scanner := New(pool, logger, []string{root}, nil)
if _, err := scanner.Scan(ctx, nil); err != nil { if _, err := scanner.Scan(ctx, nil); err != nil {
t.Fatalf("first scan: %v", err) t.Fatalf("first scan: %v", err)
} }
+16 -1
View File
@@ -100,6 +100,11 @@ type Server struct {
// and serves the admin tuning endpoints from it. Router() constructs // and serves the admin tuning endpoints from it. Router() constructs
// a fallback when nil (tests). // a fallback when nil (tests).
RecSettings *recsettings.Service RecSettings *recsettings.Service
// FingerprintSettings is the DB-backed fingerprinting policy (M400 #3913).
// Constructed in cmd/minstrel/main.go and shared with the scanner and the
// fingerprint workers, so a save from the admin card reaches them without a
// restart. Router() constructs a fallback when nil (tests).
FingerprintSettings *library.FingerprintSettingsService
// StreamSecret is the HMAC key used by /api/cast/stream-token to // StreamSecret is the HMAC key used by /api/cast/stream-token to
// mint signed UPnP / Sonos stream URLs and by /api/tracks/{id}/stream // mint signed UPnP / Sonos stream URLs and by /api/tracks/{id}/stream
// to verify them. Sourced from config.Config.StreamSecret. Tests that // to verify them. Sourced from config.Config.StreamSecret. Tests that
@@ -186,7 +191,17 @@ func (s *Server) Router() http.Handler {
s.Logger.Error("server: recsettings boot failed", "err", err) s.Logger.Error("server: recsettings boot failed", "err", err)
} }
} }
api.Mount(r, s.Pool, s.Logger, writer, s.RecommendationCfg, recSettings, lidarrCfg, lidarrReqs, lidarrQuar, tracksSvc, playlistsSvc, s.CoverEnricher, s.CoverSettings, s.TagSettings, s.LibraryScanner, s.ScanCfg, s.DataDir, smtpSender, bus, s.PlaylistScheduler, s.StreamSecret, netSettings, reacqSettings) fpSettings := s.FingerprintSettings
if fpSettings == nil {
// Test contexts construct Server without main.go's wiring. Always
// usable: a failed load serves the defaults.
var err error
fpSettings, err = library.NewFingerprintSettingsService(context.Background(), s.Pool)
if err != nil {
s.Logger.Warn("fingerprint settings unavailable; serving defaults", "err", err)
}
}
api.Mount(r, s.Pool, s.Logger, writer, s.RecommendationCfg, recSettings, lidarrCfg, lidarrReqs, lidarrQuar, tracksSvc, playlistsSvc, s.CoverEnricher, s.CoverSettings, s.TagSettings, s.LibraryScanner, s.ScanCfg, s.DataDir, smtpSender, bus, s.PlaylistScheduler, s.StreamSecret, netSettings, reacqSettings, fpSettings)
// /api/admin/scan is the only admin route owned by the server package // /api/admin/scan is the only admin route owned by the server package
// (it needs the Scanner). Register it as a single inline-middleware // (it needs the Scanner). Register it as a single inline-middleware
// route — using r.Route("/api/admin", ...) here would create a second // route — using r.Route("/api/admin", ...) here would create a second
+33 -4
View File
@@ -1,8 +1,14 @@
import { describe, it, expect, vi, beforeEach } from 'vitest'; import { describe, it, expect, vi, beforeEach } from 'vitest';
import { getFingerprintCoverage, type FingerprintCoverage } from './admin'; import {
getFingerprintCoverage,
getFingerprintSettings,
updateFingerprintSettings,
type FingerprintCoverage,
type FingerprintSettings
} from './admin';
vi.mock('./client', () => ({ vi.mock('./client', () => ({
api: { get: vi.fn(), post: vi.fn() } api: { get: vi.fn(), post: vi.fn(), put: vi.fn() }
})); }));
import { api } from './client'; import { api } from './client';
@@ -15,7 +21,8 @@ describe('admin fingerprint coverage API', () => {
total: 18026, total: 18026,
fingerprinted: 9400, fingerprinted: 9400,
rejected: 12, rejected: 12,
pending: 8614 pending: 8614,
enabled: true
}; };
(api.get as unknown as ReturnType<typeof vi.fn>).mockResolvedValueOnce(sample); (api.get as unknown as ReturnType<typeof vi.fn>).mockResolvedValueOnce(sample);
const got = await getFingerprintCoverage(); const got = await getFingerprintCoverage();
@@ -24,9 +31,31 @@ describe('admin fingerprint coverage API', () => {
}); });
it('buckets sum to the total', async () => { it('buckets sum to the total', async () => {
const sample: FingerprintCoverage = { total: 10, fingerprinted: 6, rejected: 1, pending: 3 }; const sample: FingerprintCoverage = {
total: 10,
fingerprinted: 6,
rejected: 1,
pending: 3,
enabled: true
};
(api.get as unknown as ReturnType<typeof vi.fn>).mockResolvedValueOnce(sample); (api.get as unknown as ReturnType<typeof vi.fn>).mockResolvedValueOnce(sample);
const got = await getFingerprintCoverage(); const got = await getFingerprintCoverage();
expect(got.fingerprinted + got.rejected + got.pending).toBe(got.total); expect(got.fingerprinted + got.rejected + got.pending).toBe(got.total);
}); });
it('reads and saves the fingerprinting settings at one path', async () => {
const settings: FingerprintSettings = {
enabled: true,
chromaprint_length_sec: 120,
acoustic_max_bit_error_rate: 0.15,
backfill_concurrency: 2,
sweep_interval_hours: 1
};
(api.get as unknown as ReturnType<typeof vi.fn>).mockResolvedValueOnce(settings);
(api.put as unknown as ReturnType<typeof vi.fn>).mockResolvedValueOnce(settings);
await getFingerprintSettings();
await updateFingerprintSettings(settings);
expect(api.get).toHaveBeenCalledWith('/api/admin/library/fingerprint-settings');
expect(api.put).toHaveBeenCalledWith('/api/admin/library/fingerprint-settings', settings);
});
}); });
+25
View File
@@ -322,6 +322,9 @@ export type FingerprintCoverage = {
fingerprinted: number; fingerprinted: number;
rejected: number; rejected: number;
pending: number; pending: number;
// False when the operator has switched fingerprinting off (#3913): pending
// then never shrinks, and nothing should read as progress.
enabled: boolean;
}; };
export async function getFingerprintCoverage(): Promise<FingerprintCoverage> { export async function getFingerprintCoverage(): Promise<FingerprintCoverage> {
@@ -339,6 +342,28 @@ export function createFingerprintCoverageQuery() {
}); });
} }
// Fingerprinting settings (#3913) ------------------------------------------
export type FingerprintSettings = {
enabled: boolean;
chromaprint_length_sec: number;
// The share of fingerprint bits two copies may disagree on and still be
// proposed as one recording. The card shows it as a match percentage.
acoustic_max_bit_error_rate: number;
backfill_concurrency: number;
sweep_interval_hours: number;
};
export async function getFingerprintSettings(): Promise<FingerprintSettings> {
return api.get<FingerprintSettings>('/api/admin/library/fingerprint-settings');
}
export async function updateFingerprintSettings(
s: FingerprintSettings
): Promise<FingerprintSettings> {
return api.put<FingerprintSettings>('/api/admin/library/fingerprint-settings', s);
}
// Cover-art providers ------------------------------------------------------ // Cover-art providers ------------------------------------------------------
export type CoverProviderCapability = 'album_cover' | 'artist_thumb' | 'artist_fanart'; export type CoverProviderCapability = 'album_cover' | 'artist_thumb' | 'artist_fanart';
+7
View File
@@ -75,6 +75,13 @@ describe('errMessage detail codes (#3918)', () => {
); );
}); });
test('invalid_setting appends the field and range the server names', () => {
const msg = 'fingerprint setting out of range: chromaprint_length_sec must be 30-600';
expect(errMessage({ code: 'invalid_setting', message: msg })).toBe(
`${ERROR_COPY.invalid_setting} ${msg}`
);
});
// Server messages are usually internal detail. Appending them for every code // Server messages are usually internal detail. Appending them for every code
// would leak things like driver errors into toasts; this pins the scope. // would leak things like driver errors into toasts; this pins the scope.
test('other codes never carry the server message', () => { test('other codes never carry the server message', () => {
+6 -1
View File
@@ -15,7 +15,12 @@ export function errCode(err: unknown): string {
* server messages are internal detail and must never reach a toast. Mirrored * server messages are internal detail and must never reach a toast. Mirrored
* in Android's ErrorCopy. * in Android's ErrorCopy.
*/ */
const DETAIL_CODES: ReadonlySet<string> = new Set(['library_not_writable', 'file_delete_failed']); const DETAIL_CODES: ReadonlySet<string> = new Set([
'library_not_writable',
'file_delete_failed',
// The server names the field and its range (#3913).
'invalid_setting'
]);
/** /**
* Returns user-facing copy for an unknown error value. Looks up the * Returns user-facing copy for an unknown error value. Looks up the
+7 -1
View File
@@ -471,7 +471,13 @@ export type MergeDuplicateResult = {
export type AdminDuplicatesResponse = { export type AdminDuplicatesResponse = {
sweep: AdminDuplicateSweep; sweep: AdminDuplicateSweep;
fingerprints: { total: number; fingerprinted: number; rejected: number; pending: number }; fingerprints: {
total: number;
fingerprinted: number;
rejected: number;
pending: number;
enabled: boolean;
};
total: number; total: number;
limit: number; limit: number;
offset: number; offset: number;
@@ -0,0 +1,218 @@
<script lang="ts">
import { onMount } from 'svelte';
import { TriangleAlert } from 'lucide-svelte';
import {
getFingerprintSettings,
updateFingerprintSettings,
type FingerprintSettings
} from '$lib/api/admin';
import { errMessage } from '$lib/api/errors';
import { pushToast } from '$lib/stores/toast.svelte';
// How Minstrel fingerprints tracks and when it looks for duplicates (#3913).
// Lives on the Duplicates page, beside the results these settings shape.
let { libraryTotal = 0, onSaved }: { libraryTotal?: number; onSaved?: () => void } =
$props();
let saved = $state<FingerprintSettings | null>(null);
let form = $state<FingerprintSettings | null>(null);
let saving = $state(false);
let loadError = $state(false);
const dirty = $derived(!!saved && !!form && JSON.stringify(saved) !== JSON.stringify(form));
// Stored as the share of bits two fingerprints may disagree on; the report
// speaks in match percentages, so the card does too.
const matchPercent = $derived(
form ? Math.round((1 - form.acoustic_max_bit_error_rate) * 100) : 0
);
function onMatchInput(value: string) {
if (!form) return;
form.acoustic_max_bit_error_rate = value === '' ? NaN : Math.round(100 - Number(value)) / 100;
}
// The one setting with a cost the operator can't see from here: fingerprints
// taken at two lengths can't be compared, so a new length redoes the library.
const lengthChanged = $derived(
!!saved && !!form && form.chromaprint_length_sec !== saved.chromaprint_length_sec
);
// Mirrors the server's ranges, so a bad value is named in the card's own terms
// (a percentage, not a bit-error rate) before anything is sent.
const between = (v: number, lo: number, hi: number) => Number.isInteger(v) && v >= lo && v <= hi;
const problems = $derived.by(() => {
if (!form) return [] as string[];
const out: string[] = [];
if (!between(form.chromaprint_length_sec, 30, 600))
out.push('Seconds of audio must be a whole number from 30 to 600.');
if (!between(matchPercent, 65, 99)) out.push('Minimum match must be from 65% to 99%.');
if (!between(form.backfill_concurrency, 1, 8))
out.push('Files fingerprinted at once must be from 1 to 8.');
if (!between(form.sweep_interval_hours, 1, 168))
out.push('Hours between sweeps must be from 1 to 168.');
return out;
});
async function load() {
try {
saved = await getFingerprintSettings();
form = { ...saved };
loadError = false;
} catch {
loadError = true;
}
}
onMount(load);
async function save() {
if (!form || problems.length > 0) return;
saving = true;
try {
saved = await updateFingerprintSettings(form);
form = { ...saved };
pushToast('Fingerprinting settings saved.');
onSaved?.();
} catch (e) {
pushToast(errMessage(e), 'error');
} finally {
saving = false;
}
}
const inputClass =
'mt-1 w-28 rounded border border-border bg-background px-2 py-1 text-sm text-text-primary ' +
'focus-visible:outline focus-visible:outline-2 focus-visible:outline-accent';
</script>
<section class="space-y-4 rounded-xl border border-border bg-surface p-5">
<div>
<h3 class="font-display text-lg font-medium text-text-primary">Fingerprinting</h3>
<p class="mt-1 text-sm text-text-secondary">
How tracks are fingerprinted, and how alike two must sound to be proposed as the same
recording. Identical files are always found, whatever these say.
</p>
</div>
{#if loadError}
<p class="text-sm text-action-destructive">
Couldn't load fingerprinting settings.
<button type="button" class="underline hover:no-underline" onclick={load}>Try again</button>
</p>
{:else if form === null}
<p class="text-sm text-text-secondary">Loading…</p>
{:else}
<label class="flex items-start gap-3">
<input type="checkbox" bind:checked={form.enabled} class="mt-1" />
<span>
<span class="text-sm text-text-primary">Fingerprint tracks</span>
<span class="block text-xs text-text-secondary">
Off stops all audio decoding: new files aren't fingerprinted and the library isn't
worked through in the background. Moved files are still recognised, because that only
reads the file. Tracks without a fingerprint aren't compared.
</span>
</span>
</label>
<div class="grid gap-4 sm:grid-cols-2">
<label class="block">
<span class="text-sm text-text-primary">Seconds of audio to fingerprint</span>
<span class="block text-xs text-text-secondary">
More tells apart recordings that only differ later in the track, and takes longer for
each file.
</span>
<input
type="number"
min="30"
max="600"
bind:value={form.chromaprint_length_sec}
class={inputClass}
/>
</label>
<label class="block">
<span class="text-sm text-text-primary">Minimum match (%)</span>
<span class="block text-xs text-text-secondary">
How alike two recordings must sound to be proposed as one. Higher proposes fewer, surer
groups. Unrelated songs score around 50%.
</span>
<input
type="number"
min="65"
max="99"
value={matchPercent}
oninput={(e) => onMatchInput(e.currentTarget.value)}
class={inputClass}
/>
</label>
<label class="block">
<span class="text-sm text-text-primary">Files fingerprinted at once</span>
<span class="block text-xs text-text-secondary">
The background pass competes with playback for CPU and with streaming for the disk.
Lower it if playback stutters while it runs.
</span>
<input
type="number"
min="1"
max="8"
bind:value={form.backfill_concurrency}
class={inputClass}
/>
</label>
<label class="block">
<span class="text-sm text-text-primary">Hours between sweeps</span>
<span class="block text-xs text-text-secondary">
The least time between automatic duplicate sweeps. A sweep only runs when something has
changed, and Sweep now doesn't wait.
</span>
<input
type="number"
min="1"
max="168"
bind:value={form.sweep_interval_hours}
class={inputClass}
/>
</label>
</div>
{#if lengthChanged}
<p
class="flex items-start gap-2 rounded-md bg-surface-hover px-3 py-2 text-xs text-text-secondary"
data-testid="length-warning"
>
<TriangleAlert size={14} class="mt-0.5 flex-shrink-0 text-action-destructive" aria-hidden="true" />
<span>
Saving re-fingerprints {libraryTotal > 0
? `all ${libraryTotal.toLocaleString()} tracks`
: 'every track'} at the new length. Fingerprints taken at different lengths can't be
compared, so a track is left out of the duplicate search until it's redone, and groups
of similar recordings come back as that work finishes.
</span>
</p>
{/if}
{#if problems.length > 0}
<ul class="space-y-1 text-xs text-action-destructive" data-testid="settings-problems">
{#each problems as problem (problem)}
<li>{problem}</li>
{/each}
</ul>
{/if}
<div class="flex justify-end">
<button
type="button"
class="rounded-md bg-action-secondary px-4 py-2 text-sm text-action-fg hover:opacity-90
focus-visible:outline focus-visible:outline-2 focus-visible:outline-accent
disabled:cursor-not-allowed disabled:opacity-50"
disabled={!dirty || saving || problems.length > 0}
onclick={save}
>
{saving ? 'Saving…' : 'Save'}
</button>
</div>
{/if}
</section>
@@ -0,0 +1,141 @@
import { afterEach, describe, expect, test, vi } from 'vitest';
import { render, screen, fireEvent, waitFor } from '@testing-library/svelte';
import type { FingerprintSettings } from '$lib/api/admin';
import { ERROR_COPY } from '$lib/api/error-copy';
vi.mock('$lib/api/admin', () => ({
getFingerprintSettings: vi.fn(),
updateFingerprintSettings: vi.fn()
}));
vi.mock('$lib/stores/toast.svelte', () => ({ pushToast: vi.fn() }));
import FingerprintSettingsCard from './FingerprintSettingsCard.svelte';
import { getFingerprintSettings, updateFingerprintSettings } from '$lib/api/admin';
import { pushToast } from '$lib/stores/toast.svelte';
const base: FingerprintSettings = {
enabled: true,
chromaprint_length_sec: 120,
acoustic_max_bit_error_rate: 0.15,
backfill_concurrency: 2,
sweep_interval_hours: 1
};
afterEach(() => vi.clearAllMocks());
async function renderCard(
over: Partial<FingerprintSettings> = {},
props: { libraryTotal?: number; onSaved?: () => void } = {}
) {
vi.mocked(getFingerprintSettings).mockResolvedValue({ ...base, ...over });
const r = render(FingerprintSettingsCard, { props });
await screen.findByRole('spinbutton', { name: /seconds of audio/i });
return r;
}
const saveButton = () => screen.getByRole('button', { name: /save/i });
describe('FingerprintSettingsCard', () => {
test('save is disabled until something changes', async () => {
await renderCard();
expect(saveButton()).toHaveProperty('disabled', true);
await fireEvent.input(screen.getByRole('spinbutton', { name: /hours between sweeps/i }), {
target: { value: '6' }
});
await waitFor(() => expect(saveButton()).toHaveProperty('disabled', false));
});
// The matcher works in bit-error rates; the report shows match percentages.
// A card that showed 0.15 beside a report saying "96% match" would leave the
// operator converting in their head.
test('the threshold reads as a match percentage and saves as a bit-error rate', async () => {
vi.mocked(updateFingerprintSettings).mockResolvedValue({
...base,
acoustic_max_bit_error_rate: 0.1
});
await renderCard();
const match = screen.getByRole('spinbutton', { name: /minimum match/i }) as HTMLInputElement;
expect(match.value).toBe('85');
await fireEvent.input(match, { target: { value: '90' } });
await fireEvent.click(saveButton());
await waitFor(() =>
expect(updateFingerprintSettings).toHaveBeenCalledWith(
expect.objectContaining({ acoustic_max_bit_error_rate: 0.1 })
)
);
});
// A new length re-fingerprints the whole library. Nothing else on the page
// would say so before the operator commits to it.
test('changing the length warns that every track is re-fingerprinted', async () => {
await renderCard({}, { libraryTotal: 18026 });
expect(screen.queryByTestId('length-warning')).toBeNull();
await fireEvent.input(screen.getByRole('spinbutton', { name: /seconds of audio/i }), {
target: { value: '60' }
});
const warning = await screen.findByTestId('length-warning');
expect(warning.textContent).toMatch(/all\s+18,026\s+tracks/);
});
test('other changes carry no re-fingerprinting warning', async () => {
await renderCard({}, { libraryTotal: 18026 });
await fireEvent.input(screen.getByRole('spinbutton', { name: /files fingerprinted at once/i }), {
target: { value: '4' }
});
await waitFor(() => expect(saveButton()).toHaveProperty('disabled', false));
expect(screen.queryByTestId('length-warning')).toBeNull();
});
test('a value out of range blocks saving and says which', async () => {
await renderCard();
await fireEvent.input(screen.getByRole('spinbutton', { name: /files fingerprinted at once/i }), {
target: { value: '12' }
});
const problems = await screen.findByTestId('settings-problems');
expect(problems.textContent).toMatch(/files fingerprinted at once must be from 1 to 8/i);
expect(saveButton()).toHaveProperty('disabled', true);
});
test('a rejected save surfaces the field the server names', async () => {
const message = 'fingerprint setting out of range: sweep_interval_hours must be 1-168';
vi.mocked(updateFingerprintSettings).mockRejectedValue({
code: 'invalid_setting',
message,
status: 400
});
await renderCard();
await fireEvent.input(screen.getByRole('spinbutton', { name: /hours between sweeps/i }), {
target: { value: '6' }
});
await fireEvent.click(saveButton());
await waitFor(() =>
expect(pushToast).toHaveBeenCalledWith(`${ERROR_COPY.invalid_setting} ${message}`, 'error')
);
});
// Switching fingerprinting off changes what the page's counts mean.
test('a save tells the page, so its counts refresh', async () => {
const onSaved = vi.fn();
vi.mocked(updateFingerprintSettings).mockResolvedValue({ ...base, enabled: false });
await renderCard({}, { onSaved });
await fireEvent.click(screen.getByRole('checkbox', { name: /fingerprint tracks/i }));
await fireEvent.click(saveButton());
await waitFor(() => expect(onSaved).toHaveBeenCalledTimes(1));
expect(updateFingerprintSettings).toHaveBeenCalledWith(
expect.objectContaining({ enabled: false })
);
});
test('a failed load offers a retry rather than an empty card', async () => {
vi.mocked(getFingerprintSettings).mockRejectedValue(new Error('nope'));
render(FingerprintSettingsCard);
await waitFor(() =>
expect(screen.getByText(/couldn't load fingerprinting settings/i)).toBeTruthy()
);
expect(screen.getByRole('button', { name: /try again/i })).toBeTruthy();
});
});
+1
View File
@@ -45,6 +45,7 @@
"sweep_in_progress": "A duplicate sweep is already running.", "sweep_in_progress": "A duplicate sweep is already running.",
"duplicate_group_not_pending": "That group has already been resolved.", "duplicate_group_not_pending": "That group has already been resolved.",
"survivor_not_in_group": "That copy isn't part of this group any more.", "survivor_not_in_group": "That copy isn't part of this group any more.",
"invalid_setting": "That setting is out of range.",
"album_not_found": "That album no longer exists.", "album_not_found": "That album no longer exists.",
"artist_not_found": "That artist no longer exists.", "artist_not_found": "That artist no longer exists.",
"playlist_not_found": "That playlist no longer exists.", "playlist_not_found": "That playlist no longer exists.",
+5
View File
@@ -445,6 +445,11 @@
<span class="text-text-muted">·</span> <span class="text-text-muted">·</span>
<span>{fingerprints.pending.toLocaleString()} pending</span> <span>{fingerprints.pending.toLocaleString()} pending</span>
{/if} {/if}
{#if fingerprints.enabled === false}
<!-- Off (#3913), pending never shrinks; say why rather than imply progress. -->
<span class="text-text-muted">·</span>
<a href="/admin/duplicates" class="underline hover:no-underline">fingerprinting is off</a>
{/if}
{#if fingerprints.rejected > 0} {#if fingerprints.rejected > 0}
<span class="text-text-muted">·</span> <span class="text-text-muted">·</span>
<span <span
+19 -4
View File
@@ -11,6 +11,7 @@
import { pushToast } from '$lib/stores/toast.svelte'; import { pushToast } from '$lib/stores/toast.svelte';
import { relativeTime } from '$lib/utils/relativeTime'; import { relativeTime } from '$lib/utils/relativeTime';
import type { AdminDuplicateGroup, AdminDuplicateMember } from '$lib/api/types'; import type { AdminDuplicateGroup, AdminDuplicateMember } from '$lib/api/types';
import FingerprintSettingsCard from '$lib/components/FingerprintSettingsCard.svelte';
// Tracks the duplicate sweep believes hold one recording (#3912). Dismissing a // Tracks the duplicate sweep believes hold one recording (#3912). Dismissing a
// group says "these are not duplicates", and the sweep will not propose that // group says "these are not duplicates", and the sweep will not propose that
@@ -173,8 +174,13 @@
{/if} {/if}
{/if} {/if}
{#if prints && prints.pending > 0} {#if prints && prints.pending > 0}
{prints.pending.toLocaleString()} tracks are still waiting for a fingerprint and join {#if prints.enabled === false}
the comparison once they have one. Fingerprinting is off, so {prints.pending.toLocaleString()} tracks without a current
fingerprint aren't compared.
{:else}
{prints.pending.toLocaleString()} tracks are still waiting for a fingerprint and join
the comparison once they have one.
{/if}
{/if} {/if}
</p> </p>
{/if} {/if}
@@ -191,8 +197,13 @@
{#if prints && prints.total > 0 && prints.fingerprinted === 0} {#if prints && prints.total > 0 && prints.fingerprinted === 0}
<p class="mt-3 text-text-primary">Nothing to compare yet.</p> <p class="mt-3 text-text-primary">Nothing to compare yet.</p>
<p class="mt-1 text-sm text-text-secondary"> <p class="mt-1 text-sm text-text-secondary">
The library is still being fingerprinted — {prints.pending.toLocaleString()} tracks to go. {#if prints.enabled === false}
Duplicates appear here as the sweep finds them. Fingerprinting is off, so no track has a fingerprint to compare. Turn it on in the
settings below.
{:else}
The library is still being fingerprinted — {prints.pending.toLocaleString()} tracks to go.
Duplicates appear here as the sweep finds them.
{/if}
</p> </p>
{:else if sweep?.state === 'never'} {:else if sweep?.state === 'never'}
<p class="mt-3 text-text-primary">The sweep hasn't run yet.</p> <p class="mt-3 text-text-primary">The sweep hasn't run yet.</p>
@@ -341,4 +352,8 @@
</nav> </nav>
{/if} {/if}
{/if} {/if}
<!-- Settings sit under the report whatever state it is in: turning
fingerprinting back on is how an empty report gets out of that state. -->
<FingerprintSettingsCard libraryTotal={prints?.total ?? 0} onSaved={() => query.refetch()} />
</div> </div>
@@ -10,7 +10,17 @@ vi.mock('$lib/api/admin', () => ({
mergeDuplicateGroup: vi.fn().mockResolvedValue({ mergeDuplicateGroup: vi.fn().mockResolvedValue({
survivor_track_id: 'www-01', survivor_track_id: 'www-01',
removed_paths: ['/music/Moe Shop/WWW (2020)/www-02.mp3'] removed_paths: ['/music/Moe Shop/WWW (2020)/www-02.mp3']
}) }),
// The page embeds FingerprintSettingsCard, which loads its own settings from
// this module; the card has its own suite.
getFingerprintSettings: vi.fn().mockResolvedValue({
enabled: true,
chromaprint_length_sec: 120,
acoustic_max_bit_error_rate: 0.15,
backfill_concurrency: 2,
sweep_interval_hours: 1
}),
updateFingerprintSettings: vi.fn()
})); }));
import AdminDuplicatesPage from './+page.svelte'; import AdminDuplicatesPage from './+page.svelte';
@@ -33,7 +43,7 @@ const finishedSweep = {
oversize_clusters: 0, oversize_clusters: 0,
error_message: null error_message: null
}; };
const allFingerprinted = { total: 1200, fingerprinted: 1200, rejected: 0, pending: 0 }; const allFingerprinted = { total: 1200, fingerprinted: 1200, rejected: 0, pending: 0, enabled: true };
function member(id: string, extra: Partial<AdminDuplicatesResponse['groups'][number]['members'][number]> = {}) { function member(id: string, extra: Partial<AdminDuplicatesResponse['groups'][number]['members'][number]> = {}) {
return { return {
@@ -120,12 +130,33 @@ describe('admin duplicates', () => {
response({ response({
groups: [], groups: [],
total: 0, total: 0,
fingerprints: { total: 1200, fingerprinted: 0, rejected: 0, pending: 1200 } fingerprints: { total: 1200, fingerprinted: 0, rejected: 0, pending: 1200, enabled: true }
}) })
); );
expect(text(screen.getByTestId('empty-state'))).toContain('still being fingerprinted'); expect(text(screen.getByTestId('empty-state'))).toContain('still being fingerprinted');
}); });
// Switched off, the backlog never shrinks. "Still being fingerprinted" would
// promise work nothing is doing (#3913).
test('with fingerprinting off the page says so instead of promising progress', () => {
renderWith(
response({
groups: [],
total: 0,
fingerprints: { total: 1200, fingerprinted: 0, rejected: 0, pending: 1200, enabled: false }
})
);
const empty = text(screen.getByTestId('empty-state'));
expect(empty).toContain('Fingerprinting is off');
expect(empty).not.toContain('still being fingerprinted');
expect(text(screen.getByTestId('sweep-status'))).toContain('Fingerprinting is off');
});
test('the fingerprinting settings are on this page', async () => {
renderWith(response());
expect(await screen.findByRole('spinbutton', { name: /seconds of audio/i })).toBeTruthy();
});
test('empty before any sweep says the sweep has not run', () => { test('empty before any sweep says the sweep has not run', () => {
renderWith( renderWith(
response({ response({