feat(admin): fingerprinting settings — on/off, length, match threshold, concurrency, sweep interval (M400 #3913)
test-go / test (push) Failing after 44s
test-web / test (push) Successful in 49s
test-go / integration (push) Failing after 2m42s
release / Build + push container image (push) Canceled after 0s
release / Verify release artifacts (tag releases only) (push) Canceled after 0s
release / Build signed APK (releases and dev) (push) Canceled after 4m8s
test-go / test (push) Failing after 44s
test-web / test (push) Successful in 49s
test-go / integration (push) Failing after 2m42s
release / Build + push container image (push) Canceled after 0s
release / Verify release artifacts (tag releases only) (push) Canceled after 0s
release / Build signed APK (releases and dev) (push) Canceled after 4m8s
Rule 25: the fingerprinting knobs move out of source into a DB-backed singleton (migration 0061), edited from a card on the Duplicates page and shared live with the scanner, the backfill and the duplicate sweep through one service instance, so a save needs no restart. The length is the knob that can silently break the library: prints taken at two lengths never match. Each track_fingerprints row now records the length it was taken at, and every reader filters on the current one — the backfill treats another length as stale, the gauge counts it pending, the sweep never streams it. Equivalent to a version bump, except that setting the length back makes rows not yet redone current again. The card warns before a length change re-fingerprints the library. Off stops every decode: the scan takes only the stream hash (a demux, and what recognises a moved file) and stores nothing, dropping a changed file's stale row; the backfill idles. A save also makes a sweep due, since a new threshold or length changes what the same prints group into, and the sweep interval gains slack so an hourly interval on an hourly tick doesn't skip every other tick. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01SQ31KQpYbStyK5y58UmPLH
This commit is contained in:
@@ -1,6 +1,7 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
|
||||
"git.fabledsword.com/bvandeusen/minstrel/internal/apierror"
|
||||
@@ -40,12 +41,32 @@ func (h *handlers) handleGetLibraryCoverage(w http.ResponseWriter, r *http.Reque
|
||||
|
||||
// fingerprintCoverageResp is the wire shape for GET /api/admin/library/fingerprints.
|
||||
// fingerprinted + rejected + pending = total. Missing tracks are not counted:
|
||||
// there is no file to fingerprint.
|
||||
// there is no file to fingerprint. Enabled travels with the counts because with
|
||||
// fingerprinting off (#3913) pending never shrinks, and a gauge that implies
|
||||
// progress would be promising work nothing is doing.
|
||||
type fingerprintCoverageResp struct {
|
||||
Total int64 `json:"total"`
|
||||
Fingerprinted int64 `json:"fingerprinted"`
|
||||
Rejected int64 `json:"rejected"`
|
||||
Pending int64 `json:"pending"`
|
||||
Enabled bool `json:"enabled"`
|
||||
}
|
||||
|
||||
// fingerprintCoverage reads the gauge against the current settings: a print at
|
||||
// another length counts as pending, because the backfill will re-derive it.
|
||||
func (h *handlers) fingerprintCoverage(ctx context.Context) (fingerprintCoverageResp, error) {
|
||||
cfg := h.fingerprintSettings.Get()
|
||||
row, err := library.FingerprintCoverage(ctx, h.pool, cfg)
|
||||
if err != nil {
|
||||
return fingerprintCoverageResp{}, err
|
||||
}
|
||||
return fingerprintCoverageResp{
|
||||
Total: row.Total,
|
||||
Fingerprinted: row.Fingerprinted,
|
||||
Rejected: row.Rejected,
|
||||
Pending: row.Pending,
|
||||
Enabled: cfg.Enabled,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// handleGetFingerprintCoverage implements GET /api/admin/library/fingerprints:
|
||||
@@ -53,15 +74,10 @@ type fingerprintCoverageResp struct {
|
||||
// worker spanning many passes, with no scan run to attach a tally to, so its
|
||||
// progress is read live here. Always 200; zeros on an empty library.
|
||||
func (h *handlers) handleGetFingerprintCoverage(w http.ResponseWriter, r *http.Request) {
|
||||
row, err := library.FingerprintCoverage(r.Context(), h.pool)
|
||||
cov, err := h.fingerprintCoverage(r.Context())
|
||||
if err != nil {
|
||||
writeErrWithLog(w, h.logger, "admin: get fingerprint coverage", apierror.InternalMsg("lookup failed", err))
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, fingerprintCoverageResp{
|
||||
Total: row.Total,
|
||||
Fingerprinted: row.Fingerprinted,
|
||||
Rejected: row.Rejected,
|
||||
Pending: row.Pending,
|
||||
})
|
||||
writeJSON(w, http.StatusOK, cov)
|
||||
}
|
||||
|
||||
@@ -95,7 +95,7 @@ func (h *handlers) handleListDuplicates(w http.ResponseWriter, r *http.Request)
|
||||
return
|
||||
}
|
||||
|
||||
cov, err := library.FingerprintCoverage(ctx, h.pool)
|
||||
cov, err := h.fingerprintCoverage(ctx)
|
||||
if err != nil {
|
||||
h.logger.Error("admin: fingerprint coverage", "err", err)
|
||||
writeAdminJSONErr(w, http.StatusInternalServerError, "server_error")
|
||||
@@ -117,14 +117,12 @@ func (h *handlers) handleListDuplicates(w http.ResponseWriter, r *http.Request)
|
||||
}
|
||||
|
||||
writeJSON(w, http.StatusOK, adminDuplicatesResponse{
|
||||
Sweep: sweep,
|
||||
Fingerprints: fingerprintCoverageResp{
|
||||
Total: cov.Total, Fingerprinted: cov.Fingerprinted, Rejected: cov.Rejected, Pending: cov.Pending,
|
||||
},
|
||||
Total: total,
|
||||
Limit: limit,
|
||||
Offset: offset,
|
||||
Groups: foldDuplicateGroups(rows),
|
||||
Sweep: sweep,
|
||||
Fingerprints: cov,
|
||||
Total: total,
|
||||
Limit: limit,
|
||||
Offset: offset,
|
||||
Groups: foldDuplicateGroups(rows),
|
||||
})
|
||||
}
|
||||
|
||||
@@ -194,8 +192,10 @@ func foldDuplicateGroups(rows []dbq.ListPendingDuplicateGroupMembersRow) []dupli
|
||||
// The sweep outlives the request, so it runs on a background context, as
|
||||
// handleTriggerScan's scan does.
|
||||
func (h *handlers) handleRunDuplicateSweep(w http.ResponseWriter, _ *http.Request) {
|
||||
// Runs whatever the sweep interval says: the interval paces the automatic
|
||||
// sweep, and an operator pressing the button has already decided.
|
||||
started, err := library.TryStartDuplicateSweep(
|
||||
context.Background(), h.pool, h.logger.With("source", "manual"),
|
||||
context.Background(), h.pool, h.logger.With("source", "manual"), h.fingerprintSettings.Get(),
|
||||
)
|
||||
if err != nil {
|
||||
h.logger.Error("admin: start duplicate sweep", "err", err)
|
||||
|
||||
@@ -0,0 +1,67 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"net/http"
|
||||
|
||||
"git.fabledsword.com/bvandeusen/minstrel/internal/apierror"
|
||||
"git.fabledsword.com/bvandeusen/minstrel/internal/library"
|
||||
)
|
||||
|
||||
// fingerprintSettingsBody is the wire shape for GET and PUT
|
||||
// /api/admin/library/fingerprint-settings (M400 #3913). The threshold travels as
|
||||
// the bit-error rate the matcher uses; the card presents it as a match percentage.
|
||||
type fingerprintSettingsBody struct {
|
||||
Enabled bool `json:"enabled"`
|
||||
ChromaprintLengthSec int32 `json:"chromaprint_length_sec"`
|
||||
AcousticMaxBitErrorRate float64 `json:"acoustic_max_bit_error_rate"`
|
||||
BackfillConcurrency int32 `json:"backfill_concurrency"`
|
||||
SweepIntervalHours int32 `json:"sweep_interval_hours"`
|
||||
}
|
||||
|
||||
func fingerprintSettingsBodyOf(s library.FingerprintSettings) fingerprintSettingsBody {
|
||||
return fingerprintSettingsBody{
|
||||
Enabled: s.Enabled,
|
||||
ChromaprintLengthSec: s.ChromaprintLengthSec,
|
||||
AcousticMaxBitErrorRate: s.AcousticMaxBitErrorRate,
|
||||
BackfillConcurrency: s.BackfillConcurrency,
|
||||
SweepIntervalHours: s.SweepIntervalHours,
|
||||
}
|
||||
}
|
||||
|
||||
// handleGetFingerprintSettings implements GET /api/admin/library/fingerprint-settings.
|
||||
func (h *handlers) handleGetFingerprintSettings(w http.ResponseWriter, _ *http.Request) {
|
||||
writeJSON(w, http.StatusOK, fingerprintSettingsBodyOf(h.fingerprintSettings.Get()))
|
||||
}
|
||||
|
||||
// handleUpdateFingerprintSettings implements PUT /api/admin/library/fingerprint-settings.
|
||||
//
|
||||
// A whole-row write. A body that leaves a field out decodes it as zero, which no
|
||||
// field accepts, so a partial save is refused rather than zeroing what it omitted.
|
||||
// The saved settings reach the scanner and both workers at once: they share the
|
||||
// service instance.
|
||||
func (h *handlers) handleUpdateFingerprintSettings(w http.ResponseWriter, r *http.Request) {
|
||||
var req fingerprintSettingsBody
|
||||
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
|
||||
writeErr(w, apierror.BadRequest("invalid_body", "malformed JSON"))
|
||||
return
|
||||
}
|
||||
saved, err := h.fingerprintSettings.Set(r.Context(), library.FingerprintSettings{
|
||||
Enabled: req.Enabled,
|
||||
ChromaprintLengthSec: req.ChromaprintLengthSec,
|
||||
AcousticMaxBitErrorRate: req.AcousticMaxBitErrorRate,
|
||||
BackfillConcurrency: req.BackfillConcurrency,
|
||||
SweepIntervalHours: req.SweepIntervalHours,
|
||||
})
|
||||
if err != nil {
|
||||
// Validation mirrors migration 0061's CHECKs and names the field.
|
||||
if errors.Is(err, library.ErrFingerprintSettingOutOfRange) {
|
||||
writeErr(w, apierror.BadRequest("invalid_setting", err.Error()))
|
||||
return
|
||||
}
|
||||
writeErrWithLog(w, h.logger, "admin fingerprint settings: update failed", apierror.Internal(err))
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, fingerprintSettingsBodyOf(saved))
|
||||
}
|
||||
@@ -0,0 +1,55 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"io"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.fabledsword.com/bvandeusen/minstrel/internal/library"
|
||||
)
|
||||
|
||||
func TestGetFingerprintSettings_ServesDefaultsWithoutAService(t *testing.T) {
|
||||
h := &handlers{logger: slog.New(slog.NewTextHandler(io.Discard, nil))}
|
||||
rec := httptest.NewRecorder()
|
||||
h.handleGetFingerprintSettings(rec, httptest.NewRequest(http.MethodGet, "/api/admin/library/fingerprint-settings", nil))
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d, want 200", rec.Code)
|
||||
}
|
||||
var got fingerprintSettingsBody
|
||||
if err := json.NewDecoder(rec.Body).Decode(&got); err != nil {
|
||||
t.Fatalf("decode: %v", err)
|
||||
}
|
||||
if want := fingerprintSettingsBodyOf(library.DefaultFingerprintSettings); got != want {
|
||||
t.Fatalf("body = %+v, want the defaults %+v", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestUpdateFingerprintSettings_Rejects(t *testing.T) {
|
||||
h := &handlers{logger: slog.New(slog.NewTextHandler(io.Discard, nil))}
|
||||
for name, tc := range map[string]struct {
|
||||
body string
|
||||
code string
|
||||
mentions string
|
||||
}{
|
||||
"a value out of range, naming the field": {
|
||||
body: `{"enabled":true,"chromaprint_length_sec":5,"acoustic_max_bit_error_rate":0.15,"backfill_concurrency":2,"sweep_interval_hours":1}`,
|
||||
code: "invalid_setting",
|
||||
mentions: "chromaprint_length_sec",
|
||||
},
|
||||
// A partial body would otherwise zero every field it left out.
|
||||
"a body missing fields": {body: `{"enabled":false}`, code: "invalid_setting"},
|
||||
"malformed JSON": {body: `{"enabled":`, code: "invalid_body"},
|
||||
} {
|
||||
rec := httptest.NewRecorder()
|
||||
h.handleUpdateFingerprintSettings(rec, httptest.NewRequest(
|
||||
http.MethodPut, "/api/admin/library/fingerprint-settings", strings.NewReader(tc.body)))
|
||||
body := rec.Body.String()
|
||||
if rec.Code != http.StatusBadRequest || !strings.Contains(body, `"`+tc.code+`"`) || !strings.Contains(body, tc.mentions) {
|
||||
t.Errorf("%s: status %d body %s; want 400 %s mentioning %q", name, rec.Code, body, tc.code, tc.mentions)
|
||||
}
|
||||
}
|
||||
}
|
||||
+28
-21
@@ -33,30 +33,31 @@ import (
|
||||
// Mount attaches /api/* handlers to r. Public endpoints (login) are outside
|
||||
// RequireUser; everything else is gated by the middleware. The events writer
|
||||
// is shared with the Subsonic mount so /rest/scrobble feeds the same store.
|
||||
func Mount(r chi.Router, pool *pgxpool.Pool, logger *slog.Logger, events *playevents.Writer, recCfg config.RecommendationConfig, recSettings *recsettings.Service, lidarrCfg *lidarrconfig.Service, lidarrReqs *lidarrrequests.Service, lidarrQuar *lidarrquarantine.Service, tracksSvc *tracks.Service, playlistsSvc *playlists.Service, coverEnricher *coverart.Enricher, coverSettings *coverart.SettingsService, tagSettings *tags.SettingsService, scanner *library.Scanner, scanCfg library.RunScanConfig, dataDir string, sender mailer.Sender, bus *eventbus.Bus, playlistScheduler *playlists.Scheduler, streamSecret []byte, netSettings *netsettings.Service, reacqSettings *reacquisition.SettingsService) {
|
||||
func Mount(r chi.Router, pool *pgxpool.Pool, logger *slog.Logger, events *playevents.Writer, recCfg config.RecommendationConfig, recSettings *recsettings.Service, lidarrCfg *lidarrconfig.Service, lidarrReqs *lidarrrequests.Service, lidarrQuar *lidarrquarantine.Service, tracksSvc *tracks.Service, playlistsSvc *playlists.Service, coverEnricher *coverart.Enricher, coverSettings *coverart.SettingsService, tagSettings *tags.SettingsService, scanner *library.Scanner, scanCfg library.RunScanConfig, dataDir string, sender mailer.Sender, bus *eventbus.Bus, playlistScheduler *playlists.Scheduler, streamSecret []byte, netSettings *netsettings.Service, reacqSettings *reacquisition.SettingsService, fpSettings *library.FingerprintSettingsService) {
|
||||
rng := rand.New(rand.NewSource(rand.Int63()))
|
||||
h := &handlers{
|
||||
pool: pool, logger: logger, events: events, recCfg: recCfg,
|
||||
recSettings: recSettings,
|
||||
rng: rng.Float64,
|
||||
lidarrCfg: lidarrCfg,
|
||||
lidarrRequests: lidarrReqs,
|
||||
lidarrQuarantine: lidarrQuar,
|
||||
tracks: tracksSvc,
|
||||
playlists: playlistsSvc,
|
||||
coverart: coverEnricher,
|
||||
coverSettings: coverSettings,
|
||||
tagSettings: tagSettings,
|
||||
scanner: scanner,
|
||||
scanCfg: scanCfg,
|
||||
dataDir: dataDir,
|
||||
mailer: sender,
|
||||
eventbus: bus,
|
||||
playlistScheduler: playlistScheduler,
|
||||
streamSecret: streamSecret,
|
||||
netSettings: netSettings,
|
||||
reacqSettings: reacqSettings,
|
||||
librarySize: recommendation.NewLibrarySize(nil),
|
||||
recSettings: recSettings,
|
||||
rng: rng.Float64,
|
||||
lidarrCfg: lidarrCfg,
|
||||
lidarrRequests: lidarrReqs,
|
||||
lidarrQuarantine: lidarrQuar,
|
||||
tracks: tracksSvc,
|
||||
playlists: playlistsSvc,
|
||||
coverart: coverEnricher,
|
||||
coverSettings: coverSettings,
|
||||
tagSettings: tagSettings,
|
||||
scanner: scanner,
|
||||
scanCfg: scanCfg,
|
||||
dataDir: dataDir,
|
||||
mailer: sender,
|
||||
eventbus: bus,
|
||||
playlistScheduler: playlistScheduler,
|
||||
streamSecret: streamSecret,
|
||||
netSettings: netSettings,
|
||||
reacqSettings: reacqSettings,
|
||||
fingerprintSettings: fpSettings,
|
||||
librarySize: recommendation.NewLibrarySize(nil),
|
||||
}
|
||||
|
||||
r.Route("/api", func(api chi.Router) {
|
||||
@@ -216,6 +217,8 @@ func Mount(r chi.Router, pool *pgxpool.Pool, logger *slog.Logger, events *playev
|
||||
|
||||
admin.Get("/library/coverage", h.handleGetLibraryCoverage)
|
||||
admin.Get("/library/fingerprints", h.handleGetFingerprintCoverage)
|
||||
admin.Get("/library/fingerprint-settings", h.handleGetFingerprintSettings)
|
||||
admin.Put("/library/fingerprint-settings", h.handleUpdateFingerprintSettings)
|
||||
// Duplicates report (#3912): proposals from the duplicate sweep, a
|
||||
// trigger to sweep now, dismissal, and the merge (#3911), which deletes
|
||||
// the removed copies' files after moving their history onto the kept one.
|
||||
@@ -306,6 +309,10 @@ type handlers struct {
|
||||
// missing files (milestone #290) — grace window, backoff, attempt caps.
|
||||
// Cached in the service, so the admin card reads it without a query.
|
||||
reacqSettings *reacquisition.SettingsService
|
||||
// fingerprintSettings is the fingerprinting policy (M400 #3913), the same
|
||||
// instance the scanner and the fingerprint workers read, so a save from the
|
||||
// admin card reaches them without a restart. Nil serves the defaults.
|
||||
fingerprintSettings *library.FingerprintSettingsService
|
||||
// netSettings caches the trusted reverse-proxy depth read by the auth
|
||||
// middleware on every request and edited from the admin network card.
|
||||
netSettings *netsettings.Service
|
||||
|
||||
Reference in New Issue
Block a user