diff --git a/cmd/minstrel/main.go b/cmd/minstrel/main.go index 669f6f88..aa57a6a7 100644 --- a/cmd/minstrel/main.go +++ b/cmd/minstrel/main.go @@ -214,6 +214,12 @@ func run() error { // SQL, no external calls; empty on single-user servers. go coplay.NewWorker(pool, logger.With("component", "coplay")).Run(ctx) + // Fingerprint backfill (M400 #3908): fingerprints the tracks the scan never + // will — everything imported before fingerprinting existed, and rows derived + // by an older method. A worker of its own rather than a scan stage; see + // internal/library/fingerprint_backfill.go for why. + go library.NewFingerprintBackfillWorker(pool, logger.With("component", "fingerprint_backfill")).Run(ctx) + // Start the tag-enrichment worker (#1490). Reconciles the compiled-in // tag providers with tag_provider_settings, bumps the sources version if // the provider set changed (re-opening settled rows), then drains tracks diff --git a/internal/api/admin_coverage.go b/internal/api/admin_coverage.go index 422dcc4d..3c5e0d58 100644 --- a/internal/api/admin_coverage.go +++ b/internal/api/admin_coverage.go @@ -5,6 +5,7 @@ import ( "git.fabledsword.com/bvandeusen/minstrel/internal/apierror" "git.fabledsword.com/bvandeusen/minstrel/internal/db/dbq" + "git.fabledsword.com/bvandeusen/minstrel/internal/library" ) // coverageRollupResp is the wire shape for GET /api/admin/library/coverage. @@ -36,3 +37,31 @@ func (h *handlers) handleGetLibraryCoverage(w http.ResponseWriter, r *http.Reque PendingNoMbid: row.PendingNoMbid, }) } + +// fingerprintCoverageResp is the wire shape for GET /api/admin/library/fingerprints. +// fingerprinted + rejected + pending = total. Missing tracks are not counted: +// there is no file to fingerprint. +type fingerprintCoverageResp struct { + Total int64 `json:"total"` + Fingerprinted int64 `json:"fingerprinted"` + Rejected int64 `json:"rejected"` + Pending int64 `json:"pending"` +} + +// handleGetFingerprintCoverage implements GET /api/admin/library/fingerprints: +// how far the fingerprint backfill (#3908) has got. The backfill is its own +// worker spanning many passes, with no scan run to attach a tally to, so its +// progress is read live here. Always 200; zeros on an empty library. +func (h *handlers) handleGetFingerprintCoverage(w http.ResponseWriter, r *http.Request) { + row, err := library.FingerprintCoverage(r.Context(), h.pool) + if err != nil { + writeErrWithLog(w, h.logger, "admin: get fingerprint coverage", apierror.InternalMsg("lookup failed", err)) + return + } + writeJSON(w, http.StatusOK, fingerprintCoverageResp{ + Total: row.Total, + Fingerprinted: row.Fingerprinted, + Rejected: row.Rejected, + Pending: row.Pending, + }) +} diff --git a/internal/api/api.go b/internal/api/api.go index 2f71c3e2..b2e76c2d 100644 --- a/internal/api/api.go +++ b/internal/api/api.go @@ -215,6 +215,7 @@ func Mount(r chi.Router, pool *pgxpool.Pool, logger *slog.Logger, events *playev admin.Get("/library/missing", h.handleListMissingTracks) admin.Get("/library/coverage", h.handleGetLibraryCoverage) + admin.Get("/library/fingerprints", h.handleGetFingerprintCoverage) admin.Get("/invites", h.handleListInvites) admin.Post("/invites", h.handleCreateInvite) diff --git a/internal/db/dbq/fingerprints.sql.go b/internal/db/dbq/fingerprints.sql.go index f1b5e244..ce1bdf03 100644 --- a/internal/db/dbq/fingerprints.sql.go +++ b/internal/db/dbq/fingerprints.sql.go @@ -23,6 +23,95 @@ func (q *Queries) DeleteTrackFingerprint(ctx context.Context, trackID pgtype.UUI return err } +const getFingerprintCoverage = `-- name: GetFingerprintCoverage :one +SELECT count(*)::bigint AS total, + count(*) FILTER ( + WHERE f.fingerprint_version >= $1 + AND f.audio_stream_sha256 IS NOT NULL AND f.chromaprint IS NOT NULL + )::bigint AS fingerprinted, + count(*) FILTER ( + WHERE f.fingerprint_version >= $1 + AND (f.audio_stream_sha256 IS NULL OR f.chromaprint IS NULL) + )::bigint AS rejected, + count(*) FILTER ( + WHERE f.track_id IS NULL OR f.fingerprint_version < $1 + )::bigint AS pending + FROM tracks t + LEFT JOIN track_fingerprints f ON f.track_id = t.id + WHERE t.missing_since IS NULL +` + +type GetFingerprintCoverageRow struct { + Total int64 + Fingerprinted int64 + Rejected int64 + Pending int64 +} + +// The admin gauge for the backfill. fingerprinted + rejected + pending = total. +// rejected is a row at the current version with a NULL half: a tool ran and +// refused the file, which is settled rather than waiting. Missing tracks are +// excluded, or the gauge could never reach the end. +func (q *Queries) GetFingerprintCoverage(ctx context.Context, currentVersion int16) (GetFingerprintCoverageRow, error) { + row := q.db.QueryRow(ctx, getFingerprintCoverage, currentVersion) + var i GetFingerprintCoverageRow + err := row.Scan( + &i.Total, + &i.Fingerprinted, + &i.Rejected, + &i.Pending, + ) + return i, err +} + +const listTracksNeedingFingerprint = `-- name: ListTracksNeedingFingerprint :many +SELECT t.id, t.file_path + FROM tracks t + LEFT JOIN track_fingerprints f ON f.track_id = t.id + WHERE t.missing_since IS NULL + AND (f.track_id IS NULL OR f.fingerprint_version < $1) + AND t.id > $2 + ORDER BY t.id + LIMIT $3 +` + +type ListTracksNeedingFingerprintParams struct { + CurrentVersion int16 + AfterID pgtype.UUID + BatchLimit int32 +} + +type ListTracksNeedingFingerprintRow struct { + ID pgtype.UUID + FilePath string +} + +// The backfill's work queue (#3908): tracks with no fingerprint, or one derived +// by an older method. Keyset-paged on id so a pass visits each track at most +// once. That cursor is load-bearing: an inconclusive attempt writes no row, so +// without it a file that keeps timing out would be listed again straight away +// and retried in a tight loop. Missing tracks are skipped — there is no file to +// read. +func (q *Queries) ListTracksNeedingFingerprint(ctx context.Context, arg ListTracksNeedingFingerprintParams) ([]ListTracksNeedingFingerprintRow, error) { + rows, err := q.db.Query(ctx, listTracksNeedingFingerprint, arg.CurrentVersion, arg.AfterID, arg.BatchLimit) + if err != nil { + return nil, err + } + defer rows.Close() + var items []ListTracksNeedingFingerprintRow + for rows.Next() { + var i ListTracksNeedingFingerprintRow + if err := rows.Scan(&i.ID, &i.FilePath); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + const upsertTrackFingerprint = `-- name: UpsertTrackFingerprint :exec INSERT INTO track_fingerprints ( track_id, audio_stream_sha256, chromaprint, fingerprint_version diff --git a/internal/db/queries/fingerprints.sql b/internal/db/queries/fingerprints.sql index ef0276b5..90a8a978 100644 --- a/internal/db/queries/fingerprints.sql +++ b/internal/db/queries/fingerprints.sql @@ -20,3 +20,40 @@ ON CONFLICT (track_id) DO UPDATE SET -- file. The stored row describes the OLD bytes, so it goes and the backfill -- re-derives it — nothing may keep trusting a stale identity. DELETE FROM track_fingerprints WHERE track_id = $1; + +-- name: ListTracksNeedingFingerprint :many +-- The backfill's work queue (#3908): tracks with no fingerprint, or one derived +-- by an older method. Keyset-paged on id so a pass visits each track at most +-- once. That cursor is load-bearing: an inconclusive attempt writes no row, so +-- without it a file that keeps timing out would be listed again straight away +-- and retried in a tight loop. Missing tracks are skipped — there is no file to +-- read. +SELECT t.id, t.file_path + FROM tracks t + LEFT JOIN track_fingerprints f ON f.track_id = t.id + WHERE t.missing_since IS NULL + AND (f.track_id IS NULL OR f.fingerprint_version < sqlc.arg(current_version)) + AND t.id > sqlc.arg(after_id) + ORDER BY t.id + LIMIT sqlc.arg(batch_limit); + +-- name: GetFingerprintCoverage :one +-- The admin gauge for the backfill. fingerprinted + rejected + pending = total. +-- rejected is a row at the current version with a NULL half: a tool ran and +-- refused the file, which is settled rather than waiting. Missing tracks are +-- excluded, or the gauge could never reach the end. +SELECT count(*)::bigint AS total, + count(*) FILTER ( + WHERE f.fingerprint_version >= sqlc.arg(current_version) + AND f.audio_stream_sha256 IS NOT NULL AND f.chromaprint IS NOT NULL + )::bigint AS fingerprinted, + count(*) FILTER ( + WHERE f.fingerprint_version >= sqlc.arg(current_version) + AND (f.audio_stream_sha256 IS NULL OR f.chromaprint IS NULL) + )::bigint AS rejected, + count(*) FILTER ( + WHERE f.track_id IS NULL OR f.fingerprint_version < sqlc.arg(current_version) + )::bigint AS pending + FROM tracks t + LEFT JOIN track_fingerprints f ON f.track_id = t.id + WHERE t.missing_since IS NULL; diff --git a/internal/library/fingerprint.go b/internal/library/fingerprint.go index 59ac7cb7..ebd6b3db 100644 --- a/internal/library/fingerprint.go +++ b/internal/library/fingerprint.go @@ -7,6 +7,7 @@ import ( "encoding/hex" "errors" "fmt" + "log/slog" "os/exec" "strconv" "strings" @@ -148,38 +149,55 @@ func (s *Scanner) fingerprintFile(ctx context.Context, path string) fingerprintR return s.fingerprint(ctx, path) } -// storeFingerprint records one attempt for a track whose bytes are new or have -// changed. It never fails the scan: a missing fingerprint only keeps a track -// out of duplicate detection, which is not worth dropping the track over. -func (s *Scanner) storeFingerprint( - ctx context.Context, q *dbq.Queries, trackID pgtype.UUID, path string, fp fingerprintResult, -) { +// fingerprintOutcome is what storeFingerprint did with one attempt. +type fingerprintOutcome int + +const ( + outcomeFingerprinted fingerprintOutcome = iota // both halves stored + outcomeRejected // stored with a NULL half: a verdict + outcomeInconclusive // nothing stored; worth trying again + outcomeStoreFailed // the write itself failed +) + +// storeFingerprint records one attempt, for the scan (new or changed bytes) and +// the backfill (#3908) alike, so there is one rule for what gets written. It +// never fails its caller: a missing fingerprint only keeps a track out of +// duplicate detection, which is not worth dropping a scan or a pass over. +func storeFingerprint( + ctx context.Context, q *dbq.Queries, logger *slog.Logger, + trackID pgtype.UUID, path string, fp fingerprintResult, +) fingerprintOutcome { if fp.hashErr != nil { - s.logger.Warn("library scan: audio stream hash failed", "path", path, "err", fp.hashErr) + logger.Warn("fingerprint: audio stream hash failed", "path", path, "err", fp.hashErr) } if fp.printErr != nil { - s.logger.Warn("library scan: chromaprint failed", "path", path, "err", fp.printErr) + logger.Warn("fingerprint: chromaprint failed", "path", path, "err", fp.printErr) } if fp.inconclusive() { - // Any row this track holds describes its PREVIOUS bytes. Drop it and - // leave the track to the backfill, rather than stamping a failure that - // says nothing about this file. + // Any row this track holds describes bytes we could not confirm — the + // previous bytes for the scan, an older derivation for the backfill. + // Drop it rather than stamp a failure that says nothing about the file. if err := q.DeleteTrackFingerprint(ctx, trackID); err != nil { - s.logger.Warn("library scan: clearing stale fingerprint failed", "path", path, "err", err) + logger.Warn("fingerprint: clearing stale fingerprint failed", "path", path, "err", err) } - return + return outcomeInconclusive } // A NULL half here is a verdict — the tool ran and rejected this file — and // is stamped at the current version so the backfill does not retry it on - // every boot. It is retried when the file changes. + // every pass. It is retried when the file changes. if err := q.UpsertTrackFingerprint(ctx, dbq.UpsertTrackFingerprintParams{ TrackID: trackID, AudioStreamSha256: fp.streamSHA256, Chromaprint: fp.chromaprint, FingerprintVersion: fingerprintVersion, }); err != nil { - s.logger.Warn("library scan: storing fingerprint failed", "path", path, "err", err) + logger.Warn("fingerprint: storing fingerprint failed", "path", path, "err", err) + return outcomeStoreFailed } + if fp.hashErr != nil || fp.printErr != nil { + return outcomeRejected + } + return outcomeFingerprinted } // computeAudioStreamSHA256 returns the SHA-256 of the file's encoded audio. diff --git a/internal/library/fingerprint_backfill.go b/internal/library/fingerprint_backfill.go new file mode 100644 index 00000000..316a335b --- /dev/null +++ b/internal/library/fingerprint_backfill.go @@ -0,0 +1,196 @@ +package library + +import ( + "context" + "fmt" + "log/slog" + "sync" + "time" + + "github.com/jackc/pgx/v5/pgtype" + "github.com/jackc/pgx/v5/pgxpool" + + "git.fabledsword.com/bvandeusen/minstrel/internal/db/dbq" +) + +// Fingerprint backfill (M400 #3908). +// +// The scan fingerprints only bytes it has not seen (see scanFile), so every track +// imported before fingerprinting existed — and every row derived by an older +// fingerprintVersion — needs a pass of its own. That pass is this worker. +// +// Its own worker rather than a stage in RunScan, for two reasons, both about +// time: +// - RunScan runs at boot and then every safetyNetScanInterval (12h), and an +// in-flight scan older than StuckScanThreshold (1h) is reaped and a second +// one started beside it. A stage would have to stop well inside the hour — a +// few hundred decodes — so a 50k-track library would take about a month. +// - A long stage holds the scan run in flight, and a manual rescan answers 409 +// for as long as it runs. +// +// Progress is read live (FingerprintCoverage, the admin gauge) rather than from a +// scan_runs tally: the work spans many passes with no single run to attach to. + +// fingerprintBackfillTick is how often the worker looks for work. Once the +// library has caught up, a tick is one indexed query; mostly the hour bounds how +// long a file that timed out on a slow mount waits before it is tried again. +const fingerprintBackfillTick = time.Hour + +// fingerprintBackfillBatch is how many tracks one query hands the worker. Small, +// so tracks the scan adds mid-pass are not stuck behind one enormous page. +const fingerprintBackfillBatch = 50 + +// fingerprintBackfillConcurrency is how many files are decoded at once. Two is +// deliberately low: fpcalc and the stream hash compete with playback transcoding +// for CPU and with streaming for the mount, and a backfill that makes playback +// stutter is worse than one that takes longer. Operator-tunable in #3913. +const fingerprintBackfillConcurrency = 2 + +// BackfillFingerprintsResult tallies one pass. +type BackfillFingerprintsResult struct { + Processed int + Fingerprinted int // both halves stored + Rejected int // stored with a NULL half: a tool refused the file (settled) + Inconclusive int // nothing stored; tried again on a later pass +} + +func (r *BackfillFingerprintsResult) add(o fingerprintOutcome) { + r.Processed++ + switch o { + case outcomeFingerprinted: + r.Fingerprinted++ + case outcomeRejected: + r.Rejected++ + default: + r.Inconclusive++ + } +} + +// FingerprintBackfillWorker fingerprints the tracks the scan never will. +type FingerprintBackfillWorker struct { + pool *pgxpool.Pool + logger *slog.Logger + tick time.Duration + batch int32 + concurrency int + // fingerprint is a field for the same reason as Scanner.fingerprint: an + // integration test pins which tracks a pass touches, not what the tools print. + fingerprint func(ctx context.Context, path string) fingerprintResult +} + +// NewFingerprintBackfillWorker builds a worker with the production cadence. +func NewFingerprintBackfillWorker(pool *pgxpool.Pool, logger *slog.Logger) *FingerprintBackfillWorker { + return &FingerprintBackfillWorker{ + pool: pool, + logger: logger, + tick: fingerprintBackfillTick, + batch: fingerprintBackfillBatch, + concurrency: fingerprintBackfillConcurrency, + fingerprint: computeFingerprint, + } +} + +// Run blocks until ctx is cancelled: one pass at start, so a fresh deploy does +// not sit idle for an hour, then one per tick. +func (w *FingerprintBackfillWorker) Run(ctx context.Context) { + w.runOnce(ctx) + t := time.NewTicker(w.tick) + defer t.Stop() + for { + select { + case <-ctx.Done(): + return + case <-t.C: + w.runOnce(ctx) + } + } +} + +// runOnce contains a pass so that nothing it does — an error, a panic — can stop +// the next tick from firing (rule 157). +func (w *FingerprintBackfillWorker) runOnce(ctx context.Context) { + defer func() { + if r := recover(); r != nil { + w.logger.Error("fingerprint backfill: pass panicked", "panic", r) + } + }() + res, err := w.pass(ctx) + if err != nil && ctx.Err() == nil { + w.logger.Warn("fingerprint backfill: pass failed", "err", err, "processed", res.Processed) + } + if res.Processed > 0 { + w.logger.Info("fingerprint backfill: pass complete", + "processed", res.Processed, "fingerprinted", res.Fingerprinted, + "rejected", res.Rejected, "inconclusive", res.Inconclusive) + } +} + +// pass walks every track needing a fingerprint once, keyset-paged on id. The +// cursor is what lets a pass end: an inconclusive attempt writes no row, so a +// file that keeps timing out would otherwise be listed again immediately and +// retried forever within the pass. +func (w *FingerprintBackfillWorker) pass(ctx context.Context) (BackfillFingerprintsResult, error) { + q := dbq.New(w.pool) + var ( + res BackfillFingerprintsResult + mu sync.Mutex + ) + // The all-zero uuid sorts before every real id. Valid must be true: a NULL + // cursor would make "id > NULL" match nothing and every pass a silent no-op. + after := pgtype.UUID{Valid: true} + for { + if err := ctx.Err(); err != nil { + return res, err + } + rows, err := q.ListTracksNeedingFingerprint(ctx, dbq.ListTracksNeedingFingerprintParams{ + CurrentVersion: fingerprintVersion, + AfterID: after, + BatchLimit: w.batch, + }) + if err != nil { + return res, fmt.Errorf("list tracks needing fingerprint: %w", err) + } + if len(rows) == 0 { + return res, nil + } + + sem := make(chan struct{}, w.concurrency) + var wg sync.WaitGroup + for _, row := range rows { + if ctx.Err() != nil { + break + } + sem <- struct{}{} + wg.Add(1) + go func(trackID pgtype.UUID, path string) { + defer wg.Done() + defer func() { <-sem }() + defer func() { + if r := recover(); r != nil { + w.logger.Error("fingerprint backfill: track panicked", "path", path, "panic", r) + } + }() + outcome := storeFingerprint(ctx, q, w.logger, trackID, path, w.fingerprintFile(ctx, path)) + mu.Lock() + res.add(outcome) + mu.Unlock() + }(row.ID, row.FilePath) + } + wg.Wait() + after = rows[len(rows)-1].ID + } +} + +func (w *FingerprintBackfillWorker) fingerprintFile(ctx context.Context, path string) fingerprintResult { + if w.fingerprint == nil { + return computeFingerprint(ctx, path) + } + return w.fingerprint(ctx, path) +} + +// FingerprintCoverage reports how much of the library carries a current +// fingerprint, for the admin gauge. It lives here, beside the backfill, so the +// version it counts against is the one the backfill writes. +func FingerprintCoverage(ctx context.Context, pool *pgxpool.Pool) (dbq.GetFingerprintCoverageRow, error) { + return dbq.New(pool).GetFingerprintCoverage(ctx, fingerprintVersion) +} diff --git a/internal/library/fingerprint_backfill_test.go b/internal/library/fingerprint_backfill_test.go new file mode 100644 index 00000000..8b5b81c9 --- /dev/null +++ b/internal/library/fingerprint_backfill_test.go @@ -0,0 +1,156 @@ +package library + +import ( + "bytes" + "context" + "errors" + "fmt" + "io" + "log/slog" + "path/filepath" + "sync" + "testing" + + "git.fabledsword.com/bvandeusen/minstrel/internal/db/dbq" +) + +// TestFingerprintBackfill_Integration pins which tracks a pass touches, that a +// pass ends, and that the coverage gauge counts what the pass wrote. +func TestFingerprintBackfill_Integration(t *testing.T) { + pool := newPool(t) + ctx := context.Background() + q := dbq.New(pool) + dir := t.TempDir() + + _, album, artist := seedTrack(t, pool, filepath.Join(dir, "unfingerprinted.mp3")) + addTrack := func(name string) dbq.Track { + t.Helper() + tr, err := q.UpsertTrack(ctx, dbq.UpsertTrackParams{ + Title: name, AlbumID: album.ID, ArtistID: artist.ID, + DurationMs: 1000, FilePath: filepath.Join(dir, name+".mp3"), FileSize: 100, FileFormat: "mp3", + }) + if err != nil { + t.Fatalf("track %s: %v", name, err) + } + return tr + } + current := addTrack("current") + stale := addTrack("stale") + missing := addTrack("missing") + + sum := bytes.Repeat([]byte{0xCD}, 32) + for _, seed := range []struct { + track dbq.Track + version int16 + }{ + {current, fingerprintVersion}, + {stale, fingerprintVersion - 1}, + } { + if err := q.UpsertTrackFingerprint(ctx, dbq.UpsertTrackFingerprintParams{ + TrackID: seed.track.ID, AudioStreamSha256: sum, Chromaprint: []int32{1}, + FingerprintVersion: seed.version, + }); err != nil { + t.Fatalf("seed fingerprint: %v", err) + } + } + if _, err := pool.Exec(ctx, "UPDATE tracks SET missing_since = now() WHERE id = $1", missing.ID); err != nil { + t.Fatalf("mark missing: %v", err) + } + + var mu sync.Mutex + calls := map[string]int{} + w := NewFingerprintBackfillWorker(pool, slog.New(slog.NewTextHandler(io.Discard, nil))) + // A batch of one forces the keyset cursor across several queries in a pass. + w.batch = 1 + w.fingerprint = func(_ context.Context, path string) fingerprintResult { + name := filepath.Base(path) + mu.Lock() + calls[name]++ + mu.Unlock() + switch name { + case "stall.mp3": + return fingerprintResult{streamSHA256: sum, printErr: fmt.Errorf("fpcalc: %w", errFingerprintTimeout)} + case "rejected.mp3": + return fingerprintResult{hashErr: errors.New("ffmpeg exited 1"), printErr: errors.New("fpcalc exited 2")} + default: + return fingerprintResult{streamSHA256: sum, chromaprint: []int32{7, -7}} + } + } + callCount := func(name string) int { + mu.Lock() + defer mu.Unlock() + return calls[name] + } + + // 1. Only the track with no row and the stale one are fingerprinted — never + // the current one, never the missing one. + res, err := w.pass(ctx) + if err != nil { + t.Fatalf("first pass: %v", err) + } + if res.Processed != 2 || res.Fingerprinted != 2 { + t.Fatalf("first pass = %+v, want 2 processed, 2 fingerprinted", res) + } + for name, want := range map[string]int{ + "unfingerprinted.mp3": 1, "stale.mp3": 1, "current.mp3": 0, "missing.mp3": 0, + } { + if got := callCount(name); got != want { + t.Errorf("%s fingerprinted %d times, want %d", name, got, want) + } + } + + // 2. A pass after a complete one is a no-op. A backfill that redoes its work + // every hour is the expensive way this could be wrong. + res, err = w.pass(ctx) + if err != nil { + t.Fatalf("second pass: %v", err) + } + if res.Processed != 0 { + t.Fatalf("second pass processed %d tracks, want 0", res.Processed) + } + + // 3. An inconclusive file is tried exactly once and the pass ENDS. Without the + // keyset cursor it would be re-listed immediately and this call would never + // return. + addTrack("stall") + addTrack("rejected") + res, err = w.pass(ctx) + if err != nil { + t.Fatalf("third pass: %v", err) + } + if res.Processed != 2 || res.Inconclusive != 1 || res.Rejected != 1 { + t.Fatalf("third pass = %+v, want 2 processed, 1 inconclusive, 1 rejected", res) + } + if got := callCount("stall.mp3"); got != 1 { + t.Fatalf("stalling file tried %d times in one pass, want exactly 1", got) + } + + // 4. The gauge counts what the passes wrote, and its buckets add up. + cov, err := FingerprintCoverage(ctx, pool) + if err != nil { + t.Fatalf("coverage: %v", err) + } + // Five present tracks: unfingerprinted, current, stale, stall, rejected. + // The missing track is not counted. + if cov.Total != 5 || cov.Fingerprinted != 3 || cov.Rejected != 1 || cov.Pending != 1 { + t.Errorf("coverage = %+v, want total 5, fingerprinted 3, rejected 1, pending 1", cov) + } + if cov.Fingerprinted+cov.Rejected+cov.Pending != cov.Total { + t.Errorf("coverage buckets %+v do not sum to the total", cov) + } +} + +func TestBackfillFingerprintsResult_Add(t *testing.T) { + var r BackfillFingerprintsResult + for _, o := range []fingerprintOutcome{ + outcomeFingerprinted, outcomeFingerprinted, outcomeRejected, outcomeInconclusive, outcomeStoreFailed, + } { + r.add(o) + } + // A failed write stored nothing, so like an inconclusive attempt it is + // tried again next pass — and counts as such. + want := BackfillFingerprintsResult{Processed: 5, Fingerprinted: 2, Rejected: 1, Inconclusive: 2} + if r != want { + t.Errorf("tally = %+v, want %+v", r, want) + } +} diff --git a/internal/library/scanner.go b/internal/library/scanner.go index c23dbd94..28770e89 100644 --- a/internal/library/scanner.go +++ b/internal/library/scanner.go @@ -384,7 +384,7 @@ func (s *Scanner) scanFile( s.logger.Warn("library scan: LogChange track upsert failed", "track_id", track.ID, "err", err) } if fingerprinted { - s.storeFingerprint(ctx, q, track.ID, path, fp) + storeFingerprint(ctx, q, s.logger, track.ID, path, fp) } if knownTrack { diff --git a/web/src/lib/api/admin.fingerprints.test.ts b/web/src/lib/api/admin.fingerprints.test.ts new file mode 100644 index 00000000..9442fe45 --- /dev/null +++ b/web/src/lib/api/admin.fingerprints.test.ts @@ -0,0 +1,32 @@ +import { describe, it, expect, vi, beforeEach } from 'vitest'; +import { getFingerprintCoverage, type FingerprintCoverage } from './admin'; + +vi.mock('./client', () => ({ + api: { get: vi.fn(), post: vi.fn() } +})); + +import { api } from './client'; + +describe('admin fingerprint coverage API', () => { + beforeEach(() => vi.clearAllMocks()); + + it('getFingerprintCoverage GETs the correct path', async () => { + const sample: FingerprintCoverage = { + total: 18026, + fingerprinted: 9400, + rejected: 12, + pending: 8614 + }; + (api.get as unknown as ReturnType).mockResolvedValueOnce(sample); + const got = await getFingerprintCoverage(); + expect(api.get).toHaveBeenCalledWith('/api/admin/library/fingerprints'); + expect(got).toEqual(sample); + }); + + it('buckets sum to the total', async () => { + const sample: FingerprintCoverage = { total: 10, fingerprinted: 6, rejected: 1, pending: 3 }; + (api.get as unknown as ReturnType).mockResolvedValueOnce(sample); + const got = await getFingerprintCoverage(); + expect(got.fingerprinted + got.rejected + got.pending).toBe(got.total); + }); +}); diff --git a/web/src/lib/api/admin.ts b/web/src/lib/api/admin.ts index 2428d79f..f4e209e0 100644 --- a/web/src/lib/api/admin.ts +++ b/web/src/lib/api/admin.ts @@ -313,6 +313,30 @@ export function createCoverageQuery() { }); } +// Fingerprint backfill (#3908) -------------------------------------------- + +export type FingerprintCoverage = { + total: number; + fingerprinted: number; + rejected: number; + pending: number; +}; + +export async function getFingerprintCoverage(): Promise { + return api.get('/api/admin/library/fingerprints'); +} + +// Polled far less often than the cover gauge: the backfill decodes files two at +// a time, so the count moves by a few tracks a minute and a 3s poll is noise. +export function createFingerprintCoverageQuery() { + return createQuery({ + queryKey: qk.fingerprintCoverage(), + queryFn: getFingerprintCoverage, + staleTime: 30_000, + refetchInterval: 30_000 + }); +} + // Cover-art providers ------------------------------------------------------ export type CoverProviderCapability = 'album_cover' | 'artist_thumb' | 'artist_fanart'; diff --git a/web/src/lib/api/queries.ts b/web/src/lib/api/queries.ts index f5586019..7e62f0f4 100644 --- a/web/src/lib/api/queries.ts +++ b/web/src/lib/api/queries.ts @@ -47,6 +47,7 @@ export const qk = { ['adminPlaybackErrors', { resolved: resolved ?? false }] as const, scanStatus: () => ['scanStatus'] as const, coverage: () => ['coverage'] as const, + fingerprintCoverage: () => ['fingerprintCoverage'] as const, coverProviders: () => ['coverProviders'] as const, tagProviders: () => ['tagProviders'] as const, adminUsers: () => ['adminUsers'] as const, diff --git a/web/src/routes/admin/+page.svelte b/web/src/routes/admin/+page.svelte index 0ca34f8d..df5334ae 100644 --- a/web/src/routes/admin/+page.svelte +++ b/web/src/routes/admin/+page.svelte @@ -10,6 +10,7 @@ createAdminQuarantineQuery, createScanStatusQuery, createCoverageQuery, + createFingerprintCoverageQuery, approveRequest, rejectRequest, resolveQuarantine, @@ -177,6 +178,11 @@ const coverageQ = $derived($coverageStore); const coverage = $derived(coverageQ.data); + // ---- Fingerprint backfill gauge (#3908) ---- + const fingerprintStore = $derived(createFingerprintCoverageQuery()); + const fingerprintQ = $derived($fingerprintStore); + const fingerprints = $derived(fingerprintQ.data); + let triggering = $state(false); let triggerResult = $state(null); @@ -428,6 +434,28 @@ {#if triggerResult}

{triggerResult}

{/if} + + + {#if fingerprints && fingerprints.total > 0} +
+ Fingerprints + {fingerprints.fingerprinted.toLocaleString()} of {fingerprints.total.toLocaleString()} tracks + {#if fingerprints.pending > 0} + · + {fingerprints.pending.toLocaleString()} pending + {/if} + {#if fingerprints.rejected > 0} + · + + {fingerprints.rejected.toLocaleString()} unreadable + + {/if} +
+ {/if}