feat(library): backfill fingerprints for the existing library — M400 #3908

The scan fingerprints only bytes it has not seen, so everything imported
before fingerprinting existed, and any row derived by an older
fingerprintVersion, needs a pass of its own.

That pass is a background worker, not a stage in RunScan. RunScan runs
at boot and then every 12h, and an in-flight scan older than an hour is
reaped and a second started beside it. A stage would have to stop inside
the hour: a few hundred decodes a run, so about a month for a 50k-track
library. It would also hold the run in flight and answer manual
rescans with 409 while it worked.

FingerprintBackfillWorker runs once at start, then hourly. Nothing a
pass does (error or panic) can stop the next tick. A pass walks tracks
with no fingerprint or a stale version, skipping missing tracks,
keyset-paged on id. The cursor is what lets a pass end: an inconclusive
attempt writes no row, so a file that keeps timing out would otherwise
be re-listed and retried forever. Two decodes at a time, deliberately:
they compete with transcoding for CPU and with streaming for the mount.

storeFingerprint is now one package function shared by the scan and
the worker, and reports whether the attempt was fingerprinted,
rejected, inconclusive or failed to store.

Progress is a live gauge on the Admin scan card, served by
GET /api/admin/library/fingerprints: fingerprinted / rejected / pending
of total, with missing tracks excluded so it can reach the end.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01SQ31KQpYbStyK5y58UmPLH

internal/library/fingerprint.go

@@ -7,6 +7,7 @@ import (
 	"encoding/hex"
 	"errors"
 	"fmt"
+	"log/slog"
 	"os/exec"
 	"strconv"
 	"strings"
@@ -148,38 +149,55 @@ func (s *Scanner) fingerprintFile(ctx context.Context, path string) fingerprintR
 	return s.fingerprint(ctx, path)
 }
 
-// storeFingerprint records one attempt for a track whose bytes are new or have
-// changed. It never fails the scan: a missing fingerprint only keeps a track
-// out of duplicate detection, which is not worth dropping the track over.
-func (s *Scanner) storeFingerprint(
-	ctx context.Context, q *dbq.Queries, trackID pgtype.UUID, path string, fp fingerprintResult,
-) {
+// fingerprintOutcome is what storeFingerprint did with one attempt.
+type fingerprintOutcome int
+
+const (
+	outcomeFingerprinted fingerprintOutcome = iota // both halves stored
+	outcomeRejected                                // stored with a NULL half: a verdict
+	outcomeInconclusive                            // nothing stored; worth trying again
+	outcomeStoreFailed                             // the write itself failed
+)
+
+// storeFingerprint records one attempt, for the scan (new or changed bytes) and
+// the backfill (#3908) alike, so there is one rule for what gets written. It
+// never fails its caller: a missing fingerprint only keeps a track out of
+// duplicate detection, which is not worth dropping a scan or a pass over.
+func storeFingerprint(
+	ctx context.Context, q *dbq.Queries, logger *slog.Logger,
+	trackID pgtype.UUID, path string, fp fingerprintResult,
+) fingerprintOutcome {
 	if fp.hashErr != nil {
-		s.logger.Warn("library scan: audio stream hash failed", "path", path, "err", fp.hashErr)
+		logger.Warn("fingerprint: audio stream hash failed", "path", path, "err", fp.hashErr)
 	}
 	if fp.printErr != nil {
-		s.logger.Warn("library scan: chromaprint failed", "path", path, "err", fp.printErr)
+		logger.Warn("fingerprint: chromaprint failed", "path", path, "err", fp.printErr)
 	}
 	if fp.inconclusive() {
-		// Any row this track holds describes its PREVIOUS bytes. Drop it and
-		// leave the track to the backfill, rather than stamping a failure that
-		// says nothing about this file.
+		// Any row this track holds describes bytes we could not confirm — the
+		// previous bytes for the scan, an older derivation for the backfill.
+		// Drop it rather than stamp a failure that says nothing about the file.
 		if err := q.DeleteTrackFingerprint(ctx, trackID); err != nil {
-			s.logger.Warn("library scan: clearing stale fingerprint failed", "path", path, "err", err)
+			logger.Warn("fingerprint: clearing stale fingerprint failed", "path", path, "err", err)
 		}
-		return
+		return outcomeInconclusive
 	}
 	// A NULL half here is a verdict — the tool ran and rejected this file — and
 	// is stamped at the current version so the backfill does not retry it on
-	// every boot. It is retried when the file changes.
+	// every pass. It is retried when the file changes.
 	if err := q.UpsertTrackFingerprint(ctx, dbq.UpsertTrackFingerprintParams{
 		TrackID:            trackID,
 		AudioStreamSha256:  fp.streamSHA256,
 		Chromaprint:        fp.chromaprint,
 		FingerprintVersion: fingerprintVersion,
 	}); err != nil {
-		s.logger.Warn("library scan: storing fingerprint failed", "path", path, "err", err)
+		logger.Warn("fingerprint: storing fingerprint failed", "path", path, "err", err)
+		return outcomeStoreFailed
 	}
+	if fp.hashErr != nil || fp.printErr != nil {
+		return outcomeRejected
+	}
+	return outcomeFingerprinted
 }
 
 // computeAudioStreamSHA256 returns the SHA-256 of the file's encoded audio.

internal/library/fingerprint_backfill.go

@@ -0,0 +1,196 @@
+package library
+
+import (
+	"context"
+	"fmt"
+	"log/slog"
+	"sync"
+	"time"
+
+	"github.com/jackc/pgx/v5/pgtype"
+	"github.com/jackc/pgx/v5/pgxpool"
+
+	"git.fabledsword.com/bvandeusen/minstrel/internal/db/dbq"
+)
+
+// Fingerprint backfill (M400 #3908).
+//
+// The scan fingerprints only bytes it has not seen (see scanFile), so every track
+// imported before fingerprinting existed — and every row derived by an older
+// fingerprintVersion — needs a pass of its own. That pass is this worker.
+//
+// Its own worker rather than a stage in RunScan, for two reasons, both about
+// time:
+//   - RunScan runs at boot and then every safetyNetScanInterval (12h), and an
+//     in-flight scan older than StuckScanThreshold (1h) is reaped and a second
+//     one started beside it. A stage would have to stop well inside the hour — a
+//     few hundred decodes — so a 50k-track library would take about a month.
+//   - A long stage holds the scan run in flight, and a manual rescan answers 409
+//     for as long as it runs.
+//
+// Progress is read live (FingerprintCoverage, the admin gauge) rather than from a
+// scan_runs tally: the work spans many passes with no single run to attach to.
+
+// fingerprintBackfillTick is how often the worker looks for work. Once the
+// library has caught up, a tick is one indexed query; mostly the hour bounds how
+// long a file that timed out on a slow mount waits before it is tried again.
+const fingerprintBackfillTick = time.Hour
+
+// fingerprintBackfillBatch is how many tracks one query hands the worker. Small,
+// so tracks the scan adds mid-pass are not stuck behind one enormous page.
+const fingerprintBackfillBatch = 50
+
+// fingerprintBackfillConcurrency is how many files are decoded at once. Two is
+// deliberately low: fpcalc and the stream hash compete with playback transcoding
+// for CPU and with streaming for the mount, and a backfill that makes playback
+// stutter is worse than one that takes longer. Operator-tunable in #3913.
+const fingerprintBackfillConcurrency = 2
+
+// BackfillFingerprintsResult tallies one pass.
+type BackfillFingerprintsResult struct {
+	Processed     int
+	Fingerprinted int // both halves stored
+	Rejected      int // stored with a NULL half: a tool refused the file (settled)
+	Inconclusive  int // nothing stored; tried again on a later pass
+}
+
+func (r *BackfillFingerprintsResult) add(o fingerprintOutcome) {
+	r.Processed++
+	switch o {
+	case outcomeFingerprinted:
+		r.Fingerprinted++
+	case outcomeRejected:
+		r.Rejected++
+	default:
+		r.Inconclusive++
+	}
+}
+
+// FingerprintBackfillWorker fingerprints the tracks the scan never will.
+type FingerprintBackfillWorker struct {
+	pool        *pgxpool.Pool
+	logger      *slog.Logger
+	tick        time.Duration
+	batch       int32
+	concurrency int
+	// fingerprint is a field for the same reason as Scanner.fingerprint: an
+	// integration test pins which tracks a pass touches, not what the tools print.
+	fingerprint func(ctx context.Context, path string) fingerprintResult
+}
+
+// NewFingerprintBackfillWorker builds a worker with the production cadence.
+func NewFingerprintBackfillWorker(pool *pgxpool.Pool, logger *slog.Logger) *FingerprintBackfillWorker {
+	return &FingerprintBackfillWorker{
+		pool:        pool,
+		logger:      logger,
+		tick:        fingerprintBackfillTick,
+		batch:       fingerprintBackfillBatch,
+		concurrency: fingerprintBackfillConcurrency,
+		fingerprint: computeFingerprint,
+	}
+}
+
+// Run blocks until ctx is cancelled: one pass at start, so a fresh deploy does
+// not sit idle for an hour, then one per tick.
+func (w *FingerprintBackfillWorker) Run(ctx context.Context) {
+	w.runOnce(ctx)
+	t := time.NewTicker(w.tick)
+	defer t.Stop()
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		case <-t.C:
+			w.runOnce(ctx)
+		}
+	}
+}
+
+// runOnce contains a pass so that nothing it does — an error, a panic — can stop
+// the next tick from firing (rule 157).
+func (w *FingerprintBackfillWorker) runOnce(ctx context.Context) {
+	defer func() {
+		if r := recover(); r != nil {
+			w.logger.Error("fingerprint backfill: pass panicked", "panic", r)
+		}
+	}()
+	res, err := w.pass(ctx)
+	if err != nil && ctx.Err() == nil {
+		w.logger.Warn("fingerprint backfill: pass failed", "err", err, "processed", res.Processed)
+	}
+	if res.Processed > 0 {
+		w.logger.Info("fingerprint backfill: pass complete",
+			"processed", res.Processed, "fingerprinted", res.Fingerprinted,
+			"rejected", res.Rejected, "inconclusive", res.Inconclusive)
+	}
+}
+
+// pass walks every track needing a fingerprint once, keyset-paged on id. The
+// cursor is what lets a pass end: an inconclusive attempt writes no row, so a
+// file that keeps timing out would otherwise be listed again immediately and
+// retried forever within the pass.
+func (w *FingerprintBackfillWorker) pass(ctx context.Context) (BackfillFingerprintsResult, error) {
+	q := dbq.New(w.pool)
+	var (
+		res BackfillFingerprintsResult
+		mu  sync.Mutex
+	)
+	// The all-zero uuid sorts before every real id. Valid must be true: a NULL
+	// cursor would make "id > NULL" match nothing and every pass a silent no-op.
+	after := pgtype.UUID{Valid: true}
+	for {
+		if err := ctx.Err(); err != nil {
+			return res, err
+		}
+		rows, err := q.ListTracksNeedingFingerprint(ctx, dbq.ListTracksNeedingFingerprintParams{
+			CurrentVersion: fingerprintVersion,
+			AfterID:        after,
+			BatchLimit:     w.batch,
+		})
+		if err != nil {
+			return res, fmt.Errorf("list tracks needing fingerprint: %w", err)
+		}
+		if len(rows) == 0 {
+			return res, nil
+		}
+
+		sem := make(chan struct{}, w.concurrency)
+		var wg sync.WaitGroup
+		for _, row := range rows {
+			if ctx.Err() != nil {
+				break
+			}
+			sem <- struct{}{}
+			wg.Add(1)
+			go func(trackID pgtype.UUID, path string) {
+				defer wg.Done()
+				defer func() { <-sem }()
+				defer func() {
+					if r := recover(); r != nil {
+						w.logger.Error("fingerprint backfill: track panicked", "path", path, "panic", r)
+					}
+				}()
+				outcome := storeFingerprint(ctx, q, w.logger, trackID, path, w.fingerprintFile(ctx, path))
+				mu.Lock()
+				res.add(outcome)
+				mu.Unlock()
+			}(row.ID, row.FilePath)
+		}
+		wg.Wait()
+		after = rows[len(rows)-1].ID
+	}
+}
+
+func (w *FingerprintBackfillWorker) fingerprintFile(ctx context.Context, path string) fingerprintResult {
+	if w.fingerprint == nil {
+		return computeFingerprint(ctx, path)
+	}
+	return w.fingerprint(ctx, path)
+}
+
+// FingerprintCoverage reports how much of the library carries a current
+// fingerprint, for the admin gauge. It lives here, beside the backfill, so the
+// version it counts against is the one the backfill writes.
+func FingerprintCoverage(ctx context.Context, pool *pgxpool.Pool) (dbq.GetFingerprintCoverageRow, error) {
+	return dbq.New(pool).GetFingerprintCoverage(ctx, fingerprintVersion)
+}

internal/library/fingerprint_backfill_test.go

@@ -0,0 +1,156 @@
+package library
+
+import (
+	"bytes"
+	"context"
+	"errors"
+	"fmt"
+	"io"
+	"log/slog"
+	"path/filepath"
+	"sync"
+	"testing"
+
+	"git.fabledsword.com/bvandeusen/minstrel/internal/db/dbq"
+)
+
+// TestFingerprintBackfill_Integration pins which tracks a pass touches, that a
+// pass ends, and that the coverage gauge counts what the pass wrote.
+func TestFingerprintBackfill_Integration(t *testing.T) {
+	pool := newPool(t)
+	ctx := context.Background()
+	q := dbq.New(pool)
+	dir := t.TempDir()
+
+	_, album, artist := seedTrack(t, pool, filepath.Join(dir, "unfingerprinted.mp3"))
+	addTrack := func(name string) dbq.Track {
+		t.Helper()
+		tr, err := q.UpsertTrack(ctx, dbq.UpsertTrackParams{
+			Title: name, AlbumID: album.ID, ArtistID: artist.ID,
+			DurationMs: 1000, FilePath: filepath.Join(dir, name+".mp3"), FileSize: 100, FileFormat: "mp3",
+		})
+		if err != nil {
+			t.Fatalf("track %s: %v", name, err)
+		}
+		return tr
+	}
+	current := addTrack("current")
+	stale := addTrack("stale")
+	missing := addTrack("missing")
+
+	sum := bytes.Repeat([]byte{0xCD}, 32)
+	for _, seed := range []struct {
+		track   dbq.Track
+		version int16
+	}{
+		{current, fingerprintVersion},
+		{stale, fingerprintVersion - 1},
+	} {
+		if err := q.UpsertTrackFingerprint(ctx, dbq.UpsertTrackFingerprintParams{
+			TrackID: seed.track.ID, AudioStreamSha256: sum, Chromaprint: []int32{1},
+			FingerprintVersion: seed.version,
+		}); err != nil {
+			t.Fatalf("seed fingerprint: %v", err)
+		}
+	}
+	if _, err := pool.Exec(ctx, "UPDATE tracks SET missing_since = now() WHERE id = $1", missing.ID); err != nil {
+		t.Fatalf("mark missing: %v", err)
+	}
+
+	var mu sync.Mutex
+	calls := map[string]int{}
+	w := NewFingerprintBackfillWorker(pool, slog.New(slog.NewTextHandler(io.Discard, nil)))
+	// A batch of one forces the keyset cursor across several queries in a pass.
+	w.batch = 1
+	w.fingerprint = func(_ context.Context, path string) fingerprintResult {
+		name := filepath.Base(path)
+		mu.Lock()
+		calls[name]++
+		mu.Unlock()
+		switch name {
+		case "stall.mp3":
+			return fingerprintResult{streamSHA256: sum, printErr: fmt.Errorf("fpcalc: %w", errFingerprintTimeout)}
+		case "rejected.mp3":
+			return fingerprintResult{hashErr: errors.New("ffmpeg exited 1"), printErr: errors.New("fpcalc exited 2")}
+		default:
+			return fingerprintResult{streamSHA256: sum, chromaprint: []int32{7, -7}}
+		}
+	}
+	callCount := func(name string) int {
+		mu.Lock()
+		defer mu.Unlock()
+		return calls[name]
+	}
+
+	// 1. Only the track with no row and the stale one are fingerprinted — never
+	// the current one, never the missing one.
+	res, err := w.pass(ctx)
+	if err != nil {
+		t.Fatalf("first pass: %v", err)
+	}
+	if res.Processed != 2 || res.Fingerprinted != 2 {
+		t.Fatalf("first pass = %+v, want 2 processed, 2 fingerprinted", res)
+	}
+	for name, want := range map[string]int{
+		"unfingerprinted.mp3": 1, "stale.mp3": 1, "current.mp3": 0, "missing.mp3": 0,
+	} {
+		if got := callCount(name); got != want {
+			t.Errorf("%s fingerprinted %d times, want %d", name, got, want)
+		}
+	}
+
+	// 2. A pass after a complete one is a no-op. A backfill that redoes its work
+	// every hour is the expensive way this could be wrong.
+	res, err = w.pass(ctx)
+	if err != nil {
+		t.Fatalf("second pass: %v", err)
+	}
+	if res.Processed != 0 {
+		t.Fatalf("second pass processed %d tracks, want 0", res.Processed)
+	}
+
+	// 3. An inconclusive file is tried exactly once and the pass ENDS. Without the
+	// keyset cursor it would be re-listed immediately and this call would never
+	// return.
+	addTrack("stall")
+	addTrack("rejected")
+	res, err = w.pass(ctx)
+	if err != nil {
+		t.Fatalf("third pass: %v", err)
+	}
+	if res.Processed != 2 || res.Inconclusive != 1 || res.Rejected != 1 {
+		t.Fatalf("third pass = %+v, want 2 processed, 1 inconclusive, 1 rejected", res)
+	}
+	if got := callCount("stall.mp3"); got != 1 {
+		t.Fatalf("stalling file tried %d times in one pass, want exactly 1", got)
+	}
+
+	// 4. The gauge counts what the passes wrote, and its buckets add up.
+	cov, err := FingerprintCoverage(ctx, pool)
+	if err != nil {
+		t.Fatalf("coverage: %v", err)
+	}
+	// Five present tracks: unfingerprinted, current, stale, stall, rejected.
+	// The missing track is not counted.
+	if cov.Total != 5 || cov.Fingerprinted != 3 || cov.Rejected != 1 || cov.Pending != 1 {
+		t.Errorf("coverage = %+v, want total 5, fingerprinted 3, rejected 1, pending 1", cov)
+	}
+	if cov.Fingerprinted+cov.Rejected+cov.Pending != cov.Total {
+		t.Errorf("coverage buckets %+v do not sum to the total", cov)
+	}
+}
+
+func TestBackfillFingerprintsResult_Add(t *testing.T) {
+	var r BackfillFingerprintsResult
+	for _, o := range []fingerprintOutcome{
+		outcomeFingerprinted, outcomeFingerprinted, outcomeRejected, outcomeInconclusive, outcomeStoreFailed,
+	} {
+		r.add(o)
+	}
+	// A failed write stored nothing, so like an inconclusive attempt it is
+	// tried again next pass — and counts as such.
+	want := BackfillFingerprintsResult{Processed: 5, Fingerprinted: 2, Rejected: 1, Inconclusive: 2}
+	if r != want {
+		t.Errorf("tally = %+v, want %+v", r, want)
+	}
+}

internal/library/scanner.go

@@ -384,7 +384,7 @@ func (s *Scanner) scanFile(
 		s.logger.Warn("library scan: LogChange track upsert failed", "track_id", track.ID, "err", err)
 	}
 	if fingerprinted {
-		s.storeFingerprint(ctx, q, track.ID, path, fp)
+		storeFingerprint(ctx, q, s.logger, track.ID, path, fp)
 	}
 
 	if knownTrack {