From ca1bc5af62b04013d9472c38f395c1d6087f0491 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Sat, 16 May 2026 14:24:45 -0400 Subject: [PATCH] =?UTF-8?q?perf(similarity):=20worker=20batch=205=E2=86=92?= =?UTF-8?q?25;=20track-MBID=20backfill=20uncapped?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - similarity.Worker batch 5→25 (tracks AND artists per 1h tick). At 5/h a freshly-scrobbled library took days to build a usable similarity pool; 25/h converges in hours, still well under ListenBrainz rate limits (429 aborts the tick). - scanrun Stage 2b track backfill now runs unbounded (-1) instead of reusing the album backfill's 5000 staged cap. It's a one-time whole-library heal with no progress UI; a cap just left tracks.mbid partially NULL (3634/18056 after one pass) until several future scans caught up, re-reading untagged files each time. One uncapped pass converges; later scans only re-read the remaining NULL rows. Album backfill keeps its 5000 cap (it has staged scan_runs UX). Co-Authored-By: Claude Opus 4.7 (1M context) --- internal/library/scanrun.go | 12 +++++++++--- internal/similarity/worker.go | 6 ++++-- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/internal/library/scanrun.go b/internal/library/scanrun.go index e53612ba..d2b72566 100644 --- a/internal/library/scanrun.go +++ b/internal/library/scanrun.go @@ -156,10 +156,16 @@ func RunScan( // Stage 2b: track recording-MBID backfill. Unblocks the ListenBrainz // similarity pipeline (gated on tracks.mbid IS NOT NULL). Log-only - // progress — no scan_runs jsonb column to avoid a schema addition; - // it's idempotent and reruns each scan until every file is healed. + // progress — no scan_runs jsonb column to avoid a schema addition. + // + // Unbounded (-1), unlike the album backfill's 5000 staged cap: this + // is a one-time whole-library heal with no progress UI, and a cap + // just means tracks.mbid stays partially NULL until N future scans + // catch up (re-reading untagged files wastefully each pass). One + // uncapped pass converges; subsequent scans only re-read the + // remaining NULL (genuinely untagged) rows, which is cheap. if cfg.BackfillCap != 0 { - if _, tberr := BackfillTrackMBIDs(ctx, pool, logger, cfg.BackfillCap, nil); tberr != nil { + if _, tberr := BackfillTrackMBIDs(ctx, pool, logger, -1, nil); tberr != nil { captureErr("track_mbid_backfill", tberr) } } diff --git a/internal/similarity/worker.go b/internal/similarity/worker.go index e37cf5be..8ce64ede 100644 --- a/internal/similarity/worker.go +++ b/internal/similarity/worker.go @@ -34,14 +34,16 @@ type Worker struct { } // NewWorker constructs a worker with production defaults: 1h tick, -// batch=5, topK=20. +// batch=25, topK=20. batch is tracks AND artists processed per tick; +// at 25/h a freshly-played library converges in hours, not days, while +// staying well under ListenBrainz rate limits (429s abort the tick). func NewWorker(pool *pgxpool.Pool, client *listenbrainz.Client, logger *slog.Logger) *Worker { return &Worker{ pool: pool, client: client, logger: logger, tick: 1 * time.Hour, - batch: 5, + batch: 25, topK: 20, } }