bvandeusen/minstrel
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 38 Wiki Activity

feat(coverart): #388 remove global cover-art backfill cap

Browse Source 
Operator feedback (2026-05-09): the 500-album cap meant cover art rolled
in over many nightly runs even when local sources (sidecar/embedded)
could finish in minutes. Remove the global cap; rely on the existing
per-provider HTTP throttle (coverart httpClient MinInterval) so local
art is disk-speed and remote providers stay TOS-friendly.

- enricher.go / artist_enricher.go: EnrichBatch, EnrichArtistBatch,
  EnrichRetryMissing now treat limit<0 as unbounded (0 still = stage
  disabled). The cap was a SQL LIMIT; unbounded uses max int32.
- main.go: RunScanConfig EnrichCap/ArtistEnrichCap = -1 (unbounded).
- Drop LibraryConfig.CoverArtBackfillCap + the
  MINSTREL_LIBRARY_COVERART_BACKFILL_CAP env var.
- Drop the now-dead coverBackfillCap param threaded through
  server.New + api.Mount + the handlers struct.
- Admin bulk refetch (/api/admin/covers/refetch-missing) now drains
  unbounded; response {queued:int} → {started:bool} (the count is
  unknowable synchronously for a fire-and-forget drain). Web copy +
  client type + Go/web tests updated to match.

No doc refs existed (config.example.yaml / docker-compose / README
never documented the env var).

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
Bryan Van Deusen
2026-05-16 18:41:21 -04:00
parent 4fca0e66cb
commit 005965d6de
14 changed files with 107 additions and 93 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cmd/minstrel/main.go
+7 -7
View File
@@ -105,8 +105,8 @@ func run() error {
logger.With("component", "scan_run"),
library.RunScanConfig{
BackfillCap: 5000,
EnrichCap: cfg.Library.CoverArtBackfillCap,
ArtistEnrichCap: cfg.Library.CoverArtBackfillCap,
EnrichCap: -1, // #388: no global cap; per-provider rate limits
ArtistEnrichCap: -1,
DataDir: cfg.Storage.DataDir,
},
); err != nil {
@@ -122,8 +122,8 @@ func run() error {
logger.With("component", "scan_run"),
library.RunScanConfig{
BackfillCap: 5000,
EnrichCap: cfg.Library.CoverArtBackfillCap,
ArtistEnrichCap: cfg.Library.CoverArtBackfillCap,
EnrichCap: -1, // #388: no global cap; per-provider rate limits
ArtistEnrichCap: -1,
DataDir: cfg.Storage.DataDir,
},
); err != nil {
@@ -194,8 +194,8 @@ func run() error {
scanCfg := library.RunScanConfig{
BackfillCap: 5000,
EnrichCap: cfg.Library.CoverArtBackfillCap,
ArtistEnrichCap: cfg.Library.CoverArtBackfillCap,
EnrichCap: -1, // #388: no global cap; per-provider rate limits
ArtistEnrichCap: -1,
DataDir: cfg.Storage.DataDir,
}
scheduler := library.NewScheduler(pool, logger.With("component", "scheduler"),
@@ -203,7 +203,7 @@ func run() error {
scheduler.Start(ctx)
srv := server.New(logger, pool, scanner, subsonic.Config{
AllowPlaintextPassword: cfg.Subsonic.AllowPlaintextPassword,
}, cfg.Events, cfg.Recommendation, cfg.Storage.DataDir, cfg.Branding, coverEnricher, cfg.Library.CoverArtBackfillCap, coverSettings, scanner, scanCfg, scheduler)
}, cfg.Events, cfg.Recommendation, cfg.Storage.DataDir, cfg.Branding, coverEnricher, coverSettings, scanner, scanCfg, scheduler)
srv.Bus = bus
srv.PlaylistScheduler = playlistScheduler
httpServer := &http.Server{
internal/api/admin_covers.go
+7 -6
View File
@@ -48,19 +48,20 @@ func (h *handlers) handleAdminAlbumRefetchCover(w http.ResponseWriter, r *http.R
}
type adminBulkRefetchResp struct {
Queued int `json:"queued"`
Started bool `json:"started"`
}
// handleAdminBulkRefetchCovers implements POST /api/admin/covers/refetch-missing.
// Returns immediately; actual drain runs in a background goroutine bounded by
// the configured backfill cap.
// Returns immediately; the drain runs UNBOUNDED in a background goroutine
// (#388 — no global cap; remote providers self-throttle per provider, local
// sources run at disk speed). The count is unknowable synchronously, so the
// response just acknowledges the drain started.
func (h *handlers) handleAdminBulkRefetchCovers(w http.ResponseWriter, _ *http.Request) {
cap := h.coverArtBackfillCap
go func() {
bgCtx := context.Background()
if _, err := h.coverart.EnrichRetryMissing(bgCtx, cap); err != nil {
if _, err := h.coverart.EnrichRetryMissing(bgCtx, -1); err != nil {
h.logger.Warn("admin: bulk cover refetch failed", "err", err)
}
}()
writeJSON(w, http.StatusOK, adminBulkRefetchResp{Queued: cap})
writeJSON(w, http.StatusOK, adminBulkRefetchResp{Started: true})
}
internal/api/admin_covers_test.go
+3 -4
View File
@@ -53,7 +53,6 @@ func testHandlersWithCovers(t *testing.T) (*handlers, *coverart.Enricher) {
}
enricher := coverart.NewEnricher(pool, slog.Default(), settings)
h.coverart = enricher
h.coverArtBackfillCap = 100
return h, enricher
}
@@ -97,7 +96,7 @@ func TestAdminAlbumRefetchCover_AdminOK(t *testing.T) {
}
}
func TestAdminBulkRefetchCovers_AdminReturnsQueuedCount(t *testing.T) {
func TestAdminBulkRefetchCovers_AdminStartsRefetch(t *testing.T) {
if os.Getenv("MINSTREL_TEST_DATABASE_URL") == "" {
t.Skip("MINSTREL_TEST_DATABASE_URL not set")
}
@@ -115,8 +114,8 @@ func TestAdminBulkRefetchCovers_AdminReturnsQueuedCount(t *testing.T) {
if err := json.Unmarshal(w.Body.Bytes(), &resp); err != nil {
t.Fatalf("decode: %v", err)
}
if resp.Queued != h.coverArtBackfillCap {
t.Errorf("queued = %d, want %d (cap)", resp.Queued, h.coverArtBackfillCap)
if !resp.Started {
t.Errorf("started = false, want true (bulk refetch should acknowledge)")
}
}
internal/api/api.go
+35 -37
View File
@@ -28,26 +28,25 @@ import (
// Mount attaches /api/* handlers to r. Public endpoints (login) are outside
// RequireUser; everything else is gated by the middleware. The events writer
// is shared with the Subsonic mount so /rest/scrobble feeds the same store.
func Mount(r chi.Router, pool *pgxpool.Pool, logger *slog.Logger, events *playevents.Writer, recCfg config.RecommendationConfig, lidarrCfg *lidarrconfig.Service, lidarrReqs *lidarrrequests.Service, lidarrQuar *lidarrquarantine.Service, tracksSvc *tracks.Service, playlistsSvc *playlists.Service, coverEnricher *coverart.Enricher, coverBackfillCap int, coverSettings *coverart.SettingsService, scanner *library.Scanner, scanCfg library.RunScanConfig, scheduler *library.Scheduler, dataDir string, sender mailer.Sender, bus *eventbus.Bus, playlistScheduler *playlists.Scheduler) {
func Mount(r chi.Router, pool *pgxpool.Pool, logger *slog.Logger, events *playevents.Writer, recCfg config.RecommendationConfig, lidarrCfg *lidarrconfig.Service, lidarrReqs *lidarrrequests.Service, lidarrQuar *lidarrquarantine.Service, tracksSvc *tracks.Service, playlistsSvc *playlists.Service, coverEnricher *coverart.Enricher, coverSettings *coverart.SettingsService, scanner *library.Scanner, scanCfg library.RunScanConfig, scheduler *library.Scheduler, dataDir string, sender mailer.Sender, bus *eventbus.Bus, playlistScheduler *playlists.Scheduler) {
rng := rand.New(rand.NewSource(rand.Int63()))
h := &handlers{
pool: pool, logger: logger, events: events, recCfg: recCfg,
rng: rng.Float64,
lidarrCfg: lidarrCfg,
lidarrRequests: lidarrReqs,
lidarrQuarantine: lidarrQuar,
tracks: tracksSvc,
playlists: playlistsSvc,
coverart: coverEnricher,
coverArtBackfillCap: coverBackfillCap,
coverSettings: coverSettings,
scanner: scanner,
scanCfg: scanCfg,
scheduler: scheduler,
dataDir: dataDir,
mailer: sender,
eventbus: bus,
playlistScheduler: playlistScheduler,
rng: rng.Float64,
lidarrCfg: lidarrCfg,
lidarrRequests: lidarrReqs,
lidarrQuarantine: lidarrQuar,
tracks: tracksSvc,
playlists: playlistsSvc,
coverart: coverEnricher,
coverSettings: coverSettings,
scanner: scanner,
scanCfg: scanCfg,
scheduler: scheduler,
dataDir: dataDir,
mailer: sender,
eventbus: bus,
playlistScheduler: playlistScheduler,
}
r.Route("/api", func(api chi.Router) {
@@ -180,24 +179,23 @@ func Mount(r chi.Router, pool *pgxpool.Pool, logger *slog.Logger, events *playev
}
type handlers struct {
pool *pgxpool.Pool
logger *slog.Logger
events *playevents.Writer
recCfg config.RecommendationConfig
rng func() float64
lidarrCfg *lidarrconfig.Service
lidarrRequests *lidarrrequests.Service
lidarrQuarantine *lidarrquarantine.Service
tracks *tracks.Service
playlists *playlists.Service
coverart *coverart.Enricher
coverArtBackfillCap int
coverSettings *coverart.SettingsService
scanner *library.Scanner
scanCfg library.RunScanConfig
scheduler *library.Scheduler
dataDir string
mailer mailer.Sender
eventbus *eventbus.Bus
playlistScheduler *playlists.Scheduler
pool *pgxpool.Pool
logger *slog.Logger
events *playevents.Writer
recCfg config.RecommendationConfig
rng func() float64
lidarrCfg *lidarrconfig.Service
lidarrRequests *lidarrrequests.Service
lidarrQuarantine *lidarrquarantine.Service
tracks *tracks.Service
playlists *playlists.Service
coverart *coverart.Enricher
coverSettings *coverart.SettingsService
scanner *library.Scanner
scanCfg library.RunScanConfig
scheduler *library.Scheduler
dataDir string
mailer mailer.Sender
eventbus *eventbus.Bus
playlistScheduler *playlists.Scheduler
}
internal/api/library_test.go
+1 -1
View File
@@ -444,7 +444,7 @@ func TestRoutesRegisteredInMount(t *testing.T) {
r := chi.NewRouter()
w := playevents.NewWriter(h.pool, slog.New(slog.NewTextHandler(io.Discard, nil)),
30*time.Minute, 0.5, 30000)
Mount(r, h.pool, h.logger, w, config.RecommendationConfig{RadioSize: 50, RadioSizeMax: 200, RecentlyPlayedHours: 1}, h.lidarrCfg, h.lidarrRequests, h.lidarrQuarantine, h.tracks, h.playlists, h.coverart, h.coverArtBackfillCap, h.coverSettings, h.scanner, h.scanCfg, nil, h.dataDir, nil, eventbus.New(), nil)
Mount(r, h.pool, h.logger, w, config.RecommendationConfig{RadioSize: 50, RadioSizeMax: 200, RecentlyPlayedHours: 1}, h.lidarrCfg, h.lidarrRequests, h.lidarrQuarantine, h.tracks, h.playlists, h.coverart, h.coverSettings, h.scanner, h.scanCfg, nil, h.dataDir, nil, eventbus.New(), nil)
paths := []string{
"/api/artists",
internal/config/config.go
+5 -12
View File
@@ -56,11 +56,10 @@ type LogConfig struct {
}
type LibraryConfig struct {
ScanPaths []string `yaml:"scan_paths"`
ScanOnStartup bool `yaml:"scan_on_startup"`
CoverArtFromMBCAA bool `yaml:"coverart_from_mbcaa"` // default true
CoverArtBackfillCap int `yaml:"coverart_backfill_cap"` // default 500
ContactEmail string `yaml:"contact_email"` // optional override for MBCAA UA
ScanPaths []string `yaml:"scan_paths"`
ScanOnStartup bool `yaml:"scan_on_startup"`
CoverArtFromMBCAA bool `yaml:"coverart_from_mbcaa"` // default true
ContactEmail string `yaml:"contact_email"` // optional override for MBCAA UA
}
// SubsonicConfig controls wire-format concessions on the /rest/* surface.
@@ -120,8 +119,7 @@ func Default() Config {
RadioSizeMax: 200,
},
Library: LibraryConfig{
CoverArtFromMBCAA: true,
CoverArtBackfillCap: 500,
CoverArtFromMBCAA: true,
},
}
}
@@ -177,11 +175,6 @@ func applyEnv(cfg *Config) {
cfg.Library.CoverArtFromMBCAA = b
}
}
if v, ok := os.LookupEnv("MINSTREL_LIBRARY_COVERART_BACKFILL_CAP"); ok {
if n, err := strconv.Atoi(v); err == nil {
cfg.Library.CoverArtBackfillCap = n
}
}
if v, ok := os.LookupEnv("MINSTREL_CONTACT_EMAIL"); ok {
cfg.Library.ContactEmail = v
}
internal/coverart/artist_enricher.go
+8 -2
View File
@@ -144,15 +144,21 @@ func (e *Enricher) recordArtistArt(ctx context.Context, artistID pgtype.UUID, th
// errored). The JSON tally written to scan_runs keeps the existing
// processed/succeeded/failed shape for backwards compat with the
// admin overview UI; the log line is the operator's diagnostic.
// limit: 0 = stage disabled; >0 = bounded; <0 = unbounded (#388 —
// no global cap; remote providers self-throttle per provider).
func (e *Enricher) EnrichArtistBatch(ctx context.Context, limit int, dataDir string,
progressCb func(processed, succeeded, failed int)) (processed, succeeded, failed int, err error) {
if limit <= 0 {
if limit == 0 {
return 0, 0, 0, nil
}
queryLimit := int32(limit)
if limit < 0 {
queryLimit = 1<<31 - 1 // unbounded
}
q := dbq.New(e.pool)
rows, qerr := q.ListArtistsMissingArt(ctx, dbq.ListArtistsMissingArtParams{
ArtistArtSourcesVersion: e.settings.CurrentVersion(),
Limit: int32(limit),
Limit: queryLimit,
})
if qerr != nil {
return 0, 0, 0, fmt.Errorf("list missing artist art: %w", qerr)
internal/coverart/enricher.go
+18 -4
View File
@@ -213,15 +213,23 @@ func (e *Enricher) recordAlbumCover(ctx context.Context, albumID pgtype.UUID, pa
// JSON tally written to scan_runs keeps the existing
// processed/succeeded/failed shape for backwards compat with the
// admin overview UI; the log line is the operator's diagnostic.
// limit semantics: 0 = stage disabled (skip); >0 = bounded batch;
// <0 = unbounded — walk every album needing art. The global cap was
// removed (#388); external providers self-throttle via their
// per-provider httpClient MinInterval, local sources run at disk speed.
func (e *Enricher) EnrichBatch(ctx context.Context, limit int,
progressCb func(processed, succeeded, failed int)) (processed, succeeded, failed int, err error) {
if limit <= 0 {
if limit == 0 {
return 0, 0, 0, nil
}
queryLimit := int32(limit)
if limit < 0 {
queryLimit = 1<<31 - 1 // unbounded
}
q := dbq.New(e.pool)
rows, qerr := q.ListAlbumsMissingCover(ctx, dbq.ListAlbumsMissingCoverParams{
CoverArtSourcesVersion: e.settings.CurrentVersion(),
Limit: int32(limit),
Limit: queryLimit,
})
if qerr != nil {
return 0, 0, 0, fmt.Errorf("list missing: %w", qerr)
@@ -293,12 +301,18 @@ func (e *Enricher) logBatchSummary(stage string, eligible, processed, succeeded,
// EnrichRetryMissing drains both NULL and 'none' rows. Used by the
// admin bulk-retry endpoint. Clears 'none' to NULL first so
// EnrichAlbum picks them up.
// limit: 0 = no-op; >0 = bounded; <0 = unbounded (retry every
// missing/none album). Admin bulk-retry passes <0 post-#388.
func (e *Enricher) EnrichRetryMissing(ctx context.Context, limit int) (int, error) {
if limit <= 0 {
if limit == 0 {
return 0, nil
}
queryLimit := int32(limit)
if limit < 0 {
queryLimit = 1<<31 - 1 // unbounded
}
q := dbq.New(e.pool)
rows, err := q.ListAlbumsRetryMissing(ctx, int32(limit))
rows, err := q.ListAlbumsRetryMissing(ctx, queryLimit)
if err != nil {
return 0, fmt.Errorf("list retry missing: %w", err)
}
internal/library/scanrun.go
+5 -3
View File
@@ -49,9 +49,11 @@ type ArtistArtEnrichStageTallies struct {
Failed int `json:"failed"`
}
// RunScanConfig assembles the knobs for the orchestrator. BackfillCap and
// EnrichCap mirror the existing boot-time caps (5000 / coverArtBackfillCap)
// so manual triggers don't accidentally drain the whole library.
// RunScanConfig assembles the knobs for the orchestrator. BackfillCap is
// the album/artist-MBID-backfill batch cap (5000); EnrichCap /
// ArtistEnrichCap gate cover/artist-art enrichment (0 = off, <0 =
// unbounded — #388 removed the global cover-art cap; remote providers
// self-throttle per provider).
type RunScanConfig struct {
BackfillCap int
EnrichCap int
internal/server/server.go
+10 -11
View File
@@ -72,14 +72,13 @@ type Server struct {
// playlist cover collages under <DataDir>/playlist_covers/). Empty
// strings are tolerated by tests that don't exercise persisted-cover
// codepaths; production callers should pass a writable directory.
DataDir string
BrandingCfg config.BrandingConfig
CoverEnricher *coverart.Enricher
CoverArtBackfillCap int
CoverSettings *coverart.SettingsService
LibraryScanner *library.Scanner
ScanCfg library.RunScanConfig
Scheduler *library.Scheduler
DataDir string
BrandingCfg config.BrandingConfig
CoverEnricher *coverart.Enricher
CoverSettings *coverart.SettingsService
LibraryScanner *library.Scanner
ScanCfg library.RunScanConfig
Scheduler *library.Scheduler
// Bus is the live-event bus shared with background workers (the
// lidarr reconciler, scan scheduler) constructed in cmd/minstrel/main.go.
// When nil, Router() constructs a local fallback (test contexts).
@@ -92,8 +91,8 @@ type Server struct {
PlaylistScheduler *playlists.Scheduler
}
func New(logger *slog.Logger, pool *pgxpool.Pool, scanner ScanTrigger, subCfg subsonic.Config, eventsCfg config.EventsConfig, recCfg config.RecommendationConfig, dataDir string, brandingCfg config.BrandingConfig, coverEnricher *coverart.Enricher, coverArtBackfillCap int, coverSettings *coverart.SettingsService, libraryScanner *library.Scanner, scanCfg library.RunScanConfig, scheduler *library.Scheduler) *Server {
return &Server{Logger: logger, Pool: pool, Scanner: scanner, SubsonicCfg: subCfg, EventsCfg: eventsCfg, RecommendationCfg: recCfg, DataDir: dataDir, BrandingCfg: brandingCfg, CoverEnricher: coverEnricher, CoverArtBackfillCap: coverArtBackfillCap, CoverSettings: coverSettings, LibraryScanner: libraryScanner, ScanCfg: scanCfg, Scheduler: scheduler}
func New(logger *slog.Logger, pool *pgxpool.Pool, scanner ScanTrigger, subCfg subsonic.Config, eventsCfg config.EventsConfig, recCfg config.RecommendationConfig, dataDir string, brandingCfg config.BrandingConfig, coverEnricher *coverart.Enricher, coverSettings *coverart.SettingsService, libraryScanner *library.Scanner, scanCfg library.RunScanConfig, scheduler *library.Scheduler) *Server {
return &Server{Logger: logger, Pool: pool, Scanner: scanner, SubsonicCfg: subCfg, EventsCfg: eventsCfg, RecommendationCfg: recCfg, DataDir: dataDir, BrandingCfg: brandingCfg, CoverEnricher: coverEnricher, CoverSettings: coverSettings, LibraryScanner: libraryScanner, ScanCfg: scanCfg, Scheduler: scheduler}
}
func (s *Server) Router() http.Handler {
@@ -139,7 +138,7 @@ func (s *Server) Router() http.Handler {
if bus == nil {
bus = eventbus.New()
}
api.Mount(r, s.Pool, s.Logger, writer, s.RecommendationCfg, lidarrCfg, lidarrReqs, lidarrQuar, tracksSvc, playlistsSvc, s.CoverEnricher, s.CoverArtBackfillCap, s.CoverSettings, s.LibraryScanner, s.ScanCfg, s.Scheduler, s.DataDir, smtpSender, bus, s.PlaylistScheduler)
api.Mount(r, s.Pool, s.Logger, writer, s.RecommendationCfg, lidarrCfg, lidarrReqs, lidarrQuar, tracksSvc, playlistsSvc, s.CoverEnricher, s.CoverSettings, s.LibraryScanner, s.ScanCfg, s.Scheduler, s.DataDir, smtpSender, bus, s.PlaylistScheduler)
// /api/admin/scan is the only admin route owned by the server package
// (it needs the Scanner). Register it as a single inline-middleware
// route — using r.Route("/api/admin", ...) here would create a second
web/src/lib/api/admin.covers.test.ts
+2 -2
View File
@@ -25,9 +25,9 @@ describe('admin covers API', () => {
});
it('refetchMissingCovers POSTs to the bulk endpoint', async () => {
(api.post as unknown as ReturnType<typeof vi.fn>).mockResolvedValueOnce({ queued: 7 });
(api.post as unknown as ReturnType<typeof vi.fn>).mockResolvedValueOnce({ started: true });
const got = await refetchMissingCovers();
expect(api.post).toHaveBeenCalledWith('/api/admin/covers/refetch-missing', {});
expect(got.queued).toBe(7);
expect(got.started).toBe(true);
});
});
web/src/lib/api/admin.ts
+1 -1
View File
@@ -187,7 +187,7 @@ export async function refetchAlbumCover(albumId: string): Promise<RefetchAlbumCo
}
export type RefetchMissingResponse = {
queued: number;
started: boolean;
};
export async function refetchMissingCovers(): Promise<RefetchMissingResponse> {
web/src/routes/admin/+page.svelte
+4 -2
View File
@@ -318,8 +318,10 @@
bulkBusy = true;
bulkResult = null;
try {
const { queued } = await refetchMissingCovers();
bulkResult = `Queued ${queued} albums for cover refetch.`;
const { started } = await refetchMissingCovers();
bulkResult = started
? 'Refetching all missing covers — local sources are fast, remote providers throttle per their limits.'
: 'Could not start the cover refetch.';
await client.invalidateQueries({ queryKey: qk.coverage() });
} catch (e) {
bulkResult = `Failed: ${errCode(e)}`;
web/src/routes/admin/admin.test.ts
+1 -1
View File
@@ -41,7 +41,7 @@ vi.mock('$lib/api/admin', async () => {
deleteQuarantineFile: vi.fn().mockResolvedValue({}),
deleteQuarantineViaLidarr: vi.fn().mockResolvedValue({}),
triggerScan: vi.fn().mockResolvedValue({}),
refetchMissingCovers: vi.fn().mockResolvedValue({ queued: 0 }),
refetchMissingCovers: vi.fn().mockResolvedValue({ started: true }),
researchMissingArt: vi.fn().mockResolvedValue({ version: 1 }),
createScanScheduleQuery: vi.fn(() =>
readable({