feat: a held album below the quality you settle for, or holding a video rip, stays wanted for a better copy, and the import replaces the old tracks with their likes and plays carried over (M502 #5524)

- acquisition_settings.upgrade_until (default High quality lossy, Lidarr's
  cutoff, note #5527; "" is off), an "Upgrade until" row under Searching.
- The wanted worker judges held albums hourly: tier per format from size per
  second (the scanner records no bitrate), the album's worst. Below the
  cutoff it is wanted for quality; a video rip among its tracks always.
- The schedule searches upgrades weekly, after open wants; a release no
  better than the copy held is refused ("not_better").
- The import matches each file to the track it would replace (recording,
  then disc/track) and replaces it when it is a rip or the file is a better
  tier: over the same path in one rename, or filed beside it and the old
  track folded in (library.ReplaceTrack: file first, then foldTrackInto).
  Nothing better: the release is set aside.
- Collection › Upgrades (Lidarr's Cutoff Unmet); Suspect sources says when a
  clean copy was looked for instead of Lidarr's search.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

internal/wanted/search.go

@@ -48,7 +48,8 @@ const (
 	passTimeout = 15 * time.Minute
 )
 
-// ErrWantHeld: the want is already answered by the library.
+// ErrWantHeld: the want is already answered by the library, and is not
+// wanted for a better copy.
 var ErrWantHeld = errors.New("wanted: the library already holds this album")
 
 // Searcher searches for wants and grabs what they need.
@@ -167,8 +168,9 @@ func (s *Searcher) pass(ctx context.Context) Report {
 			rep.Paced = ctx.Err() == nil
 			return rep
 		}
-		want := s.wantFor(ctx, row.ID, row.ArtistName, row.Title, row.PrimaryType, row.SecondaryTypes,
-			row.FirstReleaseDate, row.ReleaseGroupMbid)
+		want := s.wantFor(ctx, searchRow{id: row.ID, artist: row.ArtistName, title: row.Title,
+			primary: row.PrimaryType, secondary: row.SecondaryTypes, date: row.FirstReleaseDate,
+			group: row.ReleaseGroupMbid, upgrade: row.Upgrade, heldTier: row.HeldTier, held: row.HeldQuality})
 		out := s.one(ctx, svc, want, settings.AutoGrab)
 		rep.Looked++
 		switch out.Outcome {
@@ -186,17 +188,31 @@ func (s *Searcher) pass(ctx context.Context) Report {
 	return rep
 }
 
+// searchRow is what a search needs of a wants row.
+type searchRow struct {
+	id                                  pgtype.UUID
+	artist, title, primary, date, group string
+	secondary                           []string
+	// upgrade, heldTier and held: a held album wanted for a better copy
+	// (#5524), and the copy it has.
+	upgrade, heldTier, held string
+}
+
 // wantFor describes a want for the matcher, with its editions' track counts
 // from the catalogue (read from MusicBrainz the first time when it can be).
-func (s *Searcher) wantFor(ctx context.Context, id pgtype.UUID, artist, title, primary string, secondary []string, date, group string) acquire.Want {
-	w := acquire.Want{ID: id, Want: release.Want{
-		Artist: artist, Title: title, Year: yearOf(date), PrimaryType: primary, SecondaryTypes: secondary,
+// A held album wanted for better quality takes its copy's tier as a floor.
+func (s *Searcher) wantFor(ctx context.Context, r searchRow) acquire.Want {
+	w := acquire.Want{ID: r.id, Want: release.Want{
+		Artist: r.artist, Title: r.title, Year: yearOf(r.date), PrimaryType: r.primary, SecondaryTypes: r.secondary,
 	}}
+	if r.upgrade == UpgradeQuality && release.KnownTier(r.heldTier) {
+		w.Floor, w.Held = release.Tier(r.heldTier), r.held
+	}
 	q := dbq.New(s.pool)
-	span, err := q.EditionTrackSpan(ctx, group)
+	span, err := q.EditionTrackSpan(ctx, r.group)
 	if err == nil && span.MinTracks == 0 && s.cat != nil {
-		if _, rerr := s.cat.Releases(ctx, group); rerr == nil {
-			span, err = q.EditionTrackSpan(ctx, group)
+		if _, rerr := s.cat.Releases(ctx, r.group); rerr == nil {
+			span, err = q.EditionTrackSpan(ctx, r.group)
 		}
 	}
 	if err == nil {
@@ -329,15 +345,17 @@ func (s *Searcher) Take(ctx context.Context, id pgtype.UUID, key string, force b
 	return took, nil
 }
 
-// load reads an open want for a search.
+// load reads a want for a search: an open one, or a held album wanted for a
+// better copy (#5524).
 func (s *Searcher) load(ctx context.Context, id pgtype.UUID) (acquire.Want, error) {
 	row, err := dbq.New(s.pool).GetWantForSearch(ctx, id)
 	if err != nil {
 		return acquire.Want{}, err
 	}
-	if row.FulfilledAt.Valid {
+	if row.FulfilledAt.Valid && row.Upgrade == "" {
 		return acquire.Want{}, ErrWantHeld
 	}
-	return s.wantFor(ctx, row.ID, row.ArtistName, row.Title, row.PrimaryType, row.SecondaryTypes,
-		row.FirstReleaseDate, row.ReleaseGroupMbid), nil
+	return s.wantFor(ctx, searchRow{id: row.ID, artist: row.ArtistName, title: row.Title,
+		primary: row.PrimaryType, secondary: row.SecondaryTypes, date: row.FirstReleaseDate,
+		group: row.ReleaseGroupMbid, upgrade: row.Upgrade, heldTier: row.HeldTier, held: row.HeldQuality}), nil
 }

internal/wanted/upgrade.go

@@ -0,0 +1,125 @@
+package wanted
+
+import (
+	"context"
+	"fmt"
+	"time"
+
+	"github.com/jackc/pgx/v5/pgtype"
+
+	"git.fabledsword.com/bvandeusen/minstrel/internal/db/dbq"
+	"git.fabledsword.com/bvandeusen/minstrel/internal/release"
+)
+
+// Upgrades (M502 #5524): a held album stays wanted when the copy the library
+// has is below the quality the operator settles for (acquisition_settings.
+// upgrade_until), or when one of its tracks is a video rip. The search
+// schedule then looks for a better release weekly, and the import replaces
+// the old tracks with the new ones, carrying their history across.
+
+// Why a held album is wanted again, matching wants_upgrade_check.
+const (
+	UpgradeQuality = "quality"
+	UpgradeRip     = "rip"
+)
+
+// upgradeCheckEvery is how often held albums are judged. Their files change
+// only by an import or a scan, and the check reads every held album.
+const upgradeCheckEvery = time.Hour
+
+// UpgradeFor says why a held album is wanted again, if it is: a video rip
+// among its tracks always (a rip is no copy of the album, whatever the
+// setting), else a copy below until. until "" switches quality upgrades off.
+func UpgradeFor(held release.Held, judged bool, rips int, until release.Tier) string {
+	switch {
+	case rips > 0:
+		return UpgradeRip
+	case until == "" || !judged:
+		return ""
+	case release.Rank(held.Tier) < release.Rank(until):
+		return UpgradeQuality
+	}
+	return ""
+}
+
+// heldVerdict is one held want as the check judged it.
+type heldVerdict struct {
+	id                       pgtype.UUID
+	upgrade, tier, words     string
+	oldUpgrade, oldTier, old string
+}
+
+func (v heldVerdict) changed() bool {
+	return v.upgrade != v.oldUpgrade || v.tier != v.oldTier || v.words != v.old
+}
+
+// checkUpgrades judges every held album, at most once an hour, and records
+// which are wanted for a better copy.
+func (s *Service) checkUpgrades(ctx context.Context, res *PassResult) error {
+	if !s.upgradesChecked.IsZero() && s.now().Sub(s.upgradesChecked) < upgradeCheckEvery {
+		return nil
+	}
+	q := dbq.New(s.pool)
+	settings, err := q.GetAcquisitionSettings(ctx)
+	if err != nil {
+		return fmt.Errorf("read settings: %w", err)
+	}
+	rows, err := q.ListHeldWantFormats(ctx)
+	if err != nil {
+		return fmt.Errorf("read held albums: %w", err)
+	}
+	until := release.Tier(settings.UpgradeUntil)
+	var out []heldVerdict
+	flush := func(id pgtype.UUID, group []dbq.ListHeldWantFormatsRow) {
+		if len(group) == 0 {
+			return
+		}
+		formats := make([]release.HeldFormat, 0, len(group))
+		rips := 0
+		for _, r := range group {
+			formats = append(formats, release.HeldFormat{Format: r.FileFormat, Bytes: r.Bytes, Ms: r.DurationMs})
+			rips += int(r.Rips)
+		}
+		held, judged := release.HeldCopy(formats)
+		v := heldVerdict{id: id, upgrade: UpgradeFor(held, judged, rips, until),
+			oldUpgrade: group[0].Upgrade, oldTier: group[0].HeldTier, old: group[0].HeldQuality}
+		if judged {
+			v.tier, v.words = string(held.Tier), held.Words
+		}
+		if v.upgrade != "" {
+			res.Upgrades++
+		}
+		if v.changed() {
+			out = append(out, v)
+		}
+	}
+	var group []dbq.ListHeldWantFormatsRow
+	for _, r := range rows {
+		if len(group) > 0 && group[0].WantID != r.WantID {
+			flush(group[0].WantID, group)
+			group = group[:0]
+		}
+		group = append(group, r)
+	}
+	if len(group) > 0 {
+		flush(group[0].WantID, group)
+	}
+
+	if len(out) > 0 {
+		p := dbq.SetWantUpgradesParams{}
+		for _, v := range out {
+			p.Ids = append(p.Ids, v.id)
+			p.Upgrades = append(p.Upgrades, v.upgrade)
+			p.HeldTiers = append(p.HeldTiers, v.tier)
+			p.HeldQualities = append(p.HeldQualities, v.words)
+		}
+		if err := q.SetWantUpgrades(ctx, p); err != nil {
+			return fmt.Errorf("record upgrades: %w", err)
+		}
+	}
+	if _, err := q.ClearHeldUpgrades(ctx); err != nil {
+		return fmt.Errorf("clear upgrades: %w", err)
+	}
+	s.upgradesChecked = s.now()
+	return nil
+}

internal/wanted/upgrade_test.go

@@ -0,0 +1,137 @@
+package wanted
+
+import (
+	"context"
+	"fmt"
+	"testing"
+	"time"
+
+	"github.com/jackc/pgx/v5/pgxpool"
+
+	"git.fabledsword.com/bvandeusen/minstrel/internal/db/dbq"
+	"git.fabledsword.com/bvandeusen/minstrel/internal/release"
+)
+
+func TestUpgradeFor(t *testing.T) {
+	low := release.Held{Tier: release.TierLossyLow}
+	flac := release.Held{Tier: release.TierLossless}
+	high := release.Held{Tier: release.TierLossyHigh}
+	cases := []struct {
+		name   string
+		held   release.Held
+		judged bool
+		rips   int
+		until  release.Tier
+		want   string
+	}{
+		{"below the cutoff", low, true, 0, release.TierLossyHigh, UpgradeQuality},
+		{"at the cutoff", high, true, 0, release.TierLossyHigh, ""},
+		{"above the cutoff", flac, true, 0, release.TierLossyHigh, ""},
+		{"upgrades off", low, true, 0, "", ""},
+		{"a rip, upgrades off", flac, true, 1, "", UpgradeRip},
+		{"nothing to judge", release.Held{}, false, 0, release.TierLossless, ""},
+	}
+	for _, c := range cases {
+		if got := UpgradeFor(c.held, c.judged, c.rips, c.until); got != c.want {
+			t.Errorf("%s: %q, want %q", c.name, got, c.want)
+		}
+	}
+}
+
+// seedMP3s puts Blur's The Ballad of Darren in the library as MP3s at about
+// kbps, two tracks, the first held back as a video rip when rip is set.
+func seedMP3s(t *testing.T, pool *pgxpool.Pool, kbps int64, rip bool) dbq.Album {
+	t.Helper()
+	ctx := context.Background()
+	q := dbq.New(pool)
+	mbid, group, rel := blur, theBallad, "r-ballad-mp3"
+	artist, err := q.UpsertArtist(ctx, dbq.UpsertArtistParams{Name: "Blur", SortName: "Blur", Mbid: &mbid})
+	if err != nil {
+		t.Fatal(err)
+	}
+	album, err := q.UpsertAlbum(ctx, dbq.UpsertAlbumParams{Title: "The Ballad of Darren", SortTitle: "Ballad of Darren",
+		ArtistID: artist.ID, Mbid: &rel, ReleaseGroupMbid: &group})
+	if err != nil {
+		t.Fatal(err)
+	}
+	for i := range 2 {
+		num := int32(i + 1)
+		tr, err := q.UpsertTrack(ctx, dbq.UpsertTrackParams{
+			Title: fmt.Sprintf("Track %d", i+1), AlbumID: album.ID, ArtistID: artist.ID, TrackNumber: &num,
+			DurationMs: 200000, FilePath: fmt.Sprintf("/music/ballad/%02d.mp3", i+1),
+			FileSize: kbps * 200000 / 8, FileFormat: "mp3",
+		})
+		if err != nil {
+			t.Fatal(err)
+		}
+		if rip && i == 0 {
+			if _, err := pool.Exec(ctx, `UPDATE tracks SET source_verdict = 'suspect' WHERE id = $1`, tr.ID); err != nil {
+				t.Fatal(err)
+			}
+		}
+	}
+	return album
+}
+
+func TestCheckUpgrades_AHeldAlbumBelowTheCutoffIsWantedAgain_Integration(t *testing.T) {
+	pool := newPool(t)
+	q := dbq.New(pool)
+	ctx := context.Background()
+	s := newService(t, pool, newFakeMB(t))
+
+	if _, err := WantWith(ctx, q, WantParams{
+		GroupMBID: theBallad, ArtistMBID: blur, ArtistName: "Blur", Title: "The Ballad of Darren", Reason: ReasonRequest,
+	}); err != nil {
+		t.Fatal(err)
+	}
+	seedMP3s(t, pool, 192, false)
+	res, err := s.Pass(ctx)
+	if err != nil || res.Held != 1 || res.Upgrades != 1 {
+		t.Fatalf("pass: %+v, %v", res, err)
+	}
+	w, _ := q.GetWantByReleaseGroup(ctx, theBallad)
+	if w.Upgrade != UpgradeQuality || w.HeldTier != string(release.TierLossyLow) || w.HeldQuality != "MP3, about 192 kbps" {
+		t.Fatalf("want = upgrade %q, tier %q, %q", w.Upgrade, w.HeldTier, w.HeldQuality)
+	}
+	due, err := q.ListWantsDueForSearch(ctx, dbq.ListWantsDueForSearchParams{Hours: 24, MaxRows: 10})
+	if err != nil || len(due) != 1 || due[0].Upgrade != UpgradeQuality {
+		t.Fatalf("due for search: %+v, %v", due, err)
+	}
+
+	// Upgrades switched off: no longer wanted again.
+	if _, err := pool.Exec(ctx, `UPDATE acquisition_settings SET upgrade_until = ''`); err != nil {
+		t.Fatal(err)
+	}
+	s.upgradesChecked = time.Time{}
+	if _, err := s.Pass(ctx); err != nil {
+		t.Fatal(err)
+	}
+	w, _ = q.GetWantByReleaseGroup(ctx, theBallad)
+	if w.Upgrade != "" {
+		t.Errorf("upgrades off: upgrade %q", w.Upgrade)
+	}
+}
+
+func TestCheckUpgrades_ARipIsReplacedWhateverTheSetting_Integration(t *testing.T) {
+	pool := newPool(t)
+	q := dbq.New(pool)
+	ctx := context.Background()
+	s := newService(t, pool, newFakeMB(t))
+
+	if _, err := WantWith(ctx, q, WantParams{
+		GroupMBID: theBallad, ArtistMBID: blur, ArtistName: "Blur", Title: "The Ballad of Darren", Reason: ReasonRequest,
+	}); err != nil {
+		t.Fatal(err)
+	}
+	seedMP3s(t, pool, 320, true)
+	if _, err := pool.Exec(ctx, `UPDATE acquisition_settings SET upgrade_until = ''`); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := s.Pass(ctx); err != nil {
+		t.Fatal(err)
+	}
+	w, _ := q.GetWantByReleaseGroup(ctx, theBallad)
+	if w.Upgrade != UpgradeRip {
+		t.Errorf("a rip: upgrade %q", w.Upgrade)
+	}
+}

internal/wanted/wanted.go

@@ -50,6 +50,9 @@ type Service struct {
 	// want recorded under a release id (#5241) to its group. A field so tests
 	// stand in for MusicBrainz.
 	releaseGroup func(ctx context.Context, releaseMBID string) (string, error)
+	// upgradesChecked is when held albums were last judged for an upgrade
+	// (#5524); the worker alone touches it.
+	upgradesChecked time.Time
 }
 
 // New makes a Service reading discographies and editions through cat.
@@ -175,9 +178,9 @@ func (s *Service) Run(ctx context.Context) {
 		switch {
 		case err != nil && ctx.Err() == nil:
 			s.logger.Warn("wanted: pass stopped", "err", err, "follows", res.Follows, "wanted", res.Wanted, "held", res.Held)
-		case res.Wanted > 0 || res.Held > 0 || res.Baselined > 0 || res.Listed > 0:
+		case res.Wanted > 0 || res.Held > 0 || res.Baselined > 0 || res.Listed > 0 || res.Upgrades > 0:
 			s.logger.Info("wanted: pass complete", "follows", res.Follows, "baselined", res.Baselined,
-				"wanted", res.Wanted, "held", res.Held, "listed", res.Listed)
+				"wanted", res.Wanted, "held", res.Held, "listed", res.Listed, "upgrades", res.Upgrades)
 		}
 		select {
 		case <-ctx.Done():
@@ -194,6 +197,9 @@ type PassResult struct {
 	Wanted    int // wants recorded from follows
 	Held      int // wants the library now answers
 	Listed    int // library albums put on the list as held (#5567)
+	// Upgrades: held albums wanted for a better copy, when the hourly check
+	// ran this pass (#5524).
+	Upgrades int
 }
 
 // Pass applies the follows due and settles open wants. An unreachable
@@ -213,6 +219,9 @@ func (s *Service) Pass(ctx context.Context) (PassResult, error) {
 	if err := s.settleWants(ctx, &res); err != nil {
 		return res, err
 	}
+	if err := s.checkUpgrades(ctx, &res); err != nil && ctx.Err() == nil {
+		s.logger.Warn("wanted: judge held albums for upgrades", "err", err)
+	}
 	return res, ferr
 }