Files
minstrel/internal/library/duplicate_resolve_test.go
T
bvandeusenandClaude Opus 5.5 4ecff52f19
release / govulncheck (push) Successful in 45s
release / web (push) Successful in 1m23s
release / go (push) Successful in 1m39s
release / integration (push) Successful in 4m25s
release / android (push) Successful in 6m17s
release / Build signed APK (releases and dev) (push) Successful in 5m57s
release / Attach APK to the Release (tag releases only) (push) Skipped
release / Build + push container image (push) Successful in 1m54s
release / Verify release artifacts (tag releases only) (push) Skipped
feat: duplicates resolve themselves where Lidarr says it is safe (M498)
The duplicate sweep proposed 4,197 groups and every one waited for the
operator. Most are safe to settle, and Lidarr defines what safe means: it
maps one file to each track of the release it monitors and downloads any
mapped file that disappears. Deleting a mapped copy opens exactly the hole
the operator saw Lidarr fill.

Classify (#5435)
- Migration 0075: duplicate_groups.class (same_release, cross_release,
  mismatch, review), resolve_note, resolved_automatically;
  duplicate_group_members.lidarr_state (tracked, unmapped);
  fingerprint_settings.auto_resolve; notification kind
  duplicates_resolved with both kind CHECKs swapped (rule 36).
- library.ClassifyDuplicateGroup, with MatchTitleKey dropping featuring
  credits, remaster notes and video-rip markers, and keeping live, demo,
  remix and instrumental. The rip markers move from api to library.

Choose the copy to keep (#5436)
- ProposeSurvivor ranks the copy Lidarr maps first, then tag fit (a
  clash-free track number, no rip marker in the name, an MBID), then the
  quality rules. File size picked the wrong Humanz copy in 6 of 21 groups.

Act (#5437)
- An hourly resolver pass reads Lidarr's unmapped files, matched by the
  last three path components, and records each copy's state.
- Same album, with at most one copy mapped: merged into the mapped copy.
  The merge is guarded, so a mapped copy can never be removed
  (MergeDuplicateGroupGuarded, ErrCopyTrackedByLidarr).
- Same album, every copy mapped: the monitored release lists the song
  twice (Humanz's 14x12" box set). The pass moves Lidarr to the release
  that lists each song once and best covers what is on disk. It never
  picks one covering less, and is capped at 10 albums per pass.
  - Fixed point (lesson #4183): the chosen release no longer repeats.
  - The album is left alone for 24h while Lidarr rescans, so "every copy
    unmapped" mid-rescan is never read as licence to merge.
- Both actions are audited with no actor and summarised to admins. The
  operator can switch them off in the Fingerprinting card (rule 25).
- Manual merges use the same guard: 409 copy_tracked_by_lidarr, or 503
  lidarr_unavailable when Lidarr cannot say.

Web
- Duplicates gets tabs: Needs review, Across releases, Resolved
  automatically. Each loads as you scroll (rule 172), replacing the
  pager.
- Each copy says whether Lidarr uses it.
- The resolver's note shows on each group.
- The merge confirm blocks, before sending, a merge that would remove
  the copy Lidarr uses.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-08 21:49:37 -04:00

350 lines
13 KiB
Go

package library
import (
"context"
"errors"
"os"
"path/filepath"
"testing"
"github.com/jackc/pgx/v5/pgtype"
"github.com/jackc/pgx/v5/pgxpool"
"git.fabledsword.com/bvandeusen/minstrel/internal/db/dbq"
"git.fabledsword.com/bvandeusen/minstrel/internal/lidarr"
syncpkg "git.fabledsword.com/bvandeusen/minstrel/internal/sync"
)
func TestLidarrPathKey(t *testing.T) {
// Minstrel and Lidarr see the library under different mounts; the key is
// what they agree on.
a := LidarrPathKey("/music/Gorillaz/Humanz (2017)/Gorillaz - Humanz - 01 - Ascension.mp3")
b := LidarrPathKey("/data/media/music/Gorillaz/Humanz (2017)/Gorillaz - Humanz - 01 - Ascension.mp3")
if a != b || a != "Gorillaz/Humanz (2017)/Gorillaz - Humanz - 01 - Ascension.mp3" {
t.Errorf("keys differ or are wrong: %q, %q", a, b)
}
if LidarrPathKey("/music/Gorillaz/Humanz (2017)/a.mp3") == LidarrPathKey("/music/Gorillaz/Demon Days (2005)/a.mp3") {
t.Error("two albums' files share a key")
}
}
func rt(titles ...string) []lidarr.ReleaseTrack {
out := make([]lidarr.ReleaseTrack, len(titles))
for i, title := range titles {
out[i] = lidarr.ReleaseTrack{ID: i + 1, Title: title}
}
return out
}
func TestReleaseRepeats(t *testing.T) {
if !releaseRepeats(rt("Ascension", "Strobelite", "Ascension")) {
t.Error("the box set lists Ascension twice, but no repeat was found")
}
if releaseRepeats(rt("Charger", "Charger (demo)")) {
t.Error("a demo was read as a repeat of the song")
}
}
// The Humanz releases as Lidarr lists them (2026-10-09): the box set is
// monitored, and the 40-track digital release is the one that lists each song
// once and covers what is on disk.
func TestPickRelease(t *testing.T) {
digital40 := lidarr.AlbumRelease{ID: 10149, Format: "Digital Media", TrackCount: 40}
deluxe26 := lidarr.AlbumRelease{ID: 10150, Format: "CD", TrackCount: 26}
standard20 := lidarr.AlbumRelease{ID: 10151, Format: "Digital Media", TrackCount: 20}
got, ok := pickRelease([]releaseCandidate{
{release: standard20, coverage: 20}, {release: deluxe26, coverage: 26}, {release: digital40, coverage: 34},
}, 34)
if !ok || got.ID != digital40.ID {
t.Errorf("picked %+v (ok=%v), want the 40-track digital release", got, ok)
}
// Equal coverage: the plainer edition, then digital.
cd := lidarr.AlbumRelease{ID: 1, Format: "CD", TrackCount: 26}
web := lidarr.AlbumRelease{ID: 2, Format: "Digital Media", TrackCount: 26}
if got, _ := pickRelease([]releaseCandidate{{release: cd, coverage: 26}, {release: web, coverage: 26}}, 26); got.ID != web.ID {
t.Errorf("picked %+v, want the digital one of two equal releases", got)
}
// Never a release that keeps fewer of the songs on disk than the current one.
if got, ok := pickRelease([]releaseCandidate{{release: deluxe26, coverage: 26}}, 34); ok {
t.Errorf("picked %+v, which covers fewer songs on disk than the current release", got)
}
}
// fakeLidarr is one Lidarr album with several releases. Switching the
// monitored release changes what ListAlbumTracks answers, as in Lidarr.
type fakeLidarr struct {
unmapped []string
unmappedErr error
album lidarr.LidarrAlbum
releases []lidarr.AlbumRelease
releaseTracks map[int][]lidarr.ReleaseTrack
setCalls [][2]int
}
func (f *fakeLidarr) ListUnmappedTrackFiles(context.Context) ([]lidarr.TrackFile, error) {
out := make([]lidarr.TrackFile, len(f.unmapped))
for i, p := range f.unmapped {
out[i] = lidarr.TrackFile{ID: i + 1, Path: p}
}
return out, f.unmappedErr
}
func (f *fakeLidarr) LookupAlbumByMBID(_ context.Context, mbid string) (lidarr.LidarrAlbum, error) {
if f.album.ForeignAlbumID != mbid {
return lidarr.LidarrAlbum{}, lidarr.ErrNotFound
}
return f.album, nil
}
func (f *fakeLidarr) ListAlbumTracks(context.Context, int) ([]lidarr.ReleaseTrack, error) {
for _, r := range f.releases {
if r.Monitored {
return f.releaseTracks[r.ID], nil
}
}
return nil, nil
}
func (f *fakeLidarr) GetAlbumReleases(context.Context, int) ([]lidarr.AlbumRelease, error) {
return append([]lidarr.AlbumRelease(nil), f.releases...), nil
}
func (f *fakeLidarr) ListReleaseTracks(_ context.Context, releaseID int) ([]lidarr.ReleaseTrack, error) {
return f.releaseTracks[releaseID], nil
}
func (f *fakeLidarr) SetMonitoredRelease(_ context.Context, albumID, releaseID int) error {
f.setCalls = append(f.setCalls, [2]int{albumID, releaseID})
for i := range f.releases {
f.releases[i].Monitored = f.releases[i].ID == releaseID
}
return nil
}
// resolveFixture is the Humanz shape: one album, one song twice, both copies on
// disk under an artist/album folder, in one pending exact-tier group.
type resolveFixture struct {
pool *pgxpool.Pool
clean, rip dbq.Track
cleanPath, ripPath string
groupID pgtype.UUID
}
func newResolveFixture(t *testing.T) resolveFixture {
t.Helper()
pool := newPool(t)
ctx := context.Background()
q := dbq.New(pool)
dir := filepath.Join(t.TempDir(), "Gorillaz", "Humanz (2017)")
if err := os.MkdirAll(dir, 0o755); err != nil {
t.Fatal(err)
}
f := resolveFixture{pool: pool}
f.cleanPath = filepath.Join(dir, "Gorillaz - Humanz - 03 - Saturnz Barz.mp3")
f.ripPath = filepath.Join(dir, "Gorillaz - Humanz - 03 - Saturnz Barz (Official Video).mp3")
for _, p := range []string{f.cleanPath, f.ripPath} {
if err := os.WriteFile(p, []byte("audio"), 0o644); err != nil {
t.Fatal(err)
}
}
var album dbq.Album
var artist dbq.Artist
f.clean, album, artist = seedTrack(t, pool, f.cleanPath)
rip, err := q.UpsertTrack(ctx, dbq.UpsertTrackParams{
Title: "Saturnz Barz (Official Video)", AlbumID: album.ID, ArtistID: artist.ID,
DurationMs: 1000, FilePath: f.ripPath, FileSize: 100, FileFormat: "mp3",
})
if err != nil {
t.Fatal(err)
}
f.rip = rip
exec := func(sql string, args ...any) {
t.Helper()
if _, err := pool.Exec(ctx, sql, args...); err != nil {
t.Fatalf("exec %q: %v", sql, err)
}
}
exec(`UPDATE tracks SET title = 'Saturnz Barz' WHERE id = $1`, f.clean.ID)
exec(`UPDATE albums SET release_group_mbid = 'rg-humanz' WHERE id = $1`, album.ID)
if err := pool.QueryRow(ctx,
`INSERT INTO duplicate_groups (member_key, tier) VALUES ('resolve-fixture', 'exact') RETURNING id`,
).Scan(&f.groupID); err != nil {
t.Fatal(err)
}
exec(`INSERT INTO duplicate_group_members (group_id, track_id) VALUES ($1, $2), ($1, $3)`, f.groupID, f.clean.ID, f.rip.ID)
return f
}
// lidarrPath is where Lidarr, mounted elsewhere, sees a fixture file.
func lidarrPath(p string) string { return "/music/" + LidarrPathKey(p) }
func (f resolveFixture) group(t *testing.T) (status string, class *string, note *string, auto bool) {
t.Helper()
if err := f.pool.QueryRow(context.Background(),
`SELECT status, class, resolve_note, resolved_automatically FROM duplicate_groups WHERE id = $1`, f.groupID,
).Scan(&status, &class, &note, &auto); err != nil {
t.Fatal(err)
}
return status, class, note, auto
}
func exists(p string) bool { _, err := os.Stat(p); return err == nil }
// Lidarr maps the clean copy and holds the rip unmapped: the rip goes, the
// clean copy stays, and the merge is recorded as the resolver's.
func TestResolveDuplicates_MergesTheUnmappedCopy_Integration(t *testing.T) {
f := newResolveFixture(t)
lid := &fakeLidarr{unmapped: []string{lidarrPath(f.ripPath)}}
res, err := ResolveDuplicates(context.Background(), f.pool, nil, "", lid, true)
if err != nil {
t.Fatalf("resolve: %v", err)
}
if res.Merged != 1 || !res.LidarrConsulted {
t.Fatalf("result = %+v, want one merge with Lidarr consulted", res)
}
if exists(f.ripPath) || !exists(f.cleanPath) {
t.Errorf("rip exists=%v clean exists=%v; want only the clean copy left", exists(f.ripPath), exists(f.cleanPath))
}
status, class, _, auto := f.group(t)
if status != "merged" || class == nil || *class != string(ClassSameRelease) || !auto {
t.Errorf("group = (%s, %v, auto=%v), want merged same_release by the resolver", status, class, auto)
}
var audits int
if err := f.pool.QueryRow(context.Background(),
`SELECT count(*) FROM audit_log WHERE action = 'duplicate_merge' AND actor_id IS NULL
AND metadata->>'group_id' = $1 AND (metadata->>'automatic')::boolean`,
syncpkg.FormatUUID(f.groupID)).Scan(&audits); err != nil {
t.Fatal(err)
}
if audits != 1 {
t.Errorf("automatic merge audit rows = %d, want 1", audits)
}
}
// Lidarr maps both copies, and its release lists the song twice: nothing is
// deleted, the release moves to the one listing each song once, and the next
// pass leaves it there (lesson #4183's fixed point).
func TestResolveDuplicates_ChangesARepeatingRelease_Integration(t *testing.T) {
f := newResolveFixture(t)
lid := &fakeLidarr{
album: lidarr.LidarrAlbum{ID: 4723, ForeignAlbumID: "rg-humanz", Title: "Humanz"},
releases: []lidarr.AlbumRelease{
{ID: 10148, Format: "14x12\" Vinyl, Digital Media", TrackCount: 4, Monitored: true},
{ID: 10149, Format: "Digital Media", TrackCount: 2},
},
releaseTracks: map[int][]lidarr.ReleaseTrack{
10148: rt("Saturnz Barz", "Ascension", "Saturnz Barz", "Ascension"),
10149: rt("Saturnz Barz", "Ascension"),
},
}
res, err := ResolveDuplicates(context.Background(), f.pool, nil, "", lid, true)
if err != nil {
t.Fatalf("resolve: %v", err)
}
if res.Merged != 0 || !exists(f.ripPath) || !exists(f.cleanPath) {
t.Fatalf("merged %d; a copy Lidarr maps was touched", res.Merged)
}
if len(lid.setCalls) != 1 || lid.setCalls[0] != [2]int{4723, 10149} {
t.Fatalf("release changes = %v, want album 4723 to release 10149", lid.setCalls)
}
status, _, note, _ := f.group(t)
if status != "pending" || note == nil || *note == "" {
t.Errorf("group = (%s, note %v), want pending with a note saying what changed", status, note)
}
// The next pass: Lidarr has not rescanned yet, so both copies still read
// as tracked. The album is settling, so nothing changes.
if _, err := ResolveDuplicates(context.Background(), f.pool, nil, "", lid, true); err != nil {
t.Fatalf("second resolve: %v", err)
}
if len(lid.setCalls) != 1 {
t.Errorf("release changed again: %v", lid.setCalls)
}
// And once it has settled, the check itself finds nothing to do: the
// release it chose lists each song once.
change, _, err := changeRelease(context.Background(), dbq.New(f.pool), lid,
&repeatingAlbum{albumID: f.clean.AlbumID, releaseGroupMbid: "rg-humanz"})
if err != nil || change != nil || len(lid.setCalls) != 1 {
t.Errorf("after the change, changeRelease = (%+v, %v) with %d calls; want no change", change, err, len(lid.setCalls))
}
}
// After a release change Lidarr unmaps every file of the album until it has
// rescanned. A pass in that window must not read "every copy unmapped" as
// licence to merge: the copy it would remove may be the one Lidarr maps next.
func TestResolveDuplicates_LeavesAnAlbumAloneWhileLidarrRescans_Integration(t *testing.T) {
f := newResolveFixture(t)
lid := &fakeLidarr{
album: lidarr.LidarrAlbum{ID: 4723, ForeignAlbumID: "rg-humanz"},
releases: []lidarr.AlbumRelease{
{ID: 10148, TrackCount: 2, Monitored: true}, {ID: 10149, TrackCount: 1},
},
releaseTracks: map[int][]lidarr.ReleaseTrack{10148: rt("Saturnz Barz", "Saturnz Barz"), 10149: rt("Saturnz Barz")},
}
if _, err := ResolveDuplicates(context.Background(), f.pool, nil, "", lid, true); err != nil {
t.Fatal(err)
}
lid.unmapped = []string{lidarrPath(f.cleanPath), lidarrPath(f.ripPath)}
res, err := ResolveDuplicates(context.Background(), f.pool, nil, "", lid, true)
if err != nil {
t.Fatal(err)
}
if res.Merged != 0 || !exists(f.ripPath) || !exists(f.cleanPath) {
t.Errorf("merged %d during the rescan window", res.Merged)
}
}
// Without Lidarr, or with auto-resolve off, the pass classifies and removes
// nothing.
func TestResolveDuplicates_ClassifiesOnlyWithoutLidarrOrPermission_Integration(t *testing.T) {
for name, run := range map[string]func(resolveFixture) (DuplicateResolveResult, error){
"lidarr disabled": func(f resolveFixture) (DuplicateResolveResult, error) {
return ResolveDuplicates(context.Background(), f.pool, nil, "", nil, true)
},
"lidarr unreachable": func(f resolveFixture) (DuplicateResolveResult, error) {
return ResolveDuplicates(context.Background(), f.pool, nil, "", &fakeLidarr{unmappedErr: errors.New("timeout")}, true)
},
"auto-resolve off": func(f resolveFixture) (DuplicateResolveResult, error) {
lid := &fakeLidarr{unmapped: []string{lidarrPath(f.ripPath)}}
return ResolveDuplicates(context.Background(), f.pool, nil, "", lid, false)
},
} {
t.Run(name, func(t *testing.T) {
f := newResolveFixture(t)
res, err := run(f)
if err != nil {
t.Fatal(err)
}
if res.Merged != 0 || !exists(f.ripPath) {
t.Errorf("merged %d", res.Merged)
}
status, class, _, _ := f.group(t)
if status != "pending" || class == nil || *class != string(ClassSameRelease) {
t.Errorf("group = (%s, %v), want pending and classified", status, class)
}
})
}
}
// The guard itself: a merge asked to remove a copy the check refuses changes
// nothing, file or row.
func TestMergeDuplicateGroupGuarded_RefusesATrackedCopy_Integration(t *testing.T) {
f := newResolveFixture(t)
refuse := func(p string) bool { return p != f.ripPath }
_, err := MergeDuplicateGroupGuarded(context.Background(), f.pool, nil, "", f.groupID, f.clean.ID, refuse)
if !errors.Is(err, ErrCopyTrackedByLidarr) {
t.Fatalf("err = %v, want ErrCopyTrackedByLidarr", err)
}
if !exists(f.ripPath) {
t.Error("the refused copy's file was removed")
}
if status, _, _, _ := f.group(t); status != "pending" {
t.Errorf("group status = %s, want pending", status)
}
}