package library import ( "context" "os" "path/filepath" "testing" "github.com/jackc/pgx/v5/pgtype" "github.com/jackc/pgx/v5/pgxpool" "git.fabledsword.com/bvandeusen/minstrel/internal/db/dbq" syncpkg "git.fabledsword.com/bvandeusen/minstrel/internal/sync" ) // resolveFixture is the Humanz shape: one album, one song twice, both copies on // disk under an artist/album folder, in one pending exact-tier group. type resolveFixture struct { pool *pgxpool.Pool clean, rip dbq.Track cleanPath, ripPath string groupID pgtype.UUID } func newResolveFixture(t *testing.T) resolveFixture { t.Helper() pool := newPool(t) ctx := context.Background() q := dbq.New(pool) dir := filepath.Join(t.TempDir(), "Gorillaz", "Humanz (2017)") if err := os.MkdirAll(dir, 0o755); err != nil { t.Fatal(err) } f := resolveFixture{pool: pool} f.cleanPath = filepath.Join(dir, "Gorillaz - Humanz - 03 - Saturnz Barz.mp3") f.ripPath = filepath.Join(dir, "Gorillaz - Humanz - 03 - Saturnz Barz (Official Video).mp3") for _, p := range []string{f.cleanPath, f.ripPath} { if err := os.WriteFile(p, []byte("audio"), 0o644); err != nil { t.Fatal(err) } } var album dbq.Album var artist dbq.Artist f.clean, album, artist = seedTrack(t, pool, f.cleanPath) rip, err := q.UpsertTrack(ctx, dbq.UpsertTrackParams{ Title: "Saturnz Barz (Official Video)", AlbumID: album.ID, ArtistID: artist.ID, DurationMs: 1000, FilePath: f.ripPath, FileSize: 100, FileFormat: "mp3", }) if err != nil { t.Fatal(err) } f.rip = rip exec := func(sql string, args ...any) { t.Helper() if _, err := pool.Exec(ctx, sql, args...); err != nil { t.Fatalf("exec %q: %v", sql, err) } } exec(`UPDATE tracks SET title = 'Saturnz Barz' WHERE id = $1`, f.clean.ID) if err := pool.QueryRow(ctx, `INSERT INTO duplicate_groups (member_key, tier) VALUES ('resolve-fixture', 'exact') RETURNING id`, ).Scan(&f.groupID); err != nil { t.Fatal(err) } exec(`INSERT INTO duplicate_group_members (group_id, track_id) VALUES ($1, $2), ($1, $3)`, f.groupID, f.clean.ID, f.rip.ID) return f } func (f resolveFixture) group(t *testing.T) (status string, class *string, note *string, auto bool) { t.Helper() if err := f.pool.QueryRow(context.Background(), `SELECT status, class, resolve_note, resolved_automatically FROM duplicate_groups WHERE id = $1`, f.groupID, ).Scan(&status, &class, ¬e, &auto); err != nil { t.Fatal(err) } return status, class, note, auto } func exists(p string) bool { _, err := os.Stat(p); return err == nil } // autoMergeAudit reads the resolver's audit row for a group: how many there // are, and the reason the last one gave for its survivor. func autoMergeAudit(t *testing.T, pool *pgxpool.Pool, groupID pgtype.UUID) (n int, reason string) { t.Helper() if err := pool.QueryRow(context.Background(), `SELECT count(*), coalesce(max(metadata->>'survivor_reason'), '') FROM audit_log WHERE action = 'duplicate_merge' AND actor_id IS NULL AND metadata->>'group_id' = $1 AND (metadata->>'automatic')::boolean`, syncpkg.FormatUUID(groupID)).Scan(&n, &reason); err != nil { t.Fatal(err) } return n, reason } // One album holding a song twice: the extra copy goes, the copy whose name and // tags fit the album stays, and the merge is recorded as the resolver's // (M502 #5517): Minstrel is the only program writing the library. func TestResolveDuplicates_MergesASameReleaseCopy_Integration(t *testing.T) { f := newResolveFixture(t) res, err := ResolveDuplicates(context.Background(), f.pool, nil, "", true) if err != nil { t.Fatalf("resolve: %v", err) } if res.Merged != 1 || res.Classes[ClassSameRelease] != 1 { t.Fatalf("result = %+v, want the same-release group merged", res) } if exists(f.ripPath) || !exists(f.cleanPath) { t.Errorf("rip exists=%v clean exists=%v; want only the clean copy left", exists(f.ripPath), exists(f.cleanPath)) } status, class, _, auto := f.group(t) if status != "merged" || class == nil || *class != string(ClassSameRelease) || !auto { t.Errorf("group = (%s, %v, auto=%v), want merged same_release by the resolver", status, class, auto) } if n, _ := autoMergeAudit(t, f.pool, f.groupID); n != 1 { t.Errorf("automatic merge audit rows = %d, want 1", n) } } // With auto-resolve off, the pass classifies and removes nothing. func TestResolveDuplicates_ClassifiesOnlyWithoutPermission_Integration(t *testing.T) { f := newResolveFixture(t) res, err := ResolveDuplicates(context.Background(), f.pool, nil, "", false) if err != nil { t.Fatal(err) } if res.Merged != 0 || !exists(f.ripPath) { t.Errorf("merged %d", res.Merged) } status, class, _, _ := f.group(t) if status != "pending" || class == nil || *class != string(ClassSameRelease) { t.Errorf("group = (%s, %v), want pending and classified", status, class) } } // crossReleaseRipFixture is a video rip on one release with the clean copy on // another: the album cut, and the same song's video filed as a single. type crossReleaseRipFixture struct { pool *pgxpool.Pool clean, rip dbq.Track cleanPath, ripPath string groupID pgtype.UUID } func newCrossReleaseRipFixture(t *testing.T) crossReleaseRipFixture { t.Helper() pool := newPool(t) ctx := context.Background() q := dbq.New(pool) root := t.TempDir() f := crossReleaseRipFixture{pool: pool} f.cleanPath = filepath.Join(root, "Gorillaz", "Demon Days (2005)", "06 - Feel Good Inc.mp3") f.ripPath = filepath.Join(root, "Gorillaz", "Feel Good Inc (2005)", "01 - Feel Good Inc. (Official Video).mp3") artist, err := q.UpsertArtist(ctx, dbq.UpsertArtistParams{Name: "Gorillaz", SortName: "Gorillaz"}) if err != nil { t.Fatal(err) } track := func(albumTitle, title, path string) dbq.Track { t.Helper() if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { t.Fatal(err) } if err := os.WriteFile(path, []byte("audio"), 0o644); err != nil { t.Fatal(err) } album, err := q.UpsertAlbum(ctx, dbq.UpsertAlbumParams{Title: albumTitle, SortTitle: albumTitle, ArtistID: artist.ID}) if err != nil { t.Fatal(err) } tr, err := q.UpsertTrack(ctx, dbq.UpsertTrackParams{ Title: title, AlbumID: album.ID, ArtistID: artist.ID, DurationMs: 1000, FilePath: path, FileSize: 100, FileFormat: "mp3", }) if err != nil { t.Fatal(err) } return tr } f.clean = track("Demon Days", "Feel Good Inc.", f.cleanPath) f.rip = track("Feel Good Inc.", "Feel Good Inc. (Official Video)", f.ripPath) if err := pool.QueryRow(ctx, `INSERT INTO duplicate_groups (member_key, tier) VALUES ('cross-rip', 'acoustic') RETURNING id`, ).Scan(&f.groupID); err != nil { t.Fatal(err) } if _, err := pool.Exec(ctx, `INSERT INTO duplicate_group_members (group_id, track_id) VALUES ($1, $2), ($1, $3)`, f.groupID, f.clean.ID, f.rip.ID); err != nil { t.Fatal(err) } return f } // A video rip on one release with the clean copy on another: the rip is no // release's copy, so it goes and the album cut stays (#5439). func TestResolveDuplicates_MergesARipAcrossReleases_Integration(t *testing.T) { f := newCrossReleaseRipFixture(t) res, err := ResolveDuplicates(context.Background(), f.pool, nil, "", true) if err != nil { t.Fatal(err) } if res.Classes[ClassCrossRelease] != 1 || res.Merged != 1 { t.Fatalf("classes %v, merged %d; want the cross-release group merged", res.Classes, res.Merged) } if exists(f.ripPath) || !exists(f.cleanPath) { t.Errorf("rip exists %v, clean exists %v; want only the clean copy", exists(f.ripPath), exists(f.cleanPath)) } if _, reason := autoMergeAudit(t, f.pool, f.groupID); reason != "the copy that is not a video rip" { t.Errorf("survivor reason %q, want the rip rule", reason) } } // Copies on two releases that are both clean each belong to their release: // nothing is removed, and they are linked as one song. A rip the operator // marked fine counts as clean. func TestResolveDuplicates_KeepsCleanCopiesOnTwoReleases_Integration(t *testing.T) { for name, prepare := range map[string]func(f crossReleaseRipFixture) (string, []any){ "renamed": func(f crossReleaseRipFixture) (string, []any) { return `UPDATE tracks SET title = 'Feel Good Inc.', file_path = $2 WHERE id = $1`, []any{f.rip.ID, filepath.Join(filepath.Dir(f.ripPath), "01 - Feel Good Inc.mp3")} }, "kept as fine": func(f crossReleaseRipFixture) (string, []any) { return `UPDATE tracks SET source_verdict = 'fine' WHERE id = $1`, []any{f.rip.ID} }, } { t.Run(name, func(t *testing.T) { f := newCrossReleaseRipFixture(t) sql, args := prepare(f) if _, err := f.pool.Exec(context.Background(), sql, args...); err != nil { t.Fatal(err) } res, err := ResolveDuplicates(context.Background(), f.pool, nil, "", true) if err != nil { t.Fatal(err) } if res.Merged != 0 || !exists(f.ripPath) { t.Errorf("merged %d; want both copies kept", res.Merged) } if res.SongsLinked != 1 { t.Errorf("linked %d, want the pair linked as one song", res.SongsLinked) } }) } } // Identical audio under two titles on two albums, neither a rip: a later // release retitled the recording ("F.O.D." and "F.O.D. (live)"). Each copy // belongs to its release, so nothing is removed and they count as one song. func TestResolveDuplicates_LinksOneRecordingUnderTwoTitles_Integration(t *testing.T) { f := newCrossReleaseRipFixture(t) ctx := context.Background() if _, err := f.pool.Exec(ctx, `UPDATE duplicate_groups SET tier = 'exact' WHERE id = $1`, f.groupID); err != nil { t.Fatal(err) } if _, err := f.pool.Exec(ctx, `UPDATE tracks SET title = 'DARE', file_path = $2 WHERE id = $1`, f.rip.ID, filepath.Join(filepath.Dir(f.ripPath), "01 - DARE.mp3")); err != nil { t.Fatal(err) } res, err := ResolveDuplicates(ctx, f.pool, nil, "", true) if err != nil { t.Fatal(err) } if res.Classes[ClassCrossRelease] != 1 || res.Merged != 0 || res.SongsLinked != 1 { t.Fatalf("classes %v, merged %d, linked %d; want the pair linked and kept", res.Classes, res.Merged, res.SongsLinked) } } // oneAlbumFixture is one album holding two tracks in one pending group, each // with a file on disk. type oneAlbumFixture struct { pool *pgxpool.Pool album dbq.Album a, b dbq.Track aPath string bPath string groupID pgtype.UUID } func newOneAlbumFixture(t *testing.T, tier, titleA, titleB string) oneAlbumFixture { t.Helper() pool := newPool(t) ctx := context.Background() q := dbq.New(pool) dir := filepath.Join(t.TempDir(), "Savant", "Void") if err := os.MkdirAll(dir, 0o755); err != nil { t.Fatal(err) } f := oneAlbumFixture{pool: pool, aPath: filepath.Join(dir, "15 - a.m4a"), bPath: filepath.Join(dir, "17 - b.m4a")} for _, p := range []string{f.aPath, f.bPath} { if err := os.WriteFile(p, []byte("audio"), 0o644); err != nil { t.Fatal(err) } } var artist dbq.Artist f.a, f.album, artist = seedTrack(t, pool, f.aPath) b, err := q.UpsertTrack(ctx, dbq.UpsertTrackParams{ Title: titleB, AlbumID: f.album.ID, ArtistID: artist.ID, DurationMs: 220000, FilePath: f.bPath, FileSize: 100, FileFormat: "m4a", }) if err != nil { t.Fatal(err) } f.b = b if _, err := pool.Exec(ctx, `UPDATE tracks SET title = $2, duration_ms = 220000 WHERE id = $1`, f.a.ID, titleA); err != nil { t.Fatal(err) } if err := pool.QueryRow(ctx, `INSERT INTO duplicate_groups (member_key, tier) VALUES ($1, $2) RETURNING id`, t.Name(), tier, ).Scan(&f.groupID); err != nil { t.Fatal(err) } if _, err := pool.Exec(ctx, `INSERT INTO duplicate_group_members (group_id, track_id) VALUES ($1, $2), ($1, $3)`, f.groupID, f.a.ID, f.b.ID); err != nil { t.Fatal(err) } return f } func auditCount(t *testing.T, pool *pgxpool.Pool, action string, groupID pgtype.UUID) int { t.Helper() var n int if err := pool.QueryRow(context.Background(), `SELECT count(*) FROM audit_log WHERE action = $1 AND metadata->>'group_id' = $2`, action, syncpkg.FormatUUID(groupID)).Scan(&n); err != nil { t.Fatal(err) } return n } // Two versions of a song on one album that sound alike are not duplicates: // the group is dismissed and both files stay. func TestResolveDuplicates_DismissesTwoVersionsOfASong_Integration(t *testing.T) { f := newOneAlbumFixture(t, "acoustic", "Lovesick", "Lovesick (Instrumental)") res, err := ResolveDuplicates(context.Background(), f.pool, nil, "", true) if err != nil { t.Fatal(err) } if res.Dismissed != 1 || res.Merged != 0 || !exists(f.aPath) || !exists(f.bPath) { t.Fatalf("result %+v; want the group dismissed and both files kept", res) } if status, _, _, auto := (resolveFixture{pool: f.pool, groupID: f.groupID}).group(t); status != "dismissed" || !auto { t.Errorf("group = (%s, auto=%v), want dismissed by the resolver", status, auto) } if n := auditCount(t, f.pool, "duplicate_dismiss", f.groupID); n != 1 { t.Errorf("dismiss audit rows = %d, want 1", n) } } // seedTrackList holds a release group whose one release lists the songs with // their lengths, and points the album at it. func seedTrackList(t *testing.T, pool *pgxpool.Pool, album dbq.Album, songs map[string]int) { t.Helper() ctx := context.Background() rg := "rg-" + syncpkg.FormatUUID(album.ID) rel := "rel-" + syncpkg.FormatUUID(album.ID) for _, stmt := range []struct { sql string args []any }{ {`INSERT INTO mb_release_groups (mbid, title) VALUES ($1, 'Void')`, []any{rg}}, {`INSERT INTO mb_releases (mbid, release_group_mbid, title) VALUES ($1, $2, 'Void')`, []any{rel, rg}}, {`UPDATE albums SET release_group_mbid = $2 WHERE id = $1`, []any{album.ID, rg}}, } { if _, err := pool.Exec(ctx, stmt.sql, stmt.args...); err != nil { t.Fatalf("exec %q: %v", stmt.sql, err) } } pos := 1 for title, ms := range songs { if _, err := pool.Exec(ctx, `INSERT INTO mb_release_tracks (release_mbid, medium_position, position, title, length_ms) VALUES ($1, 1, $2, $3, $4)`, rel, pos, title, ms); err != nil { t.Fatal(err) } pos++ } } // One recording in two slots of an album: MusicBrainz says Void is as long as // the file and Reach is a minute longer, so the copy in Reach's slot goes. Its // track stays, marked missing, for the album to be fetched again. func TestResolveDuplicates_RemovesACopyFromAnotherSongsSlot_Integration(t *testing.T) { f := newOneAlbumFixture(t, "acoustic", "Void", "Reach") seedTrackList(t, f.pool, f.album, map[string]int{"Void": 220500, "Reach": 281000}) res, err := ResolveDuplicates(context.Background(), f.pool, nil, "", true) if err != nil { t.Fatal(err) } if res.WrongFiles != 1 || !exists(f.aPath) || exists(f.bPath) { t.Fatalf("result %+v, void exists %v, reach exists %v; want only Reach's copy removed", res, exists(f.aPath), exists(f.bPath)) } var missing bool if err := f.pool.QueryRow(context.Background(), `SELECT missing_since IS NOT NULL FROM tracks WHERE id = $1`, f.b.ID).Scan(&missing); err != nil { t.Fatalf("Reach's track is gone: %v", err) } if !missing { t.Error("Reach's track is not marked missing, so nothing will fetch it again") } if status, _, _, auto := (resolveFixture{pool: f.pool, groupID: f.groupID}).group(t); status != "merged" || !auto { t.Errorf("group = (%s, auto=%v), want settled by the resolver", status, auto) } if n := auditCount(t, f.pool, "duplicate_wrong_file", f.groupID); n != 1 { t.Errorf("wrong-file audit rows = %d, want 1", n) } } // An album whose track list the catalogue never read: the resolver asks for // it, once, and decides with what comes back. func TestResolveDuplicates_FetchesAMissingTrackList_Integration(t *testing.T) { f := newOneAlbumFixture(t, "acoustic", "Void", "Reach") rg := "rg-" + syncpkg.FormatUUID(f.album.ID) if _, err := f.pool.Exec(context.Background(), `UPDATE albums SET release_group_mbid = $2 WHERE id = $1`, f.album.ID, rg); err != nil { t.Fatal(err) } calls := 0 fetch := func(_ context.Context, asked string) error { calls++ if asked != rg { t.Errorf("asked for %s, want %s", asked, rg) } seedTrackList(t, f.pool, f.album, map[string]int{"Void": 220500, "Reach": 281000}) return nil } res, err := ResolveDuplicatesFetching(context.Background(), f.pool, nil, "", true, fetch) if err != nil { t.Fatal(err) } if calls != 1 || res.WrongFiles != 1 || exists(f.bPath) { t.Fatalf("fetches %d, result %+v; want one fetch and Reach's copy removed", calls, res) } } // Without a track list that decides, nothing is removed. func TestResolveDuplicates_LeavesAnUndecidedSlotForTheOperator_Integration(t *testing.T) { f := newOneAlbumFixture(t, "acoustic", "Void", "Reach") seedTrackList(t, f.pool, f.album, map[string]int{"Void": 220500, "Reach": 221000}) res, err := ResolveDuplicates(context.Background(), f.pool, nil, "", true) if err != nil { t.Fatal(err) } if res.WrongFiles != 0 || !exists(f.aPath) || !exists(f.bPath) { t.Fatalf("result %+v; want both copies kept when the lengths cannot tell", res) } if status, class, _, _ := (resolveFixture{pool: f.pool, groupID: f.groupID}).group(t); status != "pending" || class == nil || *class != string(ClassMismatch) { t.Errorf("group = (%s, %v), want a pending mismatch", status, class) } } // A rip is held back from radio and the mixes; the operator's "fine" sticks // through later passes; a renamed file is released. func TestResolveDuplicates_HoldsBackRips_Integration(t *testing.T) { f := newCrossReleaseRipFixture(t) ctx := context.Background() q := dbq.New(f.pool) // Without its group the rip has no clean copy to merge into, so it stays // on disk and only its verdict changes. if _, err := f.pool.Exec(ctx, `DELETE FROM duplicate_groups WHERE id = $1`, f.groupID); err != nil { t.Fatal(err) } verdict := func(id pgtype.UUID) *string { t.Helper() var v *string if err := f.pool.QueryRow(ctx, `SELECT source_verdict FROM tracks WHERE id = $1`, id).Scan(&v); err != nil { t.Fatal(err) } return v } resolve := func() DuplicateResolveResult { t.Helper() res, err := ResolveDuplicates(ctx, f.pool, nil, "", true) if err != nil { t.Fatal(err) } return res } if res := resolve(); res.HeldBack != 1 { t.Errorf("held back %d, want the rip", res.HeldBack) } if v := verdict(f.rip.ID); v == nil || *v != "suspect" { t.Errorf("rip verdict %v, want suspect", v) } if v := verdict(f.clean.ID); v != nil { t.Errorf("clean copy verdict %v, want none", *v) } if n, err := q.SetTrackSourceVerdict(ctx, dbq.SetTrackSourceVerdictParams{Verdict: "fine", ID: f.rip.ID}); err != nil || n != 1 { t.Fatalf("set fine: %d, %v", n, err) } if res := resolve(); res.HeldBack != 0 { t.Errorf("held back %d after the operator's fine, want 0", res.HeldBack) } if v := verdict(f.rip.ID); v == nil || *v != "fine" { t.Errorf("rip verdict %v, want fine kept", v) } // Held back again, then renamed: the next pass lets it go. if _, err := q.SetTrackSourceVerdict(ctx, dbq.SetTrackSourceVerdictParams{Verdict: "suspect", ID: f.rip.ID}); err != nil { t.Fatal(err) } if _, err := f.pool.Exec(ctx, `UPDATE tracks SET file_path = $2 WHERE id = $1`, f.rip.ID, filepath.Join(filepath.Dir(f.ripPath), "01 - Feel Good Inc.mp3")); err != nil { t.Fatal(err) } if res := resolve(); res.Released != 1 { t.Errorf("released %d, want the renamed track", res.Released) } if v := verdict(f.rip.ID); v != nil { t.Errorf("renamed track verdict %v, want none", *v) } // The clean copy can't be flagged by hand: there is nothing to judge. if n, err := q.SetTrackSourceVerdict(ctx, dbq.SetTrackSourceVerdictParams{Verdict: "fine", ID: f.clean.ID}); err != nil || n != 0 { t.Errorf("set on an unflagged track: %d, %v; want 0 rows", n, err) } }