feat: duplicates settle themselves: retitled copies across releases link, alike versions on one album are dismissed, and a file in another song's slot is removed so the album is fetched again (M498 #6159-#6162)
release / govulncheck (push) Successful in 14s
release / web (push) Failing after 1m26s
release / go (push) Successful in 1m35s
release / integration (push) Successful in 5m8s
release / android (push) Successful in 5m12s
release / Build signed APK (releases and dev) (push) Successful in 5m26s
release / Attach APK to the Release (tag releases only) (push) Skipped
release / Build + push container image (push) Skipped
release / Verify release artifacts (tag releases only) (push) Skipped
release / govulncheck (push) Successful in 14s
release / web (push) Failing after 1m26s
release / go (push) Successful in 1m35s
release / integration (push) Successful in 5m8s
release / android (push) Successful in 5m12s
release / Build signed APK (releases and dev) (push) Successful in 5m26s
release / Attach APK to the Release (tag releases only) (push) Skipped
release / Build + push container image (push) Skipped
release / Verify release artifacts (tag releases only) (push) Skipped
The operator's library had 536 groups waiting for review, 2026-10-19. A sample
showed most were one recording under two titles on two releases.
- The headline and the Duplicates count leave out cross-release groups, which
the resolver keeps and links: the headline read 3,479 when 536 needed review.
- Any group spanning albums is cross_release and is linked as one song,
whatever its titles say. Nothing is deleted.
- One album, two versions named as such ("Lovesick (Instrumental)") and not
byte-identical: a new class, distinct, dismissed by the resolver.
- One album, one recording under different song titles (mismatch): the
album's cached MusicBrainz track list decides. When exactly one slot's song
is as long as the file and every other slot's song is clearly not, the other
copies are removed and their tracks marked missing, so the reacquisition
sweeper wants the album back. Otherwise the group stays for review.
- The duplicates_found notice moves from the sweep to the resolver: it names
only groups a pass first left for review.
- Resolved automatically lists dismissals and wrong-slot removals beside
merges.
Migration 0094 adds 'distinct' to the class CHECK (rule 36).
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -242,9 +242,10 @@ func TestResolveDuplicates_KeepsCleanCopiesOnTwoReleases_Integration(t *testing.
|
||||
}
|
||||
}
|
||||
|
||||
// Identical audio under two titles on two albums, neither a rip: which tags
|
||||
// are wrong is the operator's call, so nothing is merged or linked.
|
||||
func TestResolveDuplicates_LeavesAMismatchForTheOperator_Integration(t *testing.T) {
|
||||
// Identical audio under two titles on two albums, neither a rip: a later
|
||||
// release retitled the recording ("F.O.D." and "F.O.D. (live)"). Each copy
|
||||
// belongs to its release, so nothing is removed and they count as one song.
|
||||
func TestResolveDuplicates_LinksOneRecordingUnderTwoTitles_Integration(t *testing.T) {
|
||||
f := newCrossReleaseRipFixture(t)
|
||||
ctx := context.Background()
|
||||
if _, err := f.pool.Exec(ctx, `UPDATE duplicate_groups SET tier = 'exact' WHERE id = $1`, f.groupID); err != nil {
|
||||
@@ -258,11 +259,166 @@ func TestResolveDuplicates_LeavesAMismatchForTheOperator_Integration(t *testing.
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if res.Classes[ClassMismatch] != 1 || res.Merged != 0 || res.SongsLinked != 0 {
|
||||
t.Fatalf("classes %v, merged %d, linked %d; want a mismatch left alone", res.Classes, res.Merged, res.SongsLinked)
|
||||
if res.Classes[ClassCrossRelease] != 1 || res.Merged != 0 || res.SongsLinked != 1 {
|
||||
t.Fatalf("classes %v, merged %d, linked %d; want the pair linked and kept", res.Classes, res.Merged, res.SongsLinked)
|
||||
}
|
||||
if status, _, note, _ := (resolveFixture{pool: f.pool, groupID: f.groupID}).group(t); status != "pending" || note == nil || *note == "" {
|
||||
t.Errorf("group = (%s, note %v), want pending with a note", status, note)
|
||||
}
|
||||
|
||||
// oneAlbumFixture is one album holding two tracks in one pending group, each
|
||||
// with a file on disk.
|
||||
type oneAlbumFixture struct {
|
||||
pool *pgxpool.Pool
|
||||
album dbq.Album
|
||||
a, b dbq.Track
|
||||
aPath string
|
||||
bPath string
|
||||
groupID pgtype.UUID
|
||||
}
|
||||
|
||||
func newOneAlbumFixture(t *testing.T, tier, titleA, titleB string) oneAlbumFixture {
|
||||
t.Helper()
|
||||
pool := newPool(t)
|
||||
ctx := context.Background()
|
||||
q := dbq.New(pool)
|
||||
dir := filepath.Join(t.TempDir(), "Savant", "Void")
|
||||
if err := os.MkdirAll(dir, 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
f := oneAlbumFixture{pool: pool, aPath: filepath.Join(dir, "15 - a.m4a"), bPath: filepath.Join(dir, "17 - b.m4a")}
|
||||
for _, p := range []string{f.aPath, f.bPath} {
|
||||
if err := os.WriteFile(p, []byte("audio"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
var artist dbq.Artist
|
||||
f.a, f.album, artist = seedTrack(t, pool, f.aPath)
|
||||
b, err := q.UpsertTrack(ctx, dbq.UpsertTrackParams{
|
||||
Title: titleB, AlbumID: f.album.ID, ArtistID: artist.ID,
|
||||
DurationMs: 220000, FilePath: f.bPath, FileSize: 100, FileFormat: "m4a",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
f.b = b
|
||||
if _, err := pool.Exec(ctx, `UPDATE tracks SET title = $2, duration_ms = 220000 WHERE id = $1`, f.a.ID, titleA); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := pool.QueryRow(ctx,
|
||||
`INSERT INTO duplicate_groups (member_key, tier) VALUES ($1, $2) RETURNING id`, t.Name(), tier,
|
||||
).Scan(&f.groupID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := pool.Exec(ctx, `INSERT INTO duplicate_group_members (group_id, track_id) VALUES ($1, $2), ($1, $3)`,
|
||||
f.groupID, f.a.ID, f.b.ID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return f
|
||||
}
|
||||
|
||||
func auditCount(t *testing.T, pool *pgxpool.Pool, action string, groupID pgtype.UUID) int {
|
||||
t.Helper()
|
||||
var n int
|
||||
if err := pool.QueryRow(context.Background(),
|
||||
`SELECT count(*) FROM audit_log WHERE action = $1 AND metadata->>'group_id' = $2`,
|
||||
action, syncpkg.FormatUUID(groupID)).Scan(&n); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
// Two versions of a song on one album that sound alike are not duplicates:
|
||||
// the group is dismissed and both files stay.
|
||||
func TestResolveDuplicates_DismissesTwoVersionsOfASong_Integration(t *testing.T) {
|
||||
f := newOneAlbumFixture(t, "acoustic", "Lovesick", "Lovesick (Instrumental)")
|
||||
res, err := ResolveDuplicates(context.Background(), f.pool, nil, "", true)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if res.Dismissed != 1 || res.Merged != 0 || !exists(f.aPath) || !exists(f.bPath) {
|
||||
t.Fatalf("result %+v; want the group dismissed and both files kept", res)
|
||||
}
|
||||
if status, _, _, auto := (resolveFixture{pool: f.pool, groupID: f.groupID}).group(t); status != "dismissed" || !auto {
|
||||
t.Errorf("group = (%s, auto=%v), want dismissed by the resolver", status, auto)
|
||||
}
|
||||
if n := auditCount(t, f.pool, "duplicate_dismiss", f.groupID); n != 1 {
|
||||
t.Errorf("dismiss audit rows = %d, want 1", n)
|
||||
}
|
||||
}
|
||||
|
||||
// seedTrackList holds a release group whose one release lists the songs with
|
||||
// their lengths, and points the album at it.
|
||||
func seedTrackList(t *testing.T, pool *pgxpool.Pool, album dbq.Album, songs map[string]int) {
|
||||
t.Helper()
|
||||
ctx := context.Background()
|
||||
rg := "rg-" + syncpkg.FormatUUID(album.ID)
|
||||
rel := "rel-" + syncpkg.FormatUUID(album.ID)
|
||||
for _, stmt := range []struct {
|
||||
sql string
|
||||
args []any
|
||||
}{
|
||||
{`INSERT INTO mb_release_groups (mbid, title) VALUES ($1, 'Void')`, []any{rg}},
|
||||
{`INSERT INTO mb_releases (mbid, release_group_mbid, title) VALUES ($1, $2, 'Void')`, []any{rel, rg}},
|
||||
{`UPDATE albums SET release_group_mbid = $2 WHERE id = $1`, []any{album.ID, rg}},
|
||||
} {
|
||||
if _, err := pool.Exec(ctx, stmt.sql, stmt.args...); err != nil {
|
||||
t.Fatalf("exec %q: %v", stmt.sql, err)
|
||||
}
|
||||
}
|
||||
pos := 1
|
||||
for title, ms := range songs {
|
||||
if _, err := pool.Exec(ctx,
|
||||
`INSERT INTO mb_release_tracks (release_mbid, medium_position, position, title, length_ms) VALUES ($1, 1, $2, $3, $4)`,
|
||||
rel, pos, title, ms); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
pos++
|
||||
}
|
||||
}
|
||||
|
||||
// One recording in two slots of an album: MusicBrainz says Void is as long as
|
||||
// the file and Reach is a minute longer, so the copy in Reach's slot goes. Its
|
||||
// track stays, marked missing, for the album to be fetched again.
|
||||
func TestResolveDuplicates_RemovesACopyFromAnotherSongsSlot_Integration(t *testing.T) {
|
||||
f := newOneAlbumFixture(t, "acoustic", "Void", "Reach")
|
||||
seedTrackList(t, f.pool, f.album, map[string]int{"Void": 220500, "Reach": 281000})
|
||||
res, err := ResolveDuplicates(context.Background(), f.pool, nil, "", true)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if res.WrongFiles != 1 || !exists(f.aPath) || exists(f.bPath) {
|
||||
t.Fatalf("result %+v, void exists %v, reach exists %v; want only Reach's copy removed",
|
||||
res, exists(f.aPath), exists(f.bPath))
|
||||
}
|
||||
var missing bool
|
||||
if err := f.pool.QueryRow(context.Background(),
|
||||
`SELECT missing_since IS NOT NULL FROM tracks WHERE id = $1`, f.b.ID).Scan(&missing); err != nil {
|
||||
t.Fatalf("Reach's track is gone: %v", err)
|
||||
}
|
||||
if !missing {
|
||||
t.Error("Reach's track is not marked missing, so nothing will fetch it again")
|
||||
}
|
||||
if status, _, _, auto := (resolveFixture{pool: f.pool, groupID: f.groupID}).group(t); status != "merged" || !auto {
|
||||
t.Errorf("group = (%s, auto=%v), want settled by the resolver", status, auto)
|
||||
}
|
||||
if n := auditCount(t, f.pool, "duplicate_wrong_file", f.groupID); n != 1 {
|
||||
t.Errorf("wrong-file audit rows = %d, want 1", n)
|
||||
}
|
||||
}
|
||||
|
||||
// Without a track list that decides, nothing is removed.
|
||||
func TestResolveDuplicates_LeavesAnUndecidedSlotForTheOperator_Integration(t *testing.T) {
|
||||
f := newOneAlbumFixture(t, "acoustic", "Void", "Reach")
|
||||
seedTrackList(t, f.pool, f.album, map[string]int{"Void": 220500, "Reach": 221000})
|
||||
res, err := ResolveDuplicates(context.Background(), f.pool, nil, "", true)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if res.WrongFiles != 0 || !exists(f.aPath) || !exists(f.bPath) {
|
||||
t.Fatalf("result %+v; want both copies kept when the lengths cannot tell", res)
|
||||
}
|
||||
if status, class, _, _ := (resolveFixture{pool: f.pool, groupID: f.groupID}).group(t); status != "pending" ||
|
||||
class == nil || *class != string(ClassMismatch) {
|
||||
t.Errorf("group = (%s, %v), want a pending mismatch", status, class)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user