package library import ( "context" "errors" "fmt" "log/slog" "os" "slices" "github.com/jackc/pgx/v5" "github.com/jackc/pgx/v5/pgtype" "github.com/jackc/pgx/v5/pgxpool" "git.fabledsword.com/bvandeusen/minstrel/internal/acquire" "git.fabledsword.com/bvandeusen/minstrel/internal/audit" "git.fabledsword.com/bvandeusen/minstrel/internal/db/dbq" "git.fabledsword.com/bvandeusen/minstrel/internal/musicbrainz" "git.fabledsword.com/bvandeusen/minstrel/internal/notifications" syncpkg "git.fabledsword.com/bvandeusen/minstrel/internal/sync" ) // Duplicates that settle without a merge (M498, operator 2026-10-19: "this // needs to be automated away"). The measured library had 536 groups waiting // on the operator; most were retitles across releases, now linked like any // cross-release group. What is left on one album is two shapes: // // - distinct: two versions of a song that sound alike ("Lovesick" and // "Lovesick (Instrumental)"). Not duplicates, so the group is dismissed. // - mismatch: one file filling several track slots (Savant's Void held one // recording as tracks 15, 17 and 20). The album's MusicBrainz track list // says how long each slot's song is. When exactly one slot's song is as // long as the file and every other slot's song is clearly not, the copies // in the other slots are removed and their tracks marked missing, so the // reacquisition sweeper wants the album back and a good copy fills them. // When every slot's song is clearly not the file, every copy goes. // - named: the lengths cannot say (Savant's Mortals held one recording as // 16 of its 50 tracks, each cut to its own slot's length), but AcoustID // can. When every copy's lookup names one recording, or the copies are // the same audio and a lookup names any, the track list says whose slot // that recording is: that copy is kept and the rest go, or every copy // when it is in none of their slots. Same audio that AcoustID does not // know at all is no copy's song, and every copy goes. // // Whenever copies go, the release the files came from is set aside, so the // album fetched again is not the same files (#6169). // // Anything less certain stays with the operator. const ( // trackListFetchesPerPass is how many albums' track lists a pass may ask // MusicBrainz for, when the catalogue has never read them. The resolver // runs hourly, so a backlog drains over a few hours without a burst. trackListFetchesPerPass = 5 // slotFit is how close a file must be to a slot's MusicBrainz length to be // that slot's song, and slotOther how far away every length of a slot's // title must be for the file to be another song's. slotFit = 3000 slotOther = 10000 // identifiesPerPass is how many groups a pass may ask AcoustID about. The // client keeps to AcoustID's rate; this keeps a pass short. identifiesPerPass = 5 // sameAudioBER is the worst bit error rate at which an acoustic group's // copies are one recording cut differently rather than takes that sound // alike: the groups measured as one recording trimmed to several slots // read 0.000, and two takes of one song read far above this (#6169). sameAudioBER = 0.02 ) // slotCopy is what deciding a wrong slot needs to know about one copy. type slotCopy struct { Title, Artist string DurationMs int32 } // wrongSlots decides which copies of a one-album mismatch sit in a slot that // is not theirs. keep is the copy whose slot fits the recording, or -1 when // every copy is clearly another song's (Shnabubula's whole album, 45 minutes // in one file, filed as both "Fading Light" and "Looking for Answers"): then // all of them go. ok is false when the track list cannot say, and nothing // should be removed. // // Several copies may fit when they carry the same title ("Killjoy" twice // beside "Dreamland"): they are one song held twice, not a wrong slot, so they // are neither kept nor lost here. Once the wrong slot is emptied, the next // sweep finds the pair on its own, and the resolver merges it as any song held // twice on one album. func wrongSlots(copies []slotCopy, tracks []dbq.MbReleaseTrack) (keep int, losers []int, ok bool) { keep = -1 keepKey := "" for i, c := range copies { key := MatchTitleKey(c.Title, c.Artist) fits, other, known := false, true, false for _, t := range tracks { if t.LengthMs <= 0 || MatchTitleKey(t.Title, c.Artist) != key { continue } known = true d := absMs(t.LengthMs - c.DurationMs) if d <= slotFit { fits = true } if d <= slotOther { other = false } } switch { case fits && keep >= 0: if key != keepKey { return -1, nil, false } case fits: keep, keepKey = i, key case known && other: losers = append(losers, i) default: return -1, nil, false } } // A copy that neither fits nor is clearly another song's has returned // above, so with no keep every copy is a loser. if len(losers) == 0 { return -1, nil, false } return keep, losers, true } func absMs(d int32) int32 { if d < 0 { return -d } return d } // dismissDistinct dismisses every group of distinct versions and records it // for the Resolved automatically tab. It adds what it settled to settled. func dismissDistinct( ctx context.Context, pool *pgxpool.Pool, logger *slog.Logger, groups []*resolveGroup, settled map[pgtype.UUID]bool, ) int { q := dbq.New(pool) n := 0 for _, g := range groups { if g.class != ClassDistinct || settled[g.id] || ctx.Err() != nil { continue } rows, err := q.SettleDuplicateGroupAutomatically(ctx, dbq.SettleDuplicateGroupAutomaticallyParams{ ID: g.id, Status: "dismissed", }) if err != nil { logger.Warn("duplicate resolve: dismiss failed", "group_id", syncpkg.FormatUUID(g.id), "err", err) continue } if rows == 0 { continue } settled[g.id] = true n++ titles := make([]string, len(g.members)) for i, m := range g.members { titles[i] = m.Title } audit.WriteOrLog(ctx, pool, logger, pgtype.UUID{}, pgtype.UUID{}, audit.ActionDuplicateDismiss, map[string]any{ "automatic": true, "group_id": syncpkg.FormatUUID(g.id), "artist": g.members[0].ArtistName, "titles": titles, }) } return n } // settleWrongSlots removes the copies that sit in another song's slot, for // every one-album mismatch the track list, or AcoustID with it, decides. It // returns how many copies went and the songs whose slots were emptied, and // adds the groups it settled to settled. func settleWrongSlots( ctx context.Context, pool *pgxpool.Pool, logger *slog.Logger, groups []*resolveGroup, settled map[pgtype.UUID]bool, src ResolveSources, ) (int, []string) { q := dbq.New(pool) tracksByGroup := map[string][]dbq.MbReleaseTrack{} fetches, identifies := trackListFetchesPerPass, identifiesPerPass if src.TrackLists == nil { fetches = 0 } if src.Identify == nil { identifies = 0 } removed := 0 var names []string for _, g := range groups { if g.class != ClassMismatch || settled[g.id] || ctx.Err() != nil { continue } rg := g.members[0].ReleaseGroupMbid if rg == "" { continue } tracks, seen := tracksByGroup[rg] if !seen { var err error if tracks, err = q.ListMBReleaseTracksForGroup(ctx, rg); err != nil { logger.Warn("duplicate resolve: reading the album's track list failed", "release_group", rg, "err", err) continue } if len(tracks) == 0 && fetches > 0 && neverRead(ctx, q, rg) { fetches-- if err := src.TrackLists(ctx, rg); err != nil { logger.Info("duplicate resolve: fetching the album's track list failed", "release_group", rg, "err", err) if errors.Is(err, musicbrainz.ErrUnreachable) { fetches = 0 } } else if tracks, err = q.ListMBReleaseTracksForGroup(ctx, rg); err != nil { logger.Warn("duplicate resolve: reading the album's track list failed", "release_group", rg, "err", err) continue } } tracksByGroup[rg] = tracks } copies := make([]slotCopy, len(g.members)) for i, m := range g.members { copies[i] = slotCopy{Title: m.Title, Artist: m.ArtistName, DurationMs: m.DurationMs} } keep, losers, ok := wrongSlots(copies, tracks) if !ok { recordings, known := groupRecordings(ctx, logger, g, src.Identify, &identifies) if !known { continue } if keep, losers, ok = namedCopies(copies, tracks, recordings); !ok { logger.Info("duplicate resolve: AcoustID's answer does not place the recording in one slot; left for review", "group_id", syncpkg.FormatUUID(g.id), "recordings", recordings) } } if !ok { continue } // Set aside before the files go: a pass that dies between the two // still does not fetch the same files again. blocked := blockReleasesHolding(ctx, pool, logger, g) emptied := emptiedSlots(ctx, q, logger, g, losers, tracks) n, err := removeWrongSlots(ctx, pool, logger, g, keep, losers, emptied, blocked) removed += n if err != nil { logger.Warn("duplicate resolve: removing wrong-slot copies failed", "group_id", syncpkg.FormatUUID(g.id), "err", err) continue } settled[g.id] = true for _, i := range losers { names = append(names, TrackLabel(g.members[i].ArtistName, g.members[i].Title)) } } return removed, names } // sameAudio reports whether a group's copies are one recording, cut however: // identical audio, or fingerprints all but identical (sameAudioBER). func sameAudio(g *resolveGroup) bool { return g.tier == "exact" || (g.worstBER != nil && *g.worstBER <= sameAudioBER) } // namedCopies decides a one-album mismatch the lengths could not, from the // recordings AcoustID says it may be. The track list names each slot's // recording: the copy in a slot of one of them is kept and the others are // losers. When none of the copies' slots is one of them, or AcoustID knows // none (recordings empty), every copy is another song's and all go. ok is // false when they fall in more than one copy's slot, the track list names no // recording for a copy's slot, or the copies are all filed as one song. // // Copies sharing the kept copy's title are one song held twice, as in // wrongSlots: neither kept nor lost here. func namedCopies(copies []slotCopy, tracks []dbq.MbReleaseTrack, recordings []string) (keep int, losers []int, ok bool) { keys := make([]string, len(copies)) theirs := map[string]bool{} for i, c := range copies { keys[i] = MatchTitleKey(c.Title, c.Artist) named := false for _, t := range tracks { if t.RecordingMbid == "" || MatchTitleKey(t.Title, c.Artist) != keys[i] { continue } named = true if slices.Contains(recordings, t.RecordingMbid) { theirs[keys[i]] = true } } if !named { return -1, nil, false } } if len(theirs) > 1 { return -1, nil, false } keep = -1 for i, key := range keys { switch { case theirs[key] && keep < 0: keep = i case theirs[key]: default: losers = append(losers, i) } } if len(losers) == 0 { return -1, nil, false } return keep, losers, true } // groupRecordings is what AcoustID says a group's recording may be. Copies // whose own lookups all name one recording are that recording, however their // fingerprints compare. Otherwise only copies that are the same audio // (sameAudio) are asked about: the recordings any copy's lookup matched, none // when every copy's lookup found no match, or else a lookup of one copy now, // while the pass has lookups left. known is false when there is no answer // yet, and the group waits. func groupRecordings( ctx context.Context, logger *slog.Logger, g *resolveGroup, identify RecordingIdentifier, left *int, ) (recordings []string, known bool) { matched, noMatch := 0, 0 for _, m := range g.members { switch { case m.AcoustidState == nil: case *m.AcoustidState == lookupMatched && m.AcoustidRecording != nil: matched++ if !slices.Contains(recordings, *m.AcoustidRecording) { recordings = append(recordings, *m.AcoustidRecording) } case *m.AcoustidState == lookupNoMatch: noMatch++ } } if matched == len(g.members) && len(recordings) == 1 { return recordings, true } if !sameAudio(g) { return nil, false } if len(recordings) > 0 || noMatch == len(g.members) { return recordings, true } if identify == nil || *left <= 0 { return nil, false } *left-- ids, err := identify(ctx, g.members[0].FilePath) if err != nil { if errors.Is(err, ErrIdentifyUnavailable) { *left = 0 } logger.Info("duplicate resolve: asking AcoustID which song a recording is failed", "group_id", syncpkg.FormatUUID(g.id), "err", err) return nil, false } return ids, true } // blockReleasesHolding sets aside the downloads a group's files were imported // from, and returns their names. An album imported before Minstrel downloaded // anything has none. func blockReleasesHolding(ctx context.Context, pool *pgxpool.Pool, logger *slog.Logger, g *resolveGroup) []string { q := dbq.New(pool) seen := map[string]bool{} var names []string for _, m := range g.members { grabs, err := q.ListImportedGrabsHolding(ctx, m.FilePath) if err != nil { logger.Warn("duplicate resolve: finding the download a file came from failed", "path", m.FilePath, "err", err) continue } for _, gr := range grabs { if seen[gr.InfoHash] { continue } seen[gr.InfoHash] = true if err := acquire.NewMemory(pool).Block(ctx, dbq.BlockReleaseParams{ InfoHash: gr.InfoHash, Name: gr.Name, Reason: acquire.BlockFiles, Detail: "one recording fills several of the album's tracks.", WantID: gr.WantID, }); err != nil { logger.Warn("duplicate resolve: setting the release aside failed", "release", gr.Name, "err", err) continue } names = append(names, gr.Name) } } return names } // emptiedSlots is the losers whose song the album holds no other copy of: // no present track on it under that title as long as the song's slot. Those // wait for a copy fetched again (migration 0095). The rest are the song held // twice, and missing-pair folding folds them into the copy held: Savant's // VOID DLC "Weeper" beside Void's. A failed read counts every loser as // emptied, which only makes the album wait for a copy it may not need. func emptiedSlots( ctx context.Context, q *dbq.Queries, logger *slog.Logger, g *resolveGroup, losers []int, tracks []dbq.MbReleaseTrack, ) []pgtype.UUID { held, err := q.ListTracksByAlbum(ctx, dbq.ListTracksByAlbumParams{AlbumID: g.members[0].AlbumID}) if err != nil { logger.Warn("duplicate resolve: reading the album's tracks failed", "group_id", syncpkg.FormatUUID(g.id), "err", err) } inGroup := map[pgtype.UUID]bool{} for _, m := range g.members { inGroup[m.TrackID] = true } var emptied []pgtype.UUID for _, i := range losers { m := g.members[i] key := MatchTitleKey(m.Title, m.ArtistName) covered := false for _, h := range held { if covered || h.MissingSince.Valid || inGroup[h.ID] || MatchTitleKey(h.Title, m.ArtistName) != key { continue } for _, t := range tracks { if t.LengthMs > 0 && MatchTitleKey(t.Title, m.ArtistName) == key && absMs(t.LengthMs-h.DurationMs) <= slotFit { covered = true break } } } if !covered { emptied = append(emptied, m.TrackID) } } return emptied } // neverRead reports whether the catalogue has never read a release group's // releases. A group read and found empty is not asked about again: the // catalogue's own refresh revisits it. func neverRead(ctx context.Context, q *dbq.Queries, rg string) bool { g, err := q.GetMBReleaseGroup(ctx, rg) return errors.Is(err, pgx.ErrNoRows) || (err == nil && !g.ReleasesRefreshedAt.Valid) } // removeWrongSlots deletes each loser's file, then marks its track missing in // the same transaction that closes the group. The row is kept, with its likes // and plays: it is the slot, and a good copy fetched again takes it over. Its // recording MBID is dropped, being the wrong file's, so missing-pair folding // does not fold the slot into the copy that was kept. The slots in emptied // are stamped to wait for a copy fetched again (emptiedSlots). blocked names // the releases set aside, for the record. A file that cannot be removed stops // the group; the ones already gone are still marked, and the group waits for // the next pass. func removeWrongSlots( ctx context.Context, pool *pgxpool.Pool, logger *slog.Logger, g *resolveGroup, keep int, losers []int, emptied []pgtype.UUID, blocked []string, ) (int, error) { var gone []pgtype.UUID var removedFiles []map[string]string var removeErr error for _, i := range losers { m := g.members[i] if err := removeFile(m.FilePath); err != nil && !errors.Is(err, os.ErrNotExist) { removeErr = fmt.Errorf("remove %s: %w", m.FilePath, err) break } gone = append(gone, m.TrackID) removedFiles = append(removedFiles, map[string]string{ "track_id": syncpkg.FormatUUID(m.TrackID), "file_path": m.FilePath, "title": m.Title, }) } if len(gone) == 0 { return 0, removeErr } err := pgx.BeginFunc(ctx, pool, func(tx pgx.Tx) error { tq := dbq.New(tx) if _, err := tq.MarkTracksMissing(ctx, gone); err != nil { return fmt.Errorf("mark missing: %w", err) } if err := tq.ClearTrackMbids(ctx, gone); err != nil { return fmt.Errorf("clear mbids: %w", err) } if err := tq.MarkTracksEmptied(ctx, emptied); err != nil { return fmt.Errorf("mark emptied: %w", err) } if err := logTrackChanges(ctx, tq, gone); err != nil { return fmt.Errorf("log changes: %w", err) } if removeErr != nil { return nil } _, err := tq.SettleDuplicateGroupAutomatically(ctx, dbq.SettleDuplicateGroupAutomaticallyParams{ ID: g.id, Status: "merged", }) return err }) if err != nil { return 0, err } details := map[string]any{ "automatic": true, "group_id": syncpkg.FormatUUID(g.id), "artist": g.members[0].ArtistName, "removed": removedFiles, } if len(blocked) > 0 { details["blocked"] = blocked } keptPath := "" if keep >= 0 { kept := g.members[keep] details["kept_title"], details["kept_path"] = kept.Title, kept.FilePath keptPath = kept.FilePath } audit.WriteOrLog(ctx, pool, logger, pgtype.UUID{}, pgtype.UUID{}, audit.ActionDuplicateWrongFile, details) logger.Info("duplicate resolve: removed copies from slots that were not theirs", "group_id", syncpkg.FormatUUID(g.id), "kept", keptPath, "removed", len(gone)) return len(gone), removeErr } // notifyNewForReview tells admins about groups this pass classified for the // first time and left for them. A group the resolver settled, linked or will // link is not news: telling admins about those was most of what the notice // used to say. func notifyNewForReview(ctx context.Context, groups []*resolveGroup, settled map[pgtype.UUID]bool) { var count int64 var items []string for _, g := range groups { if g.members[0].PriorClass != nil || settled[g.id] { continue } if g.class != ClassReview && g.class != ClassMismatch { continue } count++ if len(items) < notifyItems { items = append(items, TrackLabel(g.members[0].ArtistName, g.members[0].Title)) } } if count > 0 { notifyAdmins(ctx, notifications.KindDuplicatesFound, notifications.Payload{Count: count, Items: items}) } }