minstrel/internal/library/duplicate_settle.go

package library

import (
	"context"
	"errors"
	"fmt"
	"log/slog"
	"os"
	"slices"

	"github.com/jackc/pgx/v5"
	"github.com/jackc/pgx/v5/pgtype"
	"github.com/jackc/pgx/v5/pgxpool"

	"git.fabledsword.com/bvandeusen/minstrel/internal/acquire"
	"git.fabledsword.com/bvandeusen/minstrel/internal/audit"
	"git.fabledsword.com/bvandeusen/minstrel/internal/db/dbq"
	"git.fabledsword.com/bvandeusen/minstrel/internal/musicbrainz"
	"git.fabledsword.com/bvandeusen/minstrel/internal/notifications"
	syncpkg "git.fabledsword.com/bvandeusen/minstrel/internal/sync"
)

// Duplicates that settle without a merge (M498, operator 2026-10-19: "this
// needs to be automated away"). The measured library had 536 groups waiting
// on the operator; most were retitles across releases, now linked like any
// cross-release group. What is left on one album is two shapes:
//
//   - distinct: two versions of a song that sound alike ("Lovesick" and
//     "Lovesick (Instrumental)"). Not duplicates, so the group is dismissed.
//   - mismatch: one file filling several track slots (Savant's Void held one
//     recording as tracks 15, 17 and 20). The album's MusicBrainz track list
//     says how long each slot's song is. When exactly one slot's song is as
//     long as the file and every other slot's song is clearly not, the copies
//     in the other slots are removed and their tracks marked missing, so the
//     reacquisition sweeper wants the album back and a good copy fills them.
//     When every slot's song is clearly not the file, every copy goes.
//   - named: the lengths cannot say (Savant's Mortals held one recording as
//     16 of its 50 tracks, each cut to its own slot's length), but AcoustID
//     can. When every copy's lookup names one recording, or the copies are
//     the same audio and a lookup names any, the track list says whose slot
//     that recording is: that copy is kept and the rest go, or every copy
//     when it is in none of their slots. Same audio that AcoustID does not
//     know at all is no copy's song, and every copy goes.
//
// Whenever copies go, the release the files came from is set aside, so the
// album fetched again is not the same files (#6169).
//
// Anything less certain stays with the operator.

const (
	// trackListFetchesPerPass is how many albums' track lists a pass may ask
	// MusicBrainz for, when the catalogue has never read them. The resolver
	// runs hourly, so a backlog drains over a few hours without a burst.
	trackListFetchesPerPass = 5
	// slotFit is how close a file must be to a slot's MusicBrainz length to be
	// that slot's song, and slotOther how far away every length of a slot's
	// title must be for the file to be another song's.
	slotFit   = 3000
	slotOther = 10000
	// identifiesPerPass is how many groups a pass may ask AcoustID about. The
	// client keeps to AcoustID's rate; this keeps a pass short.
	identifiesPerPass = 5
	// sameAudioBER is the worst bit error rate at which an acoustic group's
	// copies are one recording cut differently rather than takes that sound
	// alike: the groups measured as one recording trimmed to several slots
	// read 0.000, and two takes of one song read far above this (#6169).
	sameAudioBER = 0.02
)

// slotCopy is what deciding a wrong slot needs to know about one copy.
type slotCopy struct {
	Title, Artist string
	DurationMs    int32
}

// wrongSlots decides which copies of a one-album mismatch sit in a slot that
// is not theirs. keep is the copy whose slot fits the recording, or -1 when
// every copy is clearly another song's (Shnabubula's whole album, 45 minutes
// in one file, filed as both "Fading Light" and "Looking for Answers"): then
// all of them go. ok is false when the track list cannot say, and nothing
// should be removed.
//
// Several copies may fit when they carry the same title ("Killjoy" twice
// beside "Dreamland"): they are one song held twice, not a wrong slot, so they
// are neither kept nor lost here. Once the wrong slot is emptied, the next
// sweep finds the pair on its own, and the resolver merges it as any song held
// twice on one album.
func wrongSlots(copies []slotCopy, tracks []dbq.MbReleaseTrack) (keep int, losers []int, ok bool) {
	keep = -1
	keepKey := ""
	for i, c := range copies {
		key := MatchTitleKey(c.Title, c.Artist)
		fits, other, known := false, true, false
		for _, t := range tracks {
			if t.LengthMs <= 0 || MatchTitleKey(t.Title, c.Artist) != key {
				continue
			}
			known = true
			d := absMs(t.LengthMs - c.DurationMs)
			if d <= slotFit {
				fits = true
			}
			if d <= slotOther {
				other = false
			}
		}
		switch {
		case fits && keep >= 0:
			if key != keepKey {
				return -1, nil, false
			}
		case fits:
			keep, keepKey = i, key
		case known && other:
			losers = append(losers, i)
		default:
			return -1, nil, false
		}
	}
	// A copy that neither fits nor is clearly another song's has returned
	// above, so with no keep every copy is a loser.
	if len(losers) == 0 {
		return -1, nil, false
	}
	return keep, losers, true
}

func absMs(d int32) int32 {
	if d < 0 {
		return -d
	}
	return d
}

// dismissDistinct dismisses every group of distinct versions and records it
// for the Resolved automatically tab. It adds what it settled to settled.
func dismissDistinct(
	ctx context.Context, pool *pgxpool.Pool, logger *slog.Logger, groups []*resolveGroup, settled map[pgtype.UUID]bool,
) int {
	q := dbq.New(pool)
	n := 0
	for _, g := range groups {
		if g.class != ClassDistinct || settled[g.id] || ctx.Err() != nil {
			continue
		}
		rows, err := q.SettleDuplicateGroupAutomatically(ctx, dbq.SettleDuplicateGroupAutomaticallyParams{
			ID: g.id, Status: "dismissed",
		})
		if err != nil {
			logger.Warn("duplicate resolve: dismiss failed", "group_id", syncpkg.FormatUUID(g.id), "err", err)
			continue
		}
		if rows == 0 {
			continue
		}
		settled[g.id] = true
		n++
		titles := make([]string, len(g.members))
		for i, m := range g.members {
			titles[i] = m.Title
		}
		audit.WriteOrLog(ctx, pool, logger, pgtype.UUID{}, pgtype.UUID{}, audit.ActionDuplicateDismiss, map[string]any{
			"automatic": true,
			"group_id":  syncpkg.FormatUUID(g.id),
			"artist":    g.members[0].ArtistName,
			"titles":    titles,
		})
	}
	return n
}

// settleWrongSlots removes the copies that sit in another song's slot, for
// every one-album mismatch the track list, or AcoustID with it, decides. It
// returns how many copies went and the songs whose slots were emptied, and
// adds the groups it settled to settled.
func settleWrongSlots(
	ctx context.Context, pool *pgxpool.Pool, logger *slog.Logger, groups []*resolveGroup, settled map[pgtype.UUID]bool,
	src ResolveSources,
) (int, []string) {
	q := dbq.New(pool)
	tracksByGroup := map[string][]dbq.MbReleaseTrack{}
	fetches, identifies := trackListFetchesPerPass, identifiesPerPass
	if src.TrackLists == nil {
		fetches = 0
	}
	if src.Identify == nil {
		identifies = 0
	}
	removed := 0
	var names []string
	for _, g := range groups {
		if g.class != ClassMismatch || settled[g.id] || ctx.Err() != nil {
			continue
		}
		rg := g.members[0].ReleaseGroupMbid
		if rg == "" {
			continue
		}
		tracks, seen := tracksByGroup[rg]
		if !seen {
			var err error
			if tracks, err = q.ListMBReleaseTracksForGroup(ctx, rg); err != nil {
				logger.Warn("duplicate resolve: reading the album's track list failed", "release_group", rg, "err", err)
				continue
			}
			if len(tracks) == 0 && fetches > 0 && neverRead(ctx, q, rg) {
				fetches--
				if err := src.TrackLists(ctx, rg); err != nil {
					logger.Info("duplicate resolve: fetching the album's track list failed", "release_group", rg, "err", err)
					if errors.Is(err, musicbrainz.ErrUnreachable) {
						fetches = 0
					}
				} else if tracks, err = q.ListMBReleaseTracksForGroup(ctx, rg); err != nil {
					logger.Warn("duplicate resolve: reading the album's track list failed", "release_group", rg, "err", err)
					continue
				}
			}
			tracksByGroup[rg] = tracks
		}
		copies := make([]slotCopy, len(g.members))
		for i, m := range g.members {
			copies[i] = slotCopy{Title: m.Title, Artist: m.ArtistName, DurationMs: m.DurationMs}
		}
		keep, losers, ok := wrongSlots(copies, tracks)
		if !ok {
			recordings, known := groupRecordings(ctx, logger, g, src.Identify, &identifies)
			if !known {
				continue
			}
			if keep, losers, ok = namedCopies(copies, tracks, recordings); !ok {
				logger.Info("duplicate resolve: AcoustID's answer does not place the recording in one slot; left for review",
					"group_id", syncpkg.FormatUUID(g.id), "recordings", recordings)
			}
		}
		if !ok {
			continue
		}
		// Set aside before the files go: a pass that dies between the two
		// still does not fetch the same files again.
		blocked := blockReleasesHolding(ctx, pool, logger, g)
		emptied := emptiedSlots(ctx, q, logger, g, losers, tracks)
		n, err := removeWrongSlots(ctx, pool, logger, g, keep, losers, emptied, blocked)
		removed += n
		if err != nil {
			logger.Warn("duplicate resolve: removing wrong-slot copies failed", "group_id", syncpkg.FormatUUID(g.id), "err", err)
			continue
		}
		settled[g.id] = true
		for _, i := range losers {
			names = append(names, TrackLabel(g.members[i].ArtistName, g.members[i].Title))
		}
	}
	return removed, names
}

// sameAudio reports whether a group's copies are one recording, cut however:
// identical audio, or fingerprints all but identical (sameAudioBER).
func sameAudio(g *resolveGroup) bool {
	return g.tier == "exact" || (g.worstBER != nil && *g.worstBER <= sameAudioBER)
}

// namedCopies decides a one-album mismatch the lengths could not, from the
// recordings AcoustID says it may be. The track list names each slot's
// recording: the copy in a slot of one of them is kept and the others are
// losers. When none of the copies' slots is one of them, or AcoustID knows
// none (recordings empty), every copy is another song's and all go. ok is
// false when they fall in more than one copy's slot, the track list names no
// recording for a copy's slot, or the copies are all filed as one song.
//
// Copies sharing the kept copy's title are one song held twice, as in
// wrongSlots: neither kept nor lost here.
func namedCopies(copies []slotCopy, tracks []dbq.MbReleaseTrack, recordings []string) (keep int, losers []int, ok bool) {
	keys := make([]string, len(copies))
	theirs := map[string]bool{}
	for i, c := range copies {
		keys[i] = MatchTitleKey(c.Title, c.Artist)
		named := false
		for _, t := range tracks {
			if t.RecordingMbid == "" || MatchTitleKey(t.Title, c.Artist) != keys[i] {
				continue
			}
			named = true
			if slices.Contains(recordings, t.RecordingMbid) {
				theirs[keys[i]] = true
			}
		}
		if !named {
			return -1, nil, false
		}
	}
	if len(theirs) > 1 {
		return -1, nil, false
	}
	keep = -1
	for i, key := range keys {
		switch {
		case theirs[key] && keep < 0:
			keep = i
		case theirs[key]:
		default:
			losers = append(losers, i)
		}
	}
	if len(losers) == 0 {
		return -1, nil, false
	}
	return keep, losers, true
}

// groupRecordings is what AcoustID says a group's recording may be. Copies
// whose own lookups all name one recording are that recording, however their
// fingerprints compare. Otherwise only copies that are the same audio
// (sameAudio) are asked about: the recordings any copy's lookup matched, none
// when every copy's lookup found no match, or else a lookup of one copy now,
// while the pass has lookups left. known is false when there is no answer
// yet, and the group waits.
func groupRecordings(
	ctx context.Context, logger *slog.Logger, g *resolveGroup, identify RecordingIdentifier, left *int,
) (recordings []string, known bool) {
	matched, noMatch := 0, 0
	for _, m := range g.members {
		switch {
		case m.AcoustidState == nil:
		case *m.AcoustidState == lookupMatched && m.AcoustidRecording != nil:
			matched++
			if !slices.Contains(recordings, *m.AcoustidRecording) {
				recordings = append(recordings, *m.AcoustidRecording)
			}
		case *m.AcoustidState == lookupNoMatch:
			noMatch++
		}
	}
	if matched == len(g.members) && len(recordings) == 1 {
		return recordings, true
	}
	if !sameAudio(g) {
		return nil, false
	}
	if len(recordings) > 0 || noMatch == len(g.members) {
		return recordings, true
	}
	if identify == nil || *left <= 0 {
		return nil, false
	}
	*left--
	ids, err := identify(ctx, g.members[0].FilePath)
	if err != nil {
		if errors.Is(err, ErrIdentifyUnavailable) {
			*left = 0
		}
		logger.Info("duplicate resolve: asking AcoustID which song a recording is failed",
			"group_id", syncpkg.FormatUUID(g.id), "err", err)
		return nil, false
	}
	return ids, true
}

// blockReleasesHolding sets aside the downloads a group's files were imported
// from, and returns their names. An album imported before Minstrel downloaded
// anything has none.
func blockReleasesHolding(ctx context.Context, pool *pgxpool.Pool, logger *slog.Logger, g *resolveGroup) []string {
	q := dbq.New(pool)
	seen := map[string]bool{}
	var names []string
	for _, m := range g.members {
		grabs, err := q.ListImportedGrabsHolding(ctx, m.FilePath)
		if err != nil {
			logger.Warn("duplicate resolve: finding the download a file came from failed", "path", m.FilePath, "err", err)
			continue
		}
		for _, gr := range grabs {
			if seen[gr.InfoHash] {
				continue
			}
			seen[gr.InfoHash] = true
			if err := acquire.NewMemory(pool).Block(ctx, dbq.BlockReleaseParams{
				InfoHash: gr.InfoHash, Name: gr.Name, Reason: acquire.BlockFiles,
				Detail: "one recording fills several of the album's tracks.", WantID: gr.WantID,
			}); err != nil {
				logger.Warn("duplicate resolve: setting the release aside failed", "release", gr.Name, "err", err)
				continue
			}
			names = append(names, gr.Name)
		}
	}
	return names
}

// emptiedSlots is the losers whose song the album holds no other copy of:
// no present track on it under that title as long as the song's slot. Those
// wait for a copy fetched again (migration 0095). The rest are the song held
// twice, and missing-pair folding folds them into the copy held: Savant's
// VOID DLC "Weeper" beside Void's. A failed read counts every loser as
// emptied, which only makes the album wait for a copy it may not need.
func emptiedSlots(
	ctx context.Context, q *dbq.Queries, logger *slog.Logger, g *resolveGroup, losers []int, tracks []dbq.MbReleaseTrack,
) []pgtype.UUID {
	held, err := q.ListTracksByAlbum(ctx, dbq.ListTracksByAlbumParams{AlbumID: g.members[0].AlbumID})
	if err != nil {
		logger.Warn("duplicate resolve: reading the album's tracks failed", "group_id", syncpkg.FormatUUID(g.id), "err", err)
	}
	inGroup := map[pgtype.UUID]bool{}
	for _, m := range g.members {
		inGroup[m.TrackID] = true
	}
	var emptied []pgtype.UUID
	for _, i := range losers {
		m := g.members[i]
		key := MatchTitleKey(m.Title, m.ArtistName)
		covered := false
		for _, h := range held {
			if covered || h.MissingSince.Valid || inGroup[h.ID] || MatchTitleKey(h.Title, m.ArtistName) != key {
				continue
			}
			for _, t := range tracks {
				if t.LengthMs > 0 && MatchTitleKey(t.Title, m.ArtistName) == key && absMs(t.LengthMs-h.DurationMs) <= slotFit {
					covered = true
					break
				}
			}
		}
		if !covered {
			emptied = append(emptied, m.TrackID)
		}
	}
	return emptied
}

// neverRead reports whether the catalogue has never read a release group's
// releases. A group read and found empty is not asked about again: the
// catalogue's own refresh revisits it.
func neverRead(ctx context.Context, q *dbq.Queries, rg string) bool {
	g, err := q.GetMBReleaseGroup(ctx, rg)
	return errors.Is(err, pgx.ErrNoRows) || (err == nil && !g.ReleasesRefreshedAt.Valid)
}

// removeWrongSlots deletes each loser's file, then marks its track missing in
// the same transaction that closes the group. The row is kept, with its likes
// and plays: it is the slot, and a good copy fetched again takes it over. Its
// recording MBID is dropped, being the wrong file's, so missing-pair folding
// does not fold the slot into the copy that was kept. The slots in emptied
// are stamped to wait for a copy fetched again (emptiedSlots). blocked names
// the releases set aside, for the record. A file that cannot be removed stops
// the group; the ones already gone are still marked, and the group waits for
// the next pass.
func removeWrongSlots(
	ctx context.Context, pool *pgxpool.Pool, logger *slog.Logger, g *resolveGroup, keep int, losers []int,
	emptied []pgtype.UUID, blocked []string,
) (int, error) {
	var gone []pgtype.UUID
	var removedFiles []map[string]string
	var removeErr error
	for _, i := range losers {
		m := g.members[i]
		if err := removeFile(m.FilePath); err != nil && !errors.Is(err, os.ErrNotExist) {
			removeErr = fmt.Errorf("remove %s: %w", m.FilePath, err)
			break
		}
		gone = append(gone, m.TrackID)
		removedFiles = append(removedFiles, map[string]string{
			"track_id": syncpkg.FormatUUID(m.TrackID), "file_path": m.FilePath, "title": m.Title,
		})
	}
	if len(gone) == 0 {
		return 0, removeErr
	}
	err := pgx.BeginFunc(ctx, pool, func(tx pgx.Tx) error {
		tq := dbq.New(tx)
		if _, err := tq.MarkTracksMissing(ctx, gone); err != nil {
			return fmt.Errorf("mark missing: %w", err)
		}
		if err := tq.ClearTrackMbids(ctx, gone); err != nil {
			return fmt.Errorf("clear mbids: %w", err)
		}
		if err := tq.MarkTracksEmptied(ctx, emptied); err != nil {
			return fmt.Errorf("mark emptied: %w", err)
		}
		if err := logTrackChanges(ctx, tq, gone); err != nil {
			return fmt.Errorf("log changes: %w", err)
		}
		if removeErr != nil {
			return nil
		}
		_, err := tq.SettleDuplicateGroupAutomatically(ctx, dbq.SettleDuplicateGroupAutomaticallyParams{
			ID: g.id, Status: "merged",
		})
		return err
	})
	if err != nil {
		return 0, err
	}
	details := map[string]any{
		"automatic": true,
		"group_id":  syncpkg.FormatUUID(g.id),
		"artist":    g.members[0].ArtistName,
		"removed":   removedFiles,
	}
	if len(blocked) > 0 {
		details["blocked"] = blocked
	}
	keptPath := ""
	if keep >= 0 {
		kept := g.members[keep]
		details["kept_title"], details["kept_path"] = kept.Title, kept.FilePath
		keptPath = kept.FilePath
	}
	audit.WriteOrLog(ctx, pool, logger, pgtype.UUID{}, pgtype.UUID{}, audit.ActionDuplicateWrongFile, details)
	logger.Info("duplicate resolve: removed copies from slots that were not theirs",
		"group_id", syncpkg.FormatUUID(g.id), "kept", keptPath, "removed", len(gone))
	return len(gone), removeErr
}

// notifyNewForReview tells admins about groups this pass classified for the
// first time and left for them. A group the resolver settled, linked or will
// link is not news: telling admins about those was most of what the notice
// used to say.
func notifyNewForReview(ctx context.Context, groups []*resolveGroup, settled map[pgtype.UUID]bool) {
	var count int64
	var items []string
	for _, g := range groups {
		if g.members[0].PriorClass != nil || settled[g.id] {
			continue
		}
		if g.class != ClassReview && g.class != ClassMismatch {
			continue
		}
		count++
		if len(items) < notifyItems {
			items = append(items, TrackLabel(g.members[0].ArtistName, g.members[0].Title))
		}
	}
	if count > 0 {
		notifyAdmins(ctx, notifications.KindDuplicatesFound, notifications.Payload{Count: count, Items: items})
	}
}