minstrel/internal/importer/verify.go

// Package importer checks a finished download against the album it was
// grabbed for and files it into the library (M502 #5522). Nothing reaches
// the library before the check says the files are the album: a wrong album,
// another version of it, a video rip or an incomplete copy is refused, its
// release is blocked and the album is searched for again.
//
// Ported from Fabled Librarian's importer and relocate packages, with the
// check rewritten for music: the files are matched to one of the album's
// MusicBrainz editions by AcoustID fingerprints, track lengths, titles and
// the tags they carry.
package importer

import (
	"fmt"
	"slices"
	"sort"
	"strings"

	"git.fabledsword.com/bvandeusen/minstrel/internal/release"
)

// Outcome is what the check decided.
type Outcome string

const (
	Accept Outcome = "accept"
	// Review: the files may be the album and the evidence cannot say. A
	// person decides.
	Review Outcome = "review"
	Refuse Outcome = "refuse"
)

// Codes say why, beside the sentence in Verdict.Reason.
const (
	CodeMatched      = "matched"
	CodeForced       = "forced"
	CodeNoAudio      = "no_audio"
	CodeVideo        = "video"
	CodeWrongAlbum   = "wrong_album"
	CodeOtherVersion = release.VetoVersion
	CodeWrongTitle   = release.VetoTitle
	CodeIncomplete   = "incomplete"
	CodeNoEdition    = "no_edition"
	CodeWeak         = "weak"
	CodeTooMany      = "too_many"
)

// maxFiles is the most audio files one album is checked for. More is a box
// set or a discography, which is a person's call, not a fingerprint queue.
const maxFiles = 60

// File is what one audio file in a download says about itself.
type File struct {
	Path string // absolute, as Minstrel sees it
	Rel  string // under the download, for the person reading the verdict
	// Disc and Track are the tags' positions; 0 when untagged.
	Disc, Track int
	Title       string
	Artist      string
	AlbumArtist string
	Album       string
	// MusicBrainz ids from the tags, when a tagger wrote them.
	GroupMBID     string
	ReleaseMBID   string
	RecordingMBID string
	LengthMs      int
	// Recordings are the MusicBrainz recordings AcoustID says the audio is,
	// at or above the operator's minimum score.
	Recordings []string
	// Video: the file name says it is a music video, a lyric video or the
	// like (library.SourceMarkersFor).
	Video bool
}

// Track is one track of an edition.
type Track struct {
	Medium        int
	Format        string // the medium's: CD, Digital Media, Vinyl
	Position      int
	Title         string
	RecordingMBID string
	LengthMs      int
}

// Edition is one MusicBrainz release of the wanted release group, its
// tracks in medium and position order.
type Edition struct {
	MBID    string
	Title   string
	Status  string
	Date    string
	Country string
	Tracks  []Track
}

// Media is how many media the edition has.
func (e Edition) Media() int {
	n := 0
	for _, t := range e.Tracks {
		n = max(n, t.Medium)
	}
	return n
}

// Want is the album the download was grabbed for.
type Want struct {
	GroupMBID      string
	ArtistMBID     string
	Artist         string
	Title          string
	Year           int
	PrimaryType    string
	SecondaryTypes []string
}

// Pair is a file matched to an edition's track, and what agreed.
type Pair struct {
	File      int  `json:"file"`
	Track     int  `json:"track"`
	Recording bool `json:"recording"`
	Length    bool `json:"length"`
	Title     bool `json:"title"`
}

// Evidence is what the check counted, kept with the verdict so a person can
// see what it rested on.
type Evidence struct {
	Files  int `json:"files"`
	Tracks int `json:"tracks"`
	// Identified: files with a recording from AcoustID or their tags.
	Identified int `json:"identified"`
	Recordings int `json:"recordings"`
	Lengths    int `json:"lengths"`
	Titles     int `json:"titles"`
	// GroupTag: "same" when the tags name this release group, "other" when
	// another, "" when they name none.
	GroupTag string `json:"group_tag,omitempty"`
	AlbumTag string `json:"album_tag,omitempty"`
	// Fingerprinted: AcoustID was asked. False when it is not set up or
	// could not be reached; the check then rests on lengths and tags.
	Fingerprinted bool `json:"fingerprinted"`
}

// Verdict is the check's answer.
type Verdict struct {
	Outcome Outcome `json:"outcome"`
	Code    string  `json:"code"`
	Reason  string  `json:"reason"`
	// Edition indexes the editions passed in; -1 when none was chosen.
	Edition  int      `json:"-"`
	Pairs    []Pair   `json:"pairs,omitempty"`
	Evidence Evidence `json:"evidence"`
}

// Verify decides whether files are the wanted album and, if so, which
// edition and which track each file is. force is a person's "import anyway":
// the best edition is taken whatever the evidence, but a download with no
// audio is still refused.
func Verify(w Want, editions []Edition, files []File, fingerprinted, force bool) Verdict {
	v := Verdict{Edition: -1, Evidence: Evidence{Files: len(files), Fingerprinted: fingerprinted}}
	if len(files) == 0 {
		return v.set(Refuse, CodeNoAudio, "The download holds no audio files.")
	}
	if !force && len(files) > maxFiles {
		return v.set(Review, CodeTooMany, fmt.Sprintf(
			"The download holds %d audio files, more than one album; a person decides.", len(files)))
	}
	if video := countIf(files, func(f File) bool { return f.Video }); !force && video*2 > len(files) {
		return v.set(Refuse, CodeVideo, fmt.Sprintf(
			"%d of its %d audio files are named as music or lyric videos: the sound of a video, not the album.", video, len(files)))
	}

	group := majority(files, func(f File) string { return f.GroupMBID })
	switch {
	case group == "":
	case strings.EqualFold(group, w.GroupMBID):
		v.Evidence.GroupTag = "same"
	default:
		v.Evidence.GroupTag = "other"
	}
	album := majority(files, func(f File) string { return f.Album })
	v.Evidence.AlbumTag = album
	var tagVeto release.Match
	albumAgrees := false
	if album != "" {
		// The want's own artist goes in front: the tags' artist spelling is
		// not what this asks. Only the album title and its version words.
		m := release.Judge(release.Want{Artist: w.Artist, Title: w.Title, PrimaryType: w.PrimaryType,
			SecondaryTypes: w.SecondaryTypes}, release.Parse(w.Artist+" - "+album))
		switch m.Veto {
		case release.VetoVersion, release.VetoTitle:
			tagVeto = m
		case "":
			albumAgrees = true
		}
	}
	if !force && v.Evidence.GroupTag == "other" {
		return v.set(Refuse, CodeWrongAlbum, fmt.Sprintf(
			"Its tags name a different MusicBrainz release group (%s), not %s.", group, w.Title))
	}

	if len(editions) == 0 {
		if force {
			v.Code, v.Outcome, v.Reason = CodeForced, Accept, "Imported anyway at a person's word; MusicBrainz lists no edition to match it to."
			return v
		}
		return v.set(Review, CodeNoEdition, fmt.Sprintf(
			"MusicBrainz lists no edition of %s to check the files against; a person decides.", w.Title))
	}

	best, bestScore := -1, -1
	shortest, longest := len(editions[0].Tracks), 0
	for i, e := range editions {
		shortest, longest = min(shortest, len(e.Tracks)), max(longest, len(e.Tracks))
		if len(e.Tracks) != len(files) && !force {
			continue
		}
		pairs := pairUp(files, e.Tracks)
		if s := score(pairs); s > bestScore || (s == bestScore && better(e, editions[best])) {
			best, bestScore = i, s
		}
	}
	if best < 0 {
		if len(files) < shortest {
			return v.set(Refuse, CodeIncomplete, fmt.Sprintf(
				"It holds %d audio files; the shortest edition of %s has %d tracks.", len(files), w.Title, shortest))
		}
		return v.set(Review, CodeNoEdition, fmt.Sprintf(
			"It holds %d audio files; no edition of %s has %d tracks (they have %s). A person decides.",
			len(files), w.Title, len(files), span(shortest, longest)))
	}

	e := editions[best]
	v.Edition = best
	v.Pairs = pairUp(files, e.Tracks)
	n := len(e.Tracks)
	ev := &v.Evidence
	ev.Tracks = n
	for _, p := range v.Pairs {
		f := files[p.File]
		if f.RecordingMBID != "" || len(f.Recordings) > 0 {
			ev.Identified++
		}
		ev.Recordings += b2i(p.Recording)
		ev.Lengths += b2i(p.Length)
		ev.Titles += b2i(p.Title)
	}
	if force {
		v.Outcome, v.Code = Accept, CodeForced
		v.Reason = fmt.Sprintf("Imported anyway at a person's word, as %s. %s", describe(e), ev.sentence())
		return v
	}

	half := (n + 1) / 2
	strong := ev.GroupTag == "same" || (ev.Identified >= half && ev.Recordings*10 >= ev.Identified*7)
	switch {
	case ev.Identified >= half && ev.Recordings*10 < ev.Identified*3 && ev.GroupTag != "same":
		return v.set(Refuse, CodeWrongAlbum, fmt.Sprintf(
			"Only %d of the %d tracks AcoustID knows are recordings on %s.", ev.Recordings, ev.Identified, w.Title))
	case tagVeto.Veto != "" && !strong:
		return v.set(Refuse, tagVeto.Veto, fmt.Sprintf("Its tags call it %q. %s", album, tagVeto.Reason))
	case !strong && ev.Identified == 0 && ev.Lengths*2 < n:
		return v.set(Refuse, CodeWrongAlbum, fmt.Sprintf(
			"Only %d of its %d tracks are the length of %s's, and nothing else says it is the album.", ev.Lengths, n, w.Title))
	case strong && ev.Lengths*10 >= n*6,
		!strong && ev.Lengths*10 >= n*8 && (ev.Titles*10 >= n*6 || albumAgrees):
		v.Outcome, v.Code = Accept, CodeMatched
		v.Reason = fmt.Sprintf("It is %s. %s", describe(e), ev.sentence())
		return v
	}
	return v.set(Review, CodeWeak, fmt.Sprintf(
		"It may be %s, but the evidence is thin: %s A person decides.", describe(e), ev.sentence()))
}

func (v Verdict) set(o Outcome, code, reason string) Verdict {
	v.Outcome, v.Code, v.Reason = o, code, reason
	return v
}

// sentence says what agreed, for the verdict's reason.
func (ev Evidence) sentence() string {
	var parts []string
	if ev.GroupTag == "same" {
		parts = append(parts, "its tags name this album")
	}
	if ev.Identified > 0 {
		parts = append(parts, fmt.Sprintf("%d of %d identified tracks are its recordings", ev.Recordings, ev.Identified))
	}
	parts = append(parts, fmt.Sprintf("%d of %d lengths fit", ev.Lengths, ev.Tracks),
		fmt.Sprintf("%d of %d titles agree", ev.Titles, ev.Tracks))
	s := strings.Join(parts, ", ")
	s = strings.ToUpper(s[:1]) + s[1:] + "."
	if !ev.Fingerprinted {
		s += " AcoustID was not asked (not set up or unreachable), so lengths and tags decided."
	}
	return s
}

// describe names an edition for a person: "the 2013 CD release (13 tracks)".
func describe(e Edition) string {
	var b strings.Builder
	b.WriteString("the ")
	if len(e.Date) >= 4 {
		b.WriteString(e.Date[:4] + " ")
	}
	if f := e.format(); f != "" {
		b.WriteString(f + " ")
	}
	fmt.Fprintf(&b, "edition (%d tracks)", len(e.Tracks))
	return b.String()
}

func (e Edition) format() string {
	if len(e.Tracks) == 0 {
		return ""
	}
	f := e.Tracks[0].Format
	for _, t := range e.Tracks {
		if t.Format != f {
			return ""
		}
	}
	return f
}

// pairUp matches files to tracks in order: by the tags' disc and track
// numbers when every file has them and no two share a place, by file path
// otherwise. With a person's "import anyway" the counts may differ; pairs
// run to the shorter.
func pairUp(files []File, tracks []Track) []Pair {
	order := make([]int, len(files))
	for i := range order {
		order[i] = i
	}
	if numbered(files) {
		sort.SliceStable(order, func(a, b int) bool {
			fa, fb := files[order[a]], files[order[b]]
			if da, db := max(fa.Disc, 1), max(fb.Disc, 1); da != db {
				return da < db
			}
			return fa.Track < fb.Track
		})
	} else {
		sort.SliceStable(order, func(a, b int) bool { return files[order[a]].Rel < files[order[b]].Rel })
	}
	n := min(len(files), len(tracks))
	pairs := make([]Pair, n)
	for i := range n {
		f, t := files[order[i]], tracks[i]
		pairs[i] = Pair{
			File:      order[i],
			Track:     i,
			Recording: t.RecordingMBID != "" && (f.RecordingMBID == t.RecordingMBID || slices.Contains(f.Recordings, t.RecordingMBID)),
			Length:    lengthFits(f.LengthMs, t.LengthMs),
			Title:     titleAgrees(f.Title, t.Title),
		}
	}
	return pairs
}

// numbered: every file carries a track number and no two share a disc and
// track.
func numbered(files []File) bool {
	seen := map[[2]int]bool{}
	for _, f := range files {
		k := [2]int{max(f.Disc, 1), f.Track}
		if f.Track <= 0 || seen[k] {
			return false
		}
		seen[k] = true
	}
	return true
}

// lengthFits allows five seconds or 3%, whichever is more: remasters and
// rips differ by a second or two, a different recording by far more. A
// track MusicBrainz has no length for says nothing against the file.
func lengthFits(fileMs, trackMs int) bool {
	if trackMs <= 0 {
		return true
	}
	if fileMs <= 0 {
		return false
	}
	tol := max(5000, trackMs*3/100)
	d := fileMs - trackMs
	return d <= tol && -d <= tol
}

// titleAgrees: the same words, or one title holding all of the other's
// ("Get Lucky (Radio Edit)" holds "Get Lucky").
func titleAgrees(a, b string) bool {
	ta, tb := release.Tokens(a), release.Tokens(b)
	if len(ta) == 0 || len(tb) == 0 {
		return false
	}
	return subset(ta, tb) || subset(tb, ta)
}

func subset(small, big []string) bool {
	for _, w := range small {
		if !slices.Contains(big, w) {
			return false
		}
	}
	return true
}

// score weighs recordings above lengths above titles: a fingerprint is the
// audio itself, a length is the audio's shape, a title is only a name.
func score(pairs []Pair) int {
	s := 0
	for _, p := range pairs {
		s += 3*b2i(p.Recording) + 2*b2i(p.Length) + b2i(p.Title)
	}
	return s
}

// better breaks a tie between equally good editions: an official one, then
// the earlier.
func better(a, b Edition) bool {
	if (a.Status == "Official") != (b.Status == "Official") {
		return a.Status == "Official"
	}
	return a.Date != "" && (b.Date == "" || a.Date < b.Date)
}

func majority(files []File, get func(File) string) string {
	counts := map[string]int{}
	best, n := "", 0
	for _, f := range files {
		v := strings.TrimSpace(get(f))
		if v == "" {
			continue
		}
		counts[strings.ToLower(v)]++
		if c := counts[strings.ToLower(v)]; c > n {
			best, n = v, c
		}
	}
	return best
}

func countIf(files []File, f func(File) bool) int {
	n := 0
	for _, x := range files {
		if f(x) {
			n++
		}
	}
	return n
}

func span(a, b int) string {
	if a == b {
		return fmt.Sprint(a)
	}
	return fmt.Sprintf("%d to %d", a, b)
}

func b2i(b bool) int {
	if b {
		return 1
	}
	return 0
}