// Package importer checks a finished download against the album it was // grabbed for and files it into the library (M502 #5522). Nothing reaches // the library before the check says the files are the album: a wrong album, // another version of it, a video rip or an incomplete copy is refused, its // release is blocked and the album is searched for again. // // Ported from Fabled Librarian's importer and relocate packages, with the // check rewritten for music: the files are matched to one of the album's // MusicBrainz editions by AcoustID fingerprints, track lengths, titles and // the tags they carry. package importer import ( "fmt" "slices" "sort" "strings" "github.com/jackc/pgx/v5/pgtype" "git.fabledsword.com/bvandeusen/minstrel/internal/release" ) // Outcome is what the check decided. type Outcome string const ( Accept Outcome = "accept" // Review: the files may be the album and the evidence cannot say. A // person decides. Review Outcome = "review" Refuse Outcome = "refuse" ) // Codes say why, beside the sentence in Verdict.Reason. const ( CodeMatched = "matched" CodeForced = "forced" CodeNoAudio = "no_audio" CodeVideo = "video" CodeWrongAlbum = "wrong_album" CodeOtherVersion = release.VetoVersion CodeWrongTitle = release.VetoTitle CodeIncomplete = "incomplete" CodeNoEdition = "no_edition" CodeWeak = "weak" CodeTooMany = "too_many" ) // maxFiles is the most audio files one album is checked for. More is a box // set or a discography, which is a person's call, not a fingerprint queue. const maxFiles = 60 // File is what one audio file in a download says about itself. type File struct { Path string // absolute, as Minstrel sees it Rel string // under the download, for the person reading the verdict // Disc and Track are the tags' positions; 0 when untagged. Disc, Track int Title string Artist string AlbumArtist string Album string // MusicBrainz ids from the tags, when a tagger wrote them. GroupMBID string ReleaseMBID string RecordingMBID string LengthMs int // Recordings are the MusicBrainz recordings AcoustID says the audio is, // at or above the operator's minimum score. Recordings []string // Video: the file name says it is a music video, a lyric video or the // like (library.SourceMarkersFor). Video bool } // Track is one track of an edition. type Track struct { Medium int Format string // the medium's: CD, Digital Media, Vinyl Position int Title string RecordingMBID string LengthMs int } // Edition is one MusicBrainz release of the wanted release group, its // tracks in medium and position order. type Edition struct { MBID string Title string Status string Date string Country string Tracks []Track } // Media is how many media the edition has. func (e Edition) Media() int { n := 0 for _, t := range e.Tracks { n = max(n, t.Medium) } return n } // Want is the album the download was grabbed for. type Want struct { GroupMBID string ArtistMBID string Artist string Title string Year int PrimaryType string SecondaryTypes []string // Upgrade: a held album wanted for a better copy (#5524), quality or // rip; AlbumID is the library album it holds, whose tracks the import // replaces. Upgrade string AlbumID pgtype.UUID } // Pair is a file matched to an edition's track, and what agreed. type Pair struct { File int `json:"file"` Track int `json:"track"` Recording bool `json:"recording"` Length bool `json:"length"` Title bool `json:"title"` } // Evidence is what the check counted, kept with the verdict so a person can // see what it rested on. type Evidence struct { Files int `json:"files"` Tracks int `json:"tracks"` // Identified: files with a recording from AcoustID or their tags. Identified int `json:"identified"` Recordings int `json:"recordings"` Lengths int `json:"lengths"` Titles int `json:"titles"` // GroupTag: "same" when the tags name this release group, "other" when // another, "" when they name none. GroupTag string `json:"group_tag,omitempty"` AlbumTag string `json:"album_tag,omitempty"` // Fingerprinted: AcoustID was asked. False when it is not set up or // could not be reached; the check then rests on lengths and tags. Fingerprinted bool `json:"fingerprinted"` } // Verdict is the check's answer. type Verdict struct { Outcome Outcome `json:"outcome"` Code string `json:"code"` Reason string `json:"reason"` // Edition indexes the editions passed in; -1 when none was chosen. Edition int `json:"-"` Pairs []Pair `json:"pairs,omitempty"` Evidence Evidence `json:"evidence"` } // Verify decides whether files are the wanted album and, if so, which // edition and which track each file is. force is a person's "import anyway": // the best edition is taken whatever the evidence, but a download with no // audio is still refused. func Verify(w Want, editions []Edition, files []File, fingerprinted, force bool) Verdict { v := Verdict{Edition: -1, Evidence: Evidence{Files: len(files), Fingerprinted: fingerprinted}} if len(files) == 0 { return v.set(Refuse, CodeNoAudio, "The download holds no audio files.") } if !force && len(files) > maxFiles { return v.set(Review, CodeTooMany, fmt.Sprintf( "The download holds %d audio files, more than one album; a person decides.", len(files))) } if video := countIf(files, func(f File) bool { return f.Video }); !force && video*2 > len(files) { return v.set(Refuse, CodeVideo, fmt.Sprintf( "%d of its %d audio files are named as music or lyric videos: the sound of a video, not the album.", video, len(files))) } group := majority(files, func(f File) string { return f.GroupMBID }) switch { case group == "": case strings.EqualFold(group, w.GroupMBID): v.Evidence.GroupTag = "same" default: v.Evidence.GroupTag = "other" } album := majority(files, func(f File) string { return f.Album }) v.Evidence.AlbumTag = album var tagVeto release.Match albumAgrees := false if album != "" { // The want's own artist goes in front: the tags' artist spelling is // not what this asks. Only the album title and its version words. m := release.Judge(release.Want{Artist: w.Artist, Title: w.Title, PrimaryType: w.PrimaryType, SecondaryTypes: w.SecondaryTypes}, release.Parse(w.Artist+" - "+album)) switch m.Veto { case release.VetoVersion, release.VetoTitle: tagVeto = m case "": albumAgrees = true } } if !force && v.Evidence.GroupTag == "other" { return v.set(Refuse, CodeWrongAlbum, fmt.Sprintf( "Its tags name a different MusicBrainz release group (%s), not %s.", group, w.Title)) } if len(editions) == 0 { if force { v.Code, v.Outcome, v.Reason = CodeForced, Accept, "Imported anyway at a person's word; MusicBrainz lists no edition to match it to." return v } return v.set(Review, CodeNoEdition, fmt.Sprintf( "MusicBrainz lists no edition of %s to check the files against; a person decides.", w.Title)) } best, bestScore := -1, -1 shortest, longest := len(editions[0].Tracks), 0 for i, e := range editions { shortest, longest = min(shortest, len(e.Tracks)), max(longest, len(e.Tracks)) if len(e.Tracks) != len(files) && !force { continue } pairs := pairUp(files, e.Tracks) if s := score(pairs); s > bestScore || (s == bestScore && better(e, editions[best])) { best, bestScore = i, s } } if best < 0 { if len(files) < shortest { return v.set(Refuse, CodeIncomplete, fmt.Sprintf( "It holds %d audio files; the shortest edition of %s has %d tracks.", len(files), w.Title, shortest)) } return v.set(Review, CodeNoEdition, fmt.Sprintf( "It holds %d audio files; no edition of %s has %d tracks (they have %s). A person decides.", len(files), w.Title, len(files), span(shortest, longest))) } e := editions[best] v.Edition = best v.Pairs = pairUp(files, e.Tracks) n := len(e.Tracks) ev := &v.Evidence ev.Tracks = n for _, p := range v.Pairs { f := files[p.File] if f.RecordingMBID != "" || len(f.Recordings) > 0 { ev.Identified++ } ev.Recordings += b2i(p.Recording) ev.Lengths += b2i(p.Length) ev.Titles += b2i(p.Title) } if force { v.Outcome, v.Code = Accept, CodeForced v.Reason = fmt.Sprintf("Imported anyway at a person's word, as %s. %s", describe(e), ev.sentence()) return v } half := (n + 1) / 2 strong := ev.GroupTag == "same" || (ev.Identified >= half && ev.Recordings*10 >= ev.Identified*7) switch { case ev.Identified >= half && ev.Recordings*10 < ev.Identified*3 && ev.GroupTag != "same": return v.set(Refuse, CodeWrongAlbum, fmt.Sprintf( "Only %d of the %d tracks AcoustID knows are recordings on %s.", ev.Recordings, ev.Identified, w.Title)) case tagVeto.Veto != "" && !strong: return v.set(Refuse, tagVeto.Veto, fmt.Sprintf("Its tags call it %q. %s", album, tagVeto.Reason)) case !strong && ev.Identified == 0 && ev.Lengths*2 < n: return v.set(Refuse, CodeWrongAlbum, fmt.Sprintf( "Only %d of its %d tracks are the length of %s's, and nothing else says it is the album.", ev.Lengths, n, w.Title)) case strong && ev.Lengths*10 >= n*6, !strong && ev.Lengths*10 >= n*8 && (ev.Titles*10 >= n*6 || albumAgrees): v.Outcome, v.Code = Accept, CodeMatched v.Reason = fmt.Sprintf("It is %s. %s", describe(e), ev.sentence()) return v } return v.set(Review, CodeWeak, fmt.Sprintf( "It may be %s, but the evidence is thin: %s A person decides.", describe(e), ev.sentence())) } func (v Verdict) set(o Outcome, code, reason string) Verdict { v.Outcome, v.Code, v.Reason = o, code, reason return v } // sentence says what agreed, for the verdict's reason. func (ev Evidence) sentence() string { var parts []string if ev.GroupTag == "same" { parts = append(parts, "its tags name this album") } if ev.Identified > 0 { parts = append(parts, fmt.Sprintf("%d of %d identified tracks are its recordings", ev.Recordings, ev.Identified)) } parts = append(parts, fmt.Sprintf("%d of %d lengths fit", ev.Lengths, ev.Tracks), fmt.Sprintf("%d of %d titles agree", ev.Titles, ev.Tracks)) s := strings.Join(parts, ", ") s = strings.ToUpper(s[:1]) + s[1:] + "." if !ev.Fingerprinted { s += " AcoustID was not asked (not set up or unreachable), so lengths and tags decided." } return s } // describe names an edition for a person: "the 2013 CD release (13 tracks)". func describe(e Edition) string { var b strings.Builder b.WriteString("the ") if len(e.Date) >= 4 { b.WriteString(e.Date[:4] + " ") } if f := e.format(); f != "" { b.WriteString(f + " ") } fmt.Fprintf(&b, "edition (%d tracks)", len(e.Tracks)) return b.String() } func (e Edition) format() string { if len(e.Tracks) == 0 { return "" } f := e.Tracks[0].Format for _, t := range e.Tracks { if t.Format != f { return "" } } return f } // pairUp matches files to tracks in order: by the tags' disc and track // numbers when every file has them and no two share a place, by file path // otherwise. With a person's "import anyway" the counts may differ; pairs // run to the shorter. func pairUp(files []File, tracks []Track) []Pair { order := make([]int, len(files)) for i := range order { order[i] = i } if numbered(files) { sort.SliceStable(order, func(a, b int) bool { fa, fb := files[order[a]], files[order[b]] if da, db := max(fa.Disc, 1), max(fb.Disc, 1); da != db { return da < db } return fa.Track < fb.Track }) } else { sort.SliceStable(order, func(a, b int) bool { return files[order[a]].Rel < files[order[b]].Rel }) } n := min(len(files), len(tracks)) pairs := make([]Pair, n) for i := range n { f, t := files[order[i]], tracks[i] pairs[i] = Pair{ File: order[i], Track: i, Recording: t.RecordingMBID != "" && (f.RecordingMBID == t.RecordingMBID || slices.Contains(f.Recordings, t.RecordingMBID)), Length: release.LengthFits(f.LengthMs, t.LengthMs), Title: release.TitleAgrees(f.Title, t.Title), } } return pairs } // numbered: every file carries a track number and no two share a disc and // track. func numbered(files []File) bool { seen := map[[2]int]bool{} for _, f := range files { k := [2]int{max(f.Disc, 1), f.Track} if f.Track <= 0 || seen[k] { return false } seen[k] = true } return true } // score weighs recordings above lengths above titles: a fingerprint is the // audio itself, a length is the audio's shape, a title is only a name. func score(pairs []Pair) int { s := 0 for _, p := range pairs { s += 3*b2i(p.Recording) + 2*b2i(p.Length) + b2i(p.Title) } return s } // better breaks a tie between equally good editions: an official one, then // the earlier. func better(a, b Edition) bool { if (a.Status == "Official") != (b.Status == "Official") { return a.Status == "Official" } return a.Date != "" && (b.Date == "" || a.Date < b.Date) } func majority(files []File, get func(File) string) string { counts := map[string]int{} best, n := "", 0 for _, f := range files { v := strings.TrimSpace(get(f)) if v == "" { continue } counts[strings.ToLower(v)]++ if c := counts[strings.ToLower(v)]; c > n { best, n = v, c } } return best } func countIf(files []File, f func(File) bool) int { n := 0 for _, x := range files { if f(x) { n++ } } return n } func span(a, b int) string { if a == b { return fmt.Sprint(a) } return fmt.Sprintf("%d to %d", a, b) } func b2i(b bool) int { if b { return 1 } return 0 }