fix(scanner): read multi-value genre frames correctly — #2499
dhowden/tag's readTFrame splits ID3v2 null-separated multi-value text frames and rejoins them with the EMPTY string, so a file tagged "Alternative Rock" + "Rock" was stored as "Alternative RockRock". It also leaves bare numeric ID3v1 references unresolved, which is why the library showed genres like "4017" and "526617". This corrupted more than the browse axis added in #367: taste_profile.sql reads tracks.genre directly, so the welded tokens were entering the taste profile's tag vocabulary, and recommendation.sql/discover.sql were comparing them as single opaque tags. Genre counts were wrong everywhere. ffprobe is not a fix — ffmpeg's read_ttag calls decode_str once with no loop, keeping only the first value. Truncating multi-genre tags would blunt the similarity signal genre mainly feeds. So the TCON frame is now parsed directly (ID3v2.2/2.3/2.4, all four text encodings, per-frame and tag-level unsynchronisation, numeric and parenthesised ID3v1 references); everything else still comes from dhowden/tag. Values are stored ";"-delimited, which the read side already splits on, so no query changes. Existing rows are repaired without an operator-run rebuild: migration 0054 adds tracks.tag_read_version DEFAULT 0, below the scanner's current tagReadVersion, so the next scan re-reads tags it would otherwise skip on mtime. Such a re-read reuses the stored duration instead of re-running ffprobe, keeping a repair pass tag-read-bound rather than one fork+exec per file. Bumping the constant is how a future extraction fix reaches an existing library. Only ID3v2 is in scope — dhowden welds nowhere else. The Vorbis/MP4 repeated-field question is #2500, unproven and deliberately not built.
This commit is contained in:
@@ -0,0 +1,179 @@
|
||||
package library
|
||||
|
||||
import (
|
||||
"io"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/dhowden/tag"
|
||||
)
|
||||
|
||||
// genreDelimiter is what we join multi-value genres with on the way into
|
||||
// tracks.genre. It has to be one of the characters the read side already splits
|
||||
// on — internal/taste and internal/recommendation both split on [;,], as do
|
||||
// browse.sql, recommendation.sql and discover.sql. Storing values joined with
|
||||
// ";" means the entire fix lands in the scanner and no query changes.
|
||||
const genreDelimiter = ";"
|
||||
|
||||
// extractGenres returns the genre values for a file, normalised and
|
||||
// deduplicated, ready to be joined with genreDelimiter.
|
||||
//
|
||||
// fellBack reports that an ID3v2 file's genre frame could not be parsed and the
|
||||
// value came from dhowden/tag instead. That path yields the old welded string,
|
||||
// so it is worth logging — but it is still the best available answer, and
|
||||
// degrading to it beats storing no genre at all.
|
||||
func extractGenres(meta tag.Metadata, rs io.ReadSeeker) (genres []string, fellBack bool) {
|
||||
switch meta.Format() {
|
||||
case tag.ID3v2_2, tag.ID3v2_3, tag.ID3v2_4:
|
||||
values, err := readID3v2GenreValues(rs)
|
||||
if err == nil {
|
||||
return normaliseGenres(values), false
|
||||
}
|
||||
// No frame at all is the common case for untagged files, and
|
||||
// dhowden/tag will have nothing either — not worth flagging.
|
||||
fellBack = meta.Genre() != ""
|
||||
default:
|
||||
// Vorbis comments (FLAC/OGG/Opus) and MP4 atoms don't go through
|
||||
// dhowden's welding path, so its value is already a faithful read of
|
||||
// the primary genre. Multi-value handling for those containers is a
|
||||
// separate, unproven concern — see #2500.
|
||||
}
|
||||
return normaliseGenres([]string{meta.Genre()}), fellBack
|
||||
}
|
||||
|
||||
// normaliseGenres expands each raw value, then drops case-insensitive
|
||||
// duplicates while keeping the first spelling seen. Duplicates are common once
|
||||
// numeric references are resolved: "(40)AlternRock" declares the same genre
|
||||
// twice, and so does a file tagged both "Rock" and "rock".
|
||||
func normaliseGenres(values []string) []string {
|
||||
out := make([]string, 0, len(values))
|
||||
seen := make(map[string]struct{}, len(values))
|
||||
for _, v := range values {
|
||||
for _, g := range normaliseGenreValue(v) {
|
||||
key := strings.ToLower(g)
|
||||
if _, dup := seen[key]; dup {
|
||||
continue
|
||||
}
|
||||
seen[key] = struct{}{}
|
||||
out = append(out, g)
|
||||
}
|
||||
}
|
||||
if len(out) == 0 {
|
||||
return nil
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// normaliseGenreValue turns one raw tag value into zero or more genre names,
|
||||
// resolving the ID3 numeric-reference syntax.
|
||||
//
|
||||
// A value may be:
|
||||
// - plain text ("Alternative Rock") — passed through
|
||||
// - a bare ID3v1 index ("17") — resolved to "Rock". This is what the spec
|
||||
// says a numeric TCON means, and what ffmpeg does. It is why the operator's
|
||||
// library showed genres like "4017" and "526617": several numeric values
|
||||
// welded together by the old reader.
|
||||
// - ID3v2.3 refinement syntax ("(17)", "(51)(39)", "(17)Hard Rock", "(RX)")
|
||||
// — each parenthesised index becomes its own genre, and trailing text
|
||||
// becomes one more.
|
||||
//
|
||||
// Values that are numeric but out of range carry no meaning as a label, so they
|
||||
// are dropped rather than stored as digits.
|
||||
func normaliseGenreValue(v string) []string {
|
||||
v = strings.TrimSpace(v)
|
||||
if v == "" {
|
||||
return nil
|
||||
}
|
||||
|
||||
var out []string
|
||||
for strings.HasPrefix(v, "(") {
|
||||
// "((" is the spec's escape for a literal "(" — the rest is plain text.
|
||||
if strings.HasPrefix(v, "((") {
|
||||
return append(out, strings.TrimSpace(v[1:]))
|
||||
}
|
||||
end := strings.IndexByte(v, ')')
|
||||
if end < 0 {
|
||||
break
|
||||
}
|
||||
inner := strings.TrimSpace(v[1:end])
|
||||
switch {
|
||||
case strings.EqualFold(inner, "RX"):
|
||||
out = append(out, "Remix")
|
||||
case strings.EqualFold(inner, "CR"):
|
||||
out = append(out, "Cover")
|
||||
default:
|
||||
n, err := strconv.Atoi(inner)
|
||||
if err != nil {
|
||||
// Parenthesised but not a reference, e.g. "(Live)". Keep the
|
||||
// whole remainder as written.
|
||||
return append(out, v)
|
||||
}
|
||||
if name, ok := id3v1GenreName(n); ok {
|
||||
out = append(out, name)
|
||||
}
|
||||
}
|
||||
v = strings.TrimSpace(v[end+1:])
|
||||
}
|
||||
|
||||
if v == "" {
|
||||
return out
|
||||
}
|
||||
if n, err := strconv.Atoi(v); err == nil {
|
||||
if name, ok := id3v1GenreName(n); ok {
|
||||
return append(out, name)
|
||||
}
|
||||
return out
|
||||
}
|
||||
return append(out, v)
|
||||
}
|
||||
|
||||
func id3v1GenreName(n int) (string, bool) {
|
||||
if n < 0 || n >= len(id3v1Genres) {
|
||||
return "", false
|
||||
}
|
||||
return id3v1Genres[n], true
|
||||
}
|
||||
|
||||
// id3v1Genres is the ID3v1 genre index: entries 0-79 are the original list,
|
||||
// 80-125 were added by Winamp, and 126-191 later still. Index is meaningful, so
|
||||
// never reorder or remove an entry — a numeric tag written years ago resolves
|
||||
// through this table by position.
|
||||
//
|
||||
// Entry 133 is "Afro-Punk"; the 1990s list used a slur there, and no file in
|
||||
// practice depends on the original spelling.
|
||||
var id3v1Genres = []string{
|
||||
"Blues", "Classic Rock", "Country", "Dance", "Disco", "Funk", "Grunge",
|
||||
"Hip-Hop", "Jazz", "Metal", "New Age", "Oldies", "Other", "Pop", "R&B",
|
||||
"Rap", "Reggae", "Rock", "Techno", "Industrial", "Alternative", "Ska",
|
||||
"Death Metal", "Pranks", "Soundtrack", "Euro-Techno", "Ambient",
|
||||
"Trip-Hop", "Vocal", "Jazz+Funk", "Fusion", "Trance", "Classical",
|
||||
"Instrumental", "Acid", "House", "Game", "Sound Clip", "Gospel", "Noise",
|
||||
"AlternRock", "Bass", "Soul", "Punk", "Space", "Meditative",
|
||||
"Instrumental Pop", "Instrumental Rock", "Ethnic", "Gothic", "Darkwave",
|
||||
"Techno-Industrial", "Electronic", "Pop-Folk", "Eurodance", "Dream",
|
||||
"Southern Rock", "Comedy", "Cult", "Gangsta", "Top 40", "Christian Rap",
|
||||
"Pop/Funk", "Jungle", "Native American", "Cabaret", "New Wave",
|
||||
"Psychadelic", "Rave", "Showtunes", "Trailer", "Lo-Fi", "Tribal",
|
||||
"Acid Punk", "Acid Jazz", "Polka", "Retro", "Musical", "Rock & Roll",
|
||||
"Hard Rock", "Folk", "Folk-Rock", "National Folk", "Swing", "Fast Fusion",
|
||||
"Bebob", "Latin", "Revival", "Celtic", "Bluegrass", "Avantgarde",
|
||||
"Gothic Rock", "Progressive Rock", "Psychedelic Rock", "Symphonic Rock",
|
||||
"Slow Rock", "Big Band", "Chorus", "Easy Listening", "Acoustic", "Humour",
|
||||
"Speech", "Chanson", "Opera", "Chamber Music", "Sonata", "Symphony",
|
||||
"Booty Bass", "Primus", "Porn Groove", "Satire", "Slow Jam", "Club",
|
||||
"Tango", "Samba", "Folklore", "Ballad", "Power Ballad", "Rhythmic Soul",
|
||||
"Freestyle", "Duet", "Punk Rock", "Drum Solo", "A capella", "Euro-House",
|
||||
"Dance Hall", "Goa", "Drum & Bass", "Club-House", "Hardcore", "Terror",
|
||||
"Indie", "BritPop", "Afro-Punk", "Polsk Punk", "Beat",
|
||||
"Christian Gangsta Rap", "Heavy Metal", "Black Metal", "Crossover",
|
||||
"Contemporary Christian", "Christian Rock", "Merengue", "Salsa",
|
||||
"Thrash Metal", "Anime", "JPop", "Synthpop", "Abstract", "Art Rock",
|
||||
"Baroque", "Bhangra", "Big Beat", "Breakbeat", "Chillout", "Downtempo",
|
||||
"Dub", "EBM", "Eclectic", "Electro", "Electroclash", "Emo",
|
||||
"Experimental", "Garage", "Global", "IDM", "Illbient", "Industro-Goth",
|
||||
"Jam Band", "Krautrock", "Leftfield", "Lounge", "Math Rock",
|
||||
"New Romantic", "Nu-Breakz", "Post-Punk", "Post-Rock", "Psytrance",
|
||||
"Shoegaze", "Space Rock", "Trop Rock", "World Music", "Neoclassical",
|
||||
"Audiobook", "Audio Theatre", "Neue Deutsche Welle", "Podcast",
|
||||
"Indie Rock", "G-Funk", "Dubstep", "Garage Rock", "Psybient",
|
||||
}
|
||||
Reference in New Issue
Block a user