fix(scanner): read multi-value genre frames correctly — #2499

dhowden/tag's readTFrame splits ID3v2 null-separated multi-value text frames and rejoins them with the EMPTY string, so a file tagged "Alternative Rock" + "Rock" was stored as "Alternative RockRock". It also leaves bare numeric ID3v1 references unresolved, which is why the library showed genres like "4017" and "526617". This corrupted more than the browse axis added in #367: taste_profile.sql reads tracks.genre directly, so the welded tokens were entering the taste profile's tag vocabulary, and recommendation.sql/discover.sql were comparing them as single opaque tags. Genre counts were wrong everywhere. ffprobe is not a fix — ffmpeg's read_ttag calls decode_str once with no loop, keeping only the first value. Truncating multi-genre tags would blunt the similarity signal genre mainly feeds. So the TCON frame is now parsed directly (ID3v2.2/2.3/2.4, all four text encodings, per-frame and tag-level unsynchronisation, numeric and parenthesised ID3v1 references); everything else still comes from dhowden/tag. Values are stored ";"-delimited, which the read side already splits on, so no query changes. Existing rows are repaired without an operator-run rebuild: migration 0054 adds tracks.tag_read_version DEFAULT 0, below the scanner's current tagReadVersion, so the next scan re-reads tags it would otherwise skip on mtime. Such a re-read reuses the stored duration instead of re-running ffprobe, keeping a repair pass tag-read-bound rather than one fork+exec per file. Bumping the constant is how a future extraction fix reaches an existing library. Only ID3v2 is in scope — dhowden welds nowhere else. The Vorbis/MP4 repeated-field question is #2500, unproven and deliberately not built.
2026-08-05 21:17:59 -04:00
parent 78aa9befb6
commit 37b396a7e4
15 changed files with 1128 additions and 49 deletions
@@ -261,7 +261,7 @@ func (q *Queries) InsertSkipEvent(ctx context.Context, arg InsertSkipEventParams
 }

 const listRecentSessionTracks = `-- name: ListRecentSessionTracks :many
-SELECT t.id, t.title, t.album_id, t.artist_id, t.track_number, t.disc_number, t.duration_ms, t.file_path, t.file_size, t.file_format, t.bitrate, t.mbid, t.genre, t.added_at, t.updated_at, t.tag_source, t.tag_sources_version FROM tracks t
+SELECT t.id, t.title, t.album_id, t.artist_id, t.track_number, t.disc_number, t.duration_ms, t.file_path, t.file_size, t.file_format, t.bitrate, t.mbid, t.genre, t.added_at, t.updated_at, t.tag_source, t.tag_sources_version, t.tag_read_version FROM tracks t
 JOIN play_events pe ON pe.track_id = t.id
 WHERE pe.session_id = $1
  AND pe.started_at < $2
@@ -305,6 +305,7 @@ func (q *Queries) ListRecentSessionTracks(ctx context.Context, arg ListRecentSes
 			&i.UpdatedAt,
 			&i.TagSource,
 			&i.TagSourcesVersion,
+			&i.TagReadVersion,
 		); err != nil {
 			return nil, err
 		}
@@ -14,7 +14,7 @@ import (
 const listUserHistory = `-- name: ListUserHistory :many
 SELECT pe.id           AS event_id,
       pe.started_at,
-       t.id, t.title, t.album_id, t.artist_id, t.track_number, t.disc_number, t.duration_ms, t.file_path, t.file_size, t.file_format, t.bitrate, t.mbid, t.genre, t.added_at, t.updated_at, t.tag_source, t.tag_sources_version,
+       t.id, t.title, t.album_id, t.artist_id, t.track_number, t.disc_number, t.duration_ms, t.file_path, t.file_size, t.file_format, t.bitrate, t.mbid, t.genre, t.added_at, t.updated_at, t.tag_source, t.tag_sources_version, t.tag_read_version,
       albums.title    AS album_title,
       artists.name    AS artist_name
  FROM play_events pe
@@ -79,6 +79,7 @@ func (q *Queries) ListUserHistory(ctx context.Context, arg ListUserHistoryParams
 			&i.Track.UpdatedAt,
 			&i.Track.TagSource,
 			&i.Track.TagSourcesVersion,
+			&i.Track.TagReadVersion,
 			&i.AlbumTitle,
 			&i.ArtistName,
 		); err != nil {
@@ -259,7 +259,7 @@ func (q *Queries) ListLikedTrackIDs(ctx context.Context, userID pgtype.UUID) ([]
 }

 const listLikedTrackRows = `-- name: ListLikedTrackRows :many
-SELECT t.id, t.title, t.album_id, t.artist_id, t.track_number, t.disc_number, t.duration_ms, t.file_path, t.file_size, t.file_format, t.bitrate, t.mbid, t.genre, t.added_at, t.updated_at, t.tag_source, t.tag_sources_version FROM tracks t
+SELECT t.id, t.title, t.album_id, t.artist_id, t.track_number, t.disc_number, t.duration_ms, t.file_path, t.file_size, t.file_format, t.bitrate, t.mbid, t.genre, t.added_at, t.updated_at, t.tag_source, t.tag_sources_version, t.tag_read_version FROM tracks t
 JOIN general_likes l ON l.track_id = t.id
 WHERE l.user_id = $1
 ORDER BY l.liked_at DESC
@@ -299,6 +299,7 @@ func (q *Queries) ListLikedTrackRows(ctx context.Context, arg ListLikedTrackRows
 			&i.UpdatedAt,
 			&i.TagSource,
 			&i.TagSourcesVersion,
+			&i.TagReadVersion,
 		); err != nil {
 			return nil, err
 		}
@@ -642,6 +642,7 @@ type Track struct {
 	UpdatedAt         pgtype.Timestamptz
 	TagSource         *string
 	TagSourcesVersion int32
+	TagReadVersion    int16
 }

 type TrackSimilarity struct {
@@ -208,7 +208,7 @@ WITH plays AS (
    WHERE user_id = $2 AND was_skipped = false
    GROUP BY track_id
 )
-SELECT t.id, t.title, t.album_id, t.artist_id, t.track_number, t.disc_number, t.duration_ms, t.file_path, t.file_size, t.file_format, t.bitrate, t.mbid, t.genre, t.added_at, t.updated_at, t.tag_source, t.tag_sources_version,
+SELECT t.id, t.title, t.album_id, t.artist_id, t.track_number, t.disc_number, t.duration_ms, t.file_path, t.file_size, t.file_format, t.bitrate, t.mbid, t.genre, t.added_at, t.updated_at, t.tag_source, t.tag_sources_version, t.tag_read_version,
       albums.title AS album_title,
       artists.name AS artist_name
 FROM plays p
@@ -267,6 +267,7 @@ func (q *Queries) ListMostPlayedTracksForArtist(ctx context.Context, arg ListMos
 			&i.Track.UpdatedAt,
 			&i.Track.TagSource,
 			&i.Track.TagSourcesVersion,
+			&i.Track.TagReadVersion,
 			&i.AlbumTitle,
 			&i.ArtistName,
 		); err != nil {
@@ -287,7 +288,7 @@ WITH plays AS (
    WHERE user_id = $1 AND was_skipped = false
    GROUP BY track_id
 )
-SELECT t.id, t.title, t.album_id, t.artist_id, t.track_number, t.disc_number, t.duration_ms, t.file_path, t.file_size, t.file_format, t.bitrate, t.mbid, t.genre, t.added_at, t.updated_at, t.tag_source, t.tag_sources_version,
+SELECT t.id, t.title, t.album_id, t.artist_id, t.track_number, t.disc_number, t.duration_ms, t.file_path, t.file_size, t.file_format, t.bitrate, t.mbid, t.genre, t.added_at, t.updated_at, t.tag_source, t.tag_sources_version, t.tag_read_version,
       albums.title AS album_title,
       artists.name AS artist_name
 FROM plays p
@@ -348,6 +349,7 @@ func (q *Queries) ListMostPlayedTracksForUser(ctx context.Context, arg ListMostP
 			&i.Track.UpdatedAt,
 			&i.Track.TagSource,
 			&i.Track.TagSourcesVersion,
+			&i.Track.TagReadVersion,
 			&i.AlbumTitle,
 			&i.ArtistName,
 		); err != nil {
@@ -685,7 +687,7 @@ func (q *Queries) ListRediscoverArtistsForUser(ctx context.Context, arg ListRedi

 const loadRadioCandidates = `-- name: LoadRadioCandidates :many
 SELECT
-    t.id, t.title, t.album_id, t.artist_id, t.track_number, t.disc_number, t.duration_ms, t.file_path, t.file_size, t.file_format, t.bitrate, t.mbid, t.genre, t.added_at, t.updated_at, t.tag_source, t.tag_sources_version,
+    t.id, t.title, t.album_id, t.artist_id, t.track_number, t.disc_number, t.duration_ms, t.file_path, t.file_size, t.file_format, t.bitrate, t.mbid, t.genre, t.added_at, t.updated_at, t.tag_source, t.tag_sources_version, t.tag_read_version,
    (l.user_id IS NOT NULL)::bool AS is_liked,
    pe.last_played_at::timestamptz AS last_played_at,
    pe.play_count,
@@ -763,6 +765,7 @@ func (q *Queries) LoadRadioCandidates(ctx context.Context, arg LoadRadioCandidat
 			&i.Track.UpdatedAt,
 			&i.Track.TagSource,
 			&i.Track.TagSourcesVersion,
+			&i.Track.TagReadVersion,
 			&i.IsLiked,
 			&i.LastPlayedAt,
 			&i.PlayCount,
@@ -895,7 +898,7 @@ random_fill AS (
    LIMIT $9
 )
 SELECT
-    t.id, t.title, t.album_id, t.artist_id, t.track_number, t.disc_number, t.duration_ms, t.file_path, t.file_size, t.file_format, t.bitrate, t.mbid, t.genre, t.added_at, t.updated_at, t.tag_source, t.tag_sources_version,
+    t.id, t.title, t.album_id, t.artist_id, t.track_number, t.disc_number, t.duration_ms, t.file_path, t.file_size, t.file_format, t.bitrate, t.mbid, t.genre, t.added_at, t.updated_at, t.tag_source, t.tag_sources_version, t.tag_read_version,
    (l.user_id IS NOT NULL)::bool AS is_liked,
    pe.last_played_at::timestamptz AS last_played_at,
    pe.play_count,
@@ -1004,6 +1007,7 @@ func (q *Queries) LoadRadioCandidatesV2(ctx context.Context, arg LoadRadioCandid
 			&i.Track.UpdatedAt,
 			&i.Track.TagSource,
 			&i.Track.TagSourcesVersion,
+			&i.Track.TagReadVersion,
 			&i.IsLiked,
 			&i.LastPlayedAt,
 			&i.PlayCount,
@@ -90,7 +90,7 @@ func (q *Queries) DeleteTrack(ctx context.Context, id pgtype.UUID) (DeleteTrackR
 }

 const getTrackByID = `-- name: GetTrackByID :one
-SELECT id, title, album_id, artist_id, track_number, disc_number, duration_ms, file_path, file_size, file_format, bitrate, mbid, genre, added_at, updated_at, tag_source, tag_sources_version FROM tracks WHERE id = $1
+SELECT id, title, album_id, artist_id, track_number, disc_number, duration_ms, file_path, file_size, file_format, bitrate, mbid, genre, added_at, updated_at, tag_source, tag_sources_version, tag_read_version FROM tracks WHERE id = $1
 `

 func (q *Queries) GetTrackByID(ctx context.Context, id pgtype.UUID) (Track, error) {
@@ -114,12 +114,13 @@ func (q *Queries) GetTrackByID(ctx context.Context, id pgtype.UUID) (Track, erro
 		&i.UpdatedAt,
 		&i.TagSource,
 		&i.TagSourcesVersion,
+		&i.TagReadVersion,
 	)
 	return i, err
 }

 const getTrackByPath = `-- name: GetTrackByPath :one
-SELECT id, title, album_id, artist_id, track_number, disc_number, duration_ms, file_path, file_size, file_format, bitrate, mbid, genre, added_at, updated_at, tag_source, tag_sources_version FROM tracks WHERE file_path = $1
+SELECT id, title, album_id, artist_id, track_number, disc_number, duration_ms, file_path, file_size, file_format, bitrate, mbid, genre, added_at, updated_at, tag_source, tag_sources_version, tag_read_version FROM tracks WHERE file_path = $1
 `

 func (q *Queries) GetTrackByPath(ctx context.Context, filePath string) (Track, error) {
@@ -143,12 +144,13 @@ func (q *Queries) GetTrackByPath(ctx context.Context, filePath string) (Track, e
 		&i.UpdatedAt,
 		&i.TagSource,
 		&i.TagSourcesVersion,
+		&i.TagReadVersion,
 	)
 	return i, err
 }

 const getTracksByIDs = `-- name: GetTracksByIDs :many
-SELECT id, title, album_id, artist_id, track_number, disc_number, duration_ms, file_path, file_size, file_format, bitrate, mbid, genre, added_at, updated_at, tag_source, tag_sources_version FROM tracks WHERE id = ANY($1::uuid[])
+SELECT id, title, album_id, artist_id, track_number, disc_number, duration_ms, file_path, file_size, file_format, bitrate, mbid, genre, added_at, updated_at, tag_source, tag_sources_version, tag_read_version FROM tracks WHERE id = ANY($1::uuid[])
 `

 // Batched lookup used by /api/library/sync to hydrate upsert payloads
@@ -180,6 +182,7 @@ func (q *Queries) GetTracksByIDs(ctx context.Context, dollar_1 []pgtype.UUID) ([
 			&i.UpdatedAt,
 			&i.TagSource,
 			&i.TagSourcesVersion,
+			&i.TagReadVersion,
 		); err != nil {
 			return nil, err
 		}
@@ -192,7 +195,7 @@ func (q *Queries) GetTracksByIDs(ctx context.Context, dollar_1 []pgtype.UUID) ([
 }

 const listArtistTracksForUser = `-- name: ListArtistTracksForUser :many
-SELECT t.id, t.title, t.album_id, t.artist_id, t.track_number, t.disc_number, t.duration_ms, t.file_path, t.file_size, t.file_format, t.bitrate, t.mbid, t.genre, t.added_at, t.updated_at, t.tag_source, t.tag_sources_version,
+SELECT t.id, t.title, t.album_id, t.artist_id, t.track_number, t.disc_number, t.duration_ms, t.file_path, t.file_size, t.file_format, t.bitrate, t.mbid, t.genre, t.added_at, t.updated_at, t.tag_source, t.tag_sources_version, t.tag_read_version,
       albums.title AS album_title,
       artists.name AS artist_name
 FROM tracks t
@@ -250,6 +253,7 @@ func (q *Queries) ListArtistTracksForUser(ctx context.Context, arg ListArtistTra
 			&i.Track.UpdatedAt,
 			&i.Track.TagSource,
 			&i.Track.TagSourcesVersion,
+			&i.Track.TagReadVersion,
 			&i.AlbumTitle,
 			&i.ArtistName,
 		); err != nil {
@@ -264,7 +268,7 @@ func (q *Queries) ListArtistTracksForUser(ctx context.Context, arg ListArtistTra
 }

 const listRandomTracksForUser = `-- name: ListRandomTracksForUser :many
-SELECT t.id, t.title, t.album_id, t.artist_id, t.track_number, t.disc_number, t.duration_ms, t.file_path, t.file_size, t.file_format, t.bitrate, t.mbid, t.genre, t.added_at, t.updated_at, t.tag_source, t.tag_sources_version,
+SELECT t.id, t.title, t.album_id, t.artist_id, t.track_number, t.disc_number, t.duration_ms, t.file_path, t.file_size, t.file_format, t.bitrate, t.mbid, t.genre, t.added_at, t.updated_at, t.tag_source, t.tag_sources_version, t.tag_read_version,
       albums.title AS album_title,
       artists.name AS artist_name
 FROM tracks t
@@ -319,6 +323,7 @@ func (q *Queries) ListRandomTracksForUser(ctx context.Context, arg ListRandomTra
 			&i.Track.UpdatedAt,
 			&i.Track.TagSource,
 			&i.Track.TagSourcesVersion,
+			&i.Track.TagReadVersion,
 			&i.AlbumTitle,
 			&i.ArtistName,
 		); err != nil {
@@ -333,7 +338,7 @@ func (q *Queries) ListRandomTracksForUser(ctx context.Context, arg ListRandomTra
 }

 const listTracksByAlbum = `-- name: ListTracksByAlbum :many
-SELECT id, title, album_id, artist_id, track_number, disc_number, duration_ms, file_path, file_size, file_format, bitrate, mbid, genre, added_at, updated_at, tag_source, tag_sources_version FROM tracks
+SELECT id, title, album_id, artist_id, track_number, disc_number, duration_ms, file_path, file_size, file_format, bitrate, mbid, genre, added_at, updated_at, tag_source, tag_sources_version, tag_read_version FROM tracks
 WHERE album_id = $1
  AND NOT EXISTS (
    SELECT 1 FROM lidarr_quarantine q
@@ -377,6 +382,7 @@ func (q *Queries) ListTracksByAlbum(ctx context.Context, arg ListTracksByAlbumPa
 			&i.UpdatedAt,
 			&i.TagSource,
 			&i.TagSourcesVersion,
+			&i.TagReadVersion,
 		); err != nil {
 			return nil, err
 		}
@@ -424,7 +430,7 @@ func (q *Queries) ListTracksMissingMbidWithPath(ctx context.Context, limit int32
 }

 const searchTracks = `-- name: SearchTracks :many
-SELECT id, title, album_id, artist_id, track_number, disc_number, duration_ms, file_path, file_size, file_format, bitrate, mbid, genre, added_at, updated_at, tag_source, tag_sources_version FROM tracks
+SELECT id, title, album_id, artist_id, track_number, disc_number, duration_ms, file_path, file_size, file_format, bitrate, mbid, genre, added_at, updated_at, tag_source, tag_sources_version, tag_read_version FROM tracks
 WHERE title ILIKE '%' || $1::text || '%'
  AND NOT EXISTS (
    SELECT 1 FROM lidarr_quarantine q
@@ -475,6 +481,7 @@ func (q *Queries) SearchTracks(ctx context.Context, arg SearchTracksParams) ([]T
 			&i.UpdatedAt,
 			&i.TagSource,
 			&i.TagSourcesVersion,
+			&i.TagReadVersion,
 		); err != nil {
 			return nil, err
 		}
@@ -507,8 +514,9 @@ func (q *Queries) SetTrackMbidIfNull(ctx context.Context, arg SetTrackMbidIfNull
 const upsertTrack = `-- name: UpsertTrack :one
 INSERT INTO tracks (
    title, album_id, artist_id, track_number, disc_number,
-    duration_ms, file_path, file_size, file_format, bitrate, mbid, genre
-) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12)
+    duration_ms, file_path, file_size, file_format, bitrate, mbid, genre,
+    tag_read_version
+) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13)
 ON CONFLICT (file_path) DO UPDATE SET
    title = EXCLUDED.title,
    album_id = EXCLUDED.album_id,
@@ -521,23 +529,27 @@ ON CONFLICT (file_path) DO UPDATE SET
    bitrate = EXCLUDED.bitrate,
    mbid = EXCLUDED.mbid,
    genre = EXCLUDED.genre,
+    -- Stamped on update too, so a tag-repair pass marks rows as done and the
+    -- next scan can short-circuit them again (#2499).
+    tag_read_version = EXCLUDED.tag_read_version,
    updated_at = now()
-RETURNING id, title, album_id, artist_id, track_number, disc_number, duration_ms, file_path, file_size, file_format, bitrate, mbid, genre, added_at, updated_at, tag_source, tag_sources_version
+RETURNING id, title, album_id, artist_id, track_number, disc_number, duration_ms, file_path, file_size, file_format, bitrate, mbid, genre, added_at, updated_at, tag_source, tag_sources_version, tag_read_version
 `

 type UpsertTrackParams struct {
-	Title       string
-	AlbumID     pgtype.UUID
-	ArtistID    pgtype.UUID
-	TrackNumber *int32
-	DiscNumber  *int32
-	DurationMs  int32
-	FilePath    string
-	FileSize    int64
-	FileFormat  string
-	Bitrate     *int32
-	Mbid        *string
-	Genre       *string
+	Title          string
+	AlbumID        pgtype.UUID
+	ArtistID       pgtype.UUID
+	TrackNumber    *int32
+	DiscNumber     *int32
+	DurationMs     int32
+	FilePath       string
+	FileSize       int64
+	FileFormat     string
+	Bitrate        *int32
+	Mbid           *string
+	Genre          *string
+	TagReadVersion int16
 }

 // file_path is the canonical identity for library scan; mbid is secondary.
@@ -555,6 +567,7 @@ func (q *Queries) UpsertTrack(ctx context.Context, arg UpsertTrackParams) (Track
 		arg.Bitrate,
 		arg.Mbid,
 		arg.Genre,
+		arg.TagReadVersion,
 	)
 	var i Track
 	err := row.Scan(
@@ -575,6 +588,7 @@ func (q *Queries) UpsertTrack(ctx context.Context, arg UpsertTrackParams) (Track
 		&i.UpdatedAt,
 		&i.TagSource,
 		&i.TagSourcesVersion,
+		&i.TagReadVersion,
 	)
 	return i, err
 }
@@ -0,0 +1,2 @@
+ALTER TABLE tracks
+    DROP COLUMN tag_read_version;
@@ -0,0 +1,15 @@
+-- Records which version of the scanner's tag-extraction logic last wrote a
+-- track's tag-derived columns (#2499).
+--
+-- DEFAULT 0 is the point of this migration: every existing row lands below the
+-- scanner's current library.tagReadVersion, so the next scan re-reads its tags
+-- instead of short-circuiting on the mtime check. That repairs genre values the
+-- old reader welded together ("Alternative Rock" + "Rock" -> "Alternative
+-- RockRock") without asking the operator to wipe and rebuild the library.
+--
+-- Bump library.tagReadVersion in Go — not this default — whenever a tag
+-- extraction fix needs to reach already-indexed files. That makes tag repairs a
+-- self-healing scan rather than a manual full rebuild, which is why this is a
+-- version number and not a boolean "needs_reread" flag.
+ALTER TABLE tracks
+    ADD COLUMN tag_read_version smallint NOT NULL DEFAULT 0;
@@ -2,8 +2,9 @@
 -- file_path is the canonical identity for library scan; mbid is secondary.
 INSERT INTO tracks (
    title, album_id, artist_id, track_number, disc_number,
-    duration_ms, file_path, file_size, file_format, bitrate, mbid, genre
-) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12)
+    duration_ms, file_path, file_size, file_format, bitrate, mbid, genre,
+    tag_read_version
+) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13)
 ON CONFLICT (file_path) DO UPDATE SET
    title = EXCLUDED.title,
    album_id = EXCLUDED.album_id,
@@ -16,6 +17,9 @@ ON CONFLICT (file_path) DO UPDATE SET
    bitrate = EXCLUDED.bitrate,
    mbid = EXCLUDED.mbid,
    genre = EXCLUDED.genre,
+    -- Stamped on update too, so a tag-repair pass marks rows as done and the
+    -- next scan can short-circuit them again (#2499).
+    tag_read_version = EXCLUDED.tag_read_version,
    updated_at = now()
 RETURNING *;