fix: every library album is matched to MusicBrainz before it is listed, by its tags, its artist's discography or a search, and checked against its tracks (M502 #5569)

The one-time listing took only albums whose tags already named their release group and artist, and left 340 off the list. The rest are now matched: the release their tags name, else the artist's release groups called what the album is called, else a MusicBrainz search by artist and title. A group found by name is taken only when its tracks agree with the album's (by disc and number, then title; length and title, or recording), so a partial copy still matches and a same-titled other album does not. library_album_matches records each album the listing decided, listed or unmatched with why, so an album taken off the list is never re-listed. The albums no match was found for are named on the Held tab. Album rows with no tracks are no longer counted. The length and title checks move from the importer to release, shared by both. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-10 17:16:05 -04:00
parent 2c677b3184
commit b887ab759b
21 changed files with 1378 additions and 601 deletions
@@ -487,6 +487,15 @@ type Grab struct {
 	SeedReleasedAt  pgtype.Timestamptz
 }

+type LibraryAlbumMatch struct {
+	AlbumID          pgtype.UUID
+	Outcome          string
+	Method           string
+	ReleaseGroupMbid string
+	Detail           string
+	DecidedAt        pgtype.Timestamptz
+}
+
 type LibraryChange struct {
 	ID         int64
 	EntityType string
@@ -125,15 +125,10 @@ func (q *Queries) DeleteWant(ctx context.Context, id pgtype.UUID) (int64, error)
 }

 const finishLibraryListing = `-- name: FinishLibraryListing :one
-UPDATE acquisition_settings s
+UPDATE acquisition_settings
   SET library_listed_at = now(),
-       library_unlisted = (
-         SELECT COUNT(*) FROM albums a
-          WHERE a.created_at <= s.library_listing_started_at
-            AND NOT EXISTS (SELECT 1 FROM wants w
-                             WHERE w.album_id = a.id
-                                OR (a.release_group_mbid <> '' AND w.release_group_mbid = a.release_group_mbid)))
- WHERE s.id = true AND s.library_listed_at IS NULL
+       library_unlisted = (SELECT COUNT(*) FROM library_album_matches WHERE outcome = 'unmatched')
+ WHERE id = true AND library_listed_at IS NULL
 RETURNING library_listed_at, library_unlisted
 `

@@ -142,8 +137,7 @@ type FinishLibraryListingRow struct {
 	LibraryUnlisted int32
 }

-// The listing is done: count the albums from before the cutoff it could not
-// list, because nothing names them to MusicBrainz.
+// The listing is done: count the albums no match was found for.
 func (q *Queries) FinishLibraryListing(ctx context.Context) (FinishLibraryListingRow, error) {
 	row := q.db.QueryRow(ctx, finishLibraryListing)
 	var i FinishLibraryListingRow
@@ -297,6 +291,44 @@ func (q *Queries) GetWantByReleaseGroup(ctx context.Context, releaseGroupMbid st
 	return i, err
 }

+const insertLibraryWant = `-- name: InsertLibraryWant :execrows
+INSERT INTO wants (release_group_mbid, artist_mbid, artist_name, title, primary_type, first_release_date,
+                   reason, album_id, wanted_at, fulfilled_at)
+VALUES ($1, $2, $3, $4,
+        $5, $6, 'library', $7::uuid, now(),
+        COALESCE((SELECT MIN(t.missing_since) FROM tracks t WHERE t.album_id = $7::uuid)
+                 - interval '1 second', now()))
+ON CONFLICT (release_group_mbid) DO NOTHING
+`
+
+type InsertLibraryWantParams struct {
+	ReleaseGroupMbid string
+	ArtistMbid       string
+	ArtistName       string
+	Title            string
+	PrimaryType      string
+	FirstReleaseDate string
+	AlbumID          pgtype.UUID
+}
+
+// Puts one matched album on the list as held, from just before its first
+// lost track if it has one. A group already on the list is left as it is.
+func (q *Queries) InsertLibraryWant(ctx context.Context, arg InsertLibraryWantParams) (int64, error) {
+	result, err := q.db.Exec(ctx, insertLibraryWant,
+		arg.ReleaseGroupMbid,
+		arg.ArtistMbid,
+		arg.ArtistName,
+		arg.Title,
+		arg.PrimaryType,
+		arg.FirstReleaseDate,
+		arg.AlbumID,
+	)
+	if err != nil {
+		return 0, err
+	}
+	return result.RowsAffected(), nil
+}
+
 const libraryHoldsReleaseGroup = `-- name: LibraryHoldsReleaseGroup :one
 SELECT EXISTS (
    SELECT 1 FROM albums a
@@ -325,6 +357,103 @@ func (q *Queries) LibraryHoldsReleaseGroup(ctx context.Context, arg LibraryHolds
 	return exists, err
 }

+const listAlbumTracksToMatch = `-- name: ListAlbumTracksToMatch :many
+SELECT title, disc_number, track_number, duration_ms, mbid
+  FROM tracks
+ WHERE album_id = $1
+ ORDER BY disc_number NULLS FIRST, track_number NULLS LAST, title
+`
+
+type ListAlbumTracksToMatchRow struct {
+	Title       string
+	DiscNumber  *int32
+	TrackNumber *int32
+	DurationMs  int32
+	Mbid        *string
+}
+
+// What an album's tracks say about themselves, to check a match against.
+func (q *Queries) ListAlbumTracksToMatch(ctx context.Context, albumID pgtype.UUID) ([]ListAlbumTracksToMatchRow, error) {
+	rows, err := q.db.Query(ctx, listAlbumTracksToMatch, albumID)
+	if err != nil {
+		return nil, err
+	}
+	defer rows.Close()
+	var items []ListAlbumTracksToMatchRow
+	for rows.Next() {
+		var i ListAlbumTracksToMatchRow
+		if err := rows.Scan(
+			&i.Title,
+			&i.DiscNumber,
+			&i.TrackNumber,
+			&i.DurationMs,
+			&i.Mbid,
+		); err != nil {
+			return nil, err
+		}
+		items = append(items, i)
+	}
+	if err := rows.Err(); err != nil {
+		return nil, err
+	}
+	return items, nil
+}
+
+const listAlbumsToMatch = `-- name: ListAlbumsToMatch :many
+SELECT a.id, a.title, a.mbid, a.release_group_mbid, ar.name AS artist_name, ar.mbid AS artist_mbid
+  FROM albums a
+  JOIN artists ar ON ar.id = a.artist_id
+ WHERE a.created_at <= $1::timestamptz
+   AND EXISTS (SELECT 1 FROM tracks t WHERE t.album_id = a.id)
+   AND NOT EXISTS (SELECT 1 FROM library_album_matches m WHERE m.album_id = a.id)
+ ORDER BY a.created_at, a.id
+ LIMIT $2
+`
+
+type ListAlbumsToMatchParams struct {
+	Cutoff  pgtype.Timestamptz
+	MaxRows int32
+}
+
+type ListAlbumsToMatchRow struct {
+	ID               pgtype.UUID
+	Title            string
+	Mbid             *string
+	ReleaseGroupMbid *string
+	ArtistName       string
+	ArtistMbid       *string
+}
+
+// Library albums from before the cutoff that the listing has not decided:
+// each is matched to MusicBrainz (#5569). An album row with no tracks is
+// not an album in the library.
+func (q *Queries) ListAlbumsToMatch(ctx context.Context, arg ListAlbumsToMatchParams) ([]ListAlbumsToMatchRow, error) {
+	rows, err := q.db.Query(ctx, listAlbumsToMatch, arg.Cutoff, arg.MaxRows)
+	if err != nil {
+		return nil, err
+	}
+	defer rows.Close()
+	var items []ListAlbumsToMatchRow
+	for rows.Next() {
+		var i ListAlbumsToMatchRow
+		if err := rows.Scan(
+			&i.ID,
+			&i.Title,
+			&i.Mbid,
+			&i.ReleaseGroupMbid,
+			&i.ArtistName,
+			&i.ArtistMbid,
+		); err != nil {
+			return nil, err
+		}
+		items = append(items, i)
+	}
+	if err := rows.Err(); err != nil {
+		return nil, err
+	}
+	return items, nil
+}
+
 const listFollowGroups = `-- name: ListFollowGroups :many
 SELECT g.mbid, g.title, g.primary_type, g.secondary_types, g.first_release_date,
       EXISTS (SELECT 1 FROM follow_seen_groups s
@@ -506,48 +635,6 @@ func (q *Queries) ListLibraryAlbums(ctx context.Context, arg ListLibraryAlbumsPa
 	return result.RowsAffected(), nil
 }

-const listLibraryAlbumsToName = `-- name: ListLibraryAlbumsToName :many
-SELECT a.id, a.title, a.mbid FROM albums a
- WHERE a.release_group_mbid IS NULL AND a.mbid IS NOT NULL AND a.mbid <> ''
-   AND a.created_at <= $1::timestamptz
- ORDER BY a.created_at, a.id
- LIMIT $2
-`
-
-type ListLibraryAlbumsToNameParams struct {
-	Cutoff  pgtype.Timestamptz
-	MaxRows int32
-}
-
-type ListLibraryAlbumsToNameRow struct {
-	ID    pgtype.UUID
-	Title string
-	Mbid  *string
-}
-
-// Library albums from before the cutoff that carry a release id but no
-// release group yet: the listing asks MusicBrainz for each. ” marks one
-// MusicBrainz could not name, so it is asked once.
-func (q *Queries) ListLibraryAlbumsToName(ctx context.Context, arg ListLibraryAlbumsToNameParams) ([]ListLibraryAlbumsToNameRow, error) {
-	rows, err := q.db.Query(ctx, listLibraryAlbumsToName, arg.Cutoff, arg.MaxRows)
-	if err != nil {
-		return nil, err
-	}
-	defer rows.Close()
-	var items []ListLibraryAlbumsToNameRow
-	for rows.Next() {
-		var i ListLibraryAlbumsToNameRow
-		if err := rows.Scan(&i.ID, &i.Title, &i.Mbid); err != nil {
-			return nil, err
-		}
-		items = append(items, i)
-	}
-	if err := rows.Err(); err != nil {
-		return nil, err
-	}
-	return items, nil
-}
-
 const listOpenWants = `-- name: ListOpenWants :many
 SELECT id, release_group_mbid, artist_mbid, artist_name, title, primary_type, first_release_date, reason, album_id, created_by, wanted_at, fulfilled_at, created_at, updated_at, searched_at, search_outcome, search_considered, best_release, best_verdict, search_error FROM wants
 WHERE fulfilled_at IS NULL
@@ -645,6 +732,49 @@ func (q *Queries) ListReleaseTypeProfiles(ctx context.Context) ([]ListReleaseTyp
 	return items, nil
 }

+const listUnmatchedLibraryAlbums = `-- name: ListUnmatchedLibraryAlbums :many
+SELECT m.album_id, a.title, ar.name AS artist_name, m.detail
+  FROM library_album_matches m
+  JOIN albums a ON a.id = m.album_id
+  JOIN artists ar ON ar.id = a.artist_id
+ WHERE m.outcome = 'unmatched'
+ ORDER BY ar.name, a.title
+ LIMIT $1
+`
+
+type ListUnmatchedLibraryAlbumsRow struct {
+	AlbumID    pgtype.UUID
+	Title      string
+	ArtistName string
+	Detail     string
+}
+
+// The albums the listing could not match, for the Held tab.
+func (q *Queries) ListUnmatchedLibraryAlbums(ctx context.Context, maxRows int32) ([]ListUnmatchedLibraryAlbumsRow, error) {
+	rows, err := q.db.Query(ctx, listUnmatchedLibraryAlbums, maxRows)
+	if err != nil {
+		return nil, err
+	}
+	defer rows.Close()
+	var items []ListUnmatchedLibraryAlbumsRow
+	for rows.Next() {
+		var i ListUnmatchedLibraryAlbumsRow
+		if err := rows.Scan(
+			&i.AlbumID,
+			&i.Title,
+			&i.ArtistName,
+			&i.Detail,
+		); err != nil {
+			return nil, err
+		}
+		items = append(items, i)
+	}
+	if err := rows.Err(); err != nil {
+		return nil, err
+	}
+	return items, nil
+}
+
 const listWants = `-- name: ListWants :many
 SELECT w.id, w.release_group_mbid, w.artist_mbid, w.artist_name, w.title, w.reason,
       w.album_id, w.wanted_at, w.fulfilled_at,
@@ -806,6 +936,52 @@ func (q *Queries) MarkGroupSeen(ctx context.Context, arg MarkGroupSeenParams) er
 	return err
 }

+const recordAlbumMatch = `-- name: RecordAlbumMatch :exec
+INSERT INTO library_album_matches (album_id, outcome, method, release_group_mbid, detail)
+VALUES ($1, $2, $3, $4, $5)
+ON CONFLICT (album_id) DO UPDATE
+   SET outcome = EXCLUDED.outcome, method = EXCLUDED.method,
+       release_group_mbid = EXCLUDED.release_group_mbid, detail = EXCLUDED.detail, decided_at = now()
+`
+
+type RecordAlbumMatchParams struct {
+	AlbumID          pgtype.UUID
+	Outcome          string
+	Method           string
+	ReleaseGroupMbid string
+	Detail           string
+}
+
+func (q *Queries) RecordAlbumMatch(ctx context.Context, arg RecordAlbumMatchParams) error {
+	_, err := q.db.Exec(ctx, recordAlbumMatch,
+		arg.AlbumID,
+		arg.Outcome,
+		arg.Method,
+		arg.ReleaseGroupMbid,
+		arg.Detail,
+	)
+	return err
+}
+
+const recordListedAlbums = `-- name: RecordListedAlbums :exec
+INSERT INTO library_album_matches (album_id, outcome, method, release_group_mbid)
+SELECT a.id, 'listed', 'tags', a.release_group_mbid
+  FROM albums a
+  JOIN artists ar ON ar.id = a.artist_id
+ WHERE a.release_group_mbid IS NOT NULL AND a.release_group_mbid <> ''
+   AND ar.mbid IS NOT NULL AND ar.mbid <> ''
+   AND a.created_at <= $1::timestamptz
+   AND EXISTS (SELECT 1 FROM tracks t WHERE t.album_id = a.id)
+ON CONFLICT (album_id) DO NOTHING
+`
+
+// The albums ListLibraryAlbums takes are decided as listed by their tags,
+// so the matching that follows leaves them alone (#5569).
+func (q *Queries) RecordListedAlbums(ctx context.Context, cutoff pgtype.Timestamptz) error {
+	_, err := q.db.Exec(ctx, recordListedAlbums, cutoff)
+	return err
+}
+
 const repointWant = `-- name: RepointWant :exec
 UPDATE wants SET release_group_mbid = $2, updated_at = now() WHERE id = $1
 `
@@ -822,8 +998,8 @@ func (q *Queries) RepointWant(ctx context.Context, arg RepointWantParams) error
 }

 const setAlbumReleaseGroup = `-- name: SetAlbumReleaseGroup :exec
-UPDATE albums SET release_group_mbid = $1
- WHERE id = $2 AND release_group_mbid IS NULL
+UPDATE albums SET release_group_mbid = $1, updated_at = now()
+ WHERE id = $2 AND (release_group_mbid IS NULL OR release_group_mbid = '')
 `

 type SetAlbumReleaseGroupParams struct {
@@ -831,6 +1007,7 @@ type SetAlbumReleaseGroupParams struct {
 	ID               pgtype.UUID
 }

+// Names a matched album's release group, filling only what nothing named.
 func (q *Queries) SetAlbumReleaseGroup(ctx context.Context, arg SetAlbumReleaseGroupParams) error {
 	_, err := q.db.Exec(ctx, setAlbumReleaseGroup, arg.ReleaseGroupMbid, arg.ID)
 	return err
@@ -0,0 +1 @@
+DROP TABLE library_album_matches;
@@ -0,0 +1,47 @@
+-- Every library album is matched to MusicBrainz before it goes on the list
+-- (#5569, operator 2026-10-10: "they should all exist in music brainz we
+-- need to use their meta data or a search system to link them up"). The
+-- listing in 0085 took only albums whose tags already named their release
+-- group and artist; the rest are now matched by the release their tags
+-- name, the artist's discography, or a search, and checked against their
+-- own tracks.
+--
+--   library_album_matches  one row per album the listing has decided:
+--                          listed (with how it was matched) or unmatched
+--                          (with why). It is the record that the listing
+--                          saw the album, so an album taken off the list
+--                          is never listed again.
+CREATE TABLE library_album_matches (
+    album_id           uuid PRIMARY KEY REFERENCES albums (id) ON DELETE CASCADE,
+    outcome            text NOT NULL
+        CONSTRAINT library_album_matches_outcome_check CHECK (outcome IN ('listed', 'unmatched')),
+    --   tags         its tags named the release group.
+    --   release      its tags named a release; MusicBrainz named the group.
+    --   discography  its title is one of the artist's release groups.
+    --   search       MusicBrainz's search found it by artist and title.
+    method             text NOT NULL DEFAULT ''
+        CONSTRAINT library_album_matches_method_check CHECK (method IN ('', 'tags', 'release', 'discography', 'search')),
+    release_group_mbid text NOT NULL DEFAULT '',
+    detail             text NOT NULL DEFAULT '',
+    decided_at         timestamptz NOT NULL DEFAULT now()
+);
+
+CREATE INDEX library_album_matches_unmatched_idx ON library_album_matches (decided_at) WHERE outcome = 'unmatched';
+
+-- The albums 0085's listing already took, by its own test, are decided:
+-- one taken off the list since stays off.
+INSERT INTO library_album_matches (album_id, outcome, method, release_group_mbid)
+SELECT a.id, 'listed', 'tags', a.release_group_mbid
+  FROM albums a
+  JOIN artists ar ON ar.id = a.artist_id
+  JOIN acquisition_settings s ON s.id = true
+ WHERE s.library_listing_started_at IS NOT NULL
+   AND a.created_at <= s.library_listing_started_at
+   AND a.release_group_mbid IS NOT NULL AND a.release_group_mbid <> ''
+   AND ar.mbid IS NOT NULL AND ar.mbid <> ''
+   AND EXISTS (SELECT 1 FROM tracks t WHERE t.album_id = a.id);
+
+-- The rest are matched by the wanted worker after boot: the listing resumes.
+UPDATE acquisition_settings
+   SET library_listed_at = NULL, library_unlisted = 0
+ WHERE library_listing_started_at IS NOT NULL;
@@ -258,30 +258,76 @@ SELECT DISTINCT ON (a.release_group_mbid)
          (SELECT COUNT(*) FROM tracks t WHERE t.album_id = a.id AND t.missing_since IS NULL) DESC, a.id
 ON CONFLICT (release_group_mbid) DO NOTHING;

-- name: ListLibraryAlbumsToName :many
-- Library albums from before the cutoff that carry a release id but no
-- release group yet: the listing asks MusicBrainz for each. '' marks one
-- MusicBrainz could not name, so it is asked once.
-SELECT a.id, a.title, a.mbid FROM albums a
- WHERE a.release_group_mbid IS NULL AND a.mbid IS NOT NULL AND a.mbid <> ''
+-- name: RecordListedAlbums :exec
+-- The albums ListLibraryAlbums takes are decided as listed by their tags,
+-- so the matching that follows leaves them alone (#5569).
+INSERT INTO library_album_matches (album_id, outcome, method, release_group_mbid)
+SELECT a.id, 'listed', 'tags', a.release_group_mbid
+  FROM albums a
+  JOIN artists ar ON ar.id = a.artist_id
+ WHERE a.release_group_mbid IS NOT NULL AND a.release_group_mbid <> ''
+   AND ar.mbid IS NOT NULL AND ar.mbid <> ''
   AND a.created_at <= sqlc.arg(cutoff)::timestamptz
+   AND EXISTS (SELECT 1 FROM tracks t WHERE t.album_id = a.id)
+ON CONFLICT (album_id) DO NOTHING;
+
+-- name: ListAlbumsToMatch :many
+-- Library albums from before the cutoff that the listing has not decided:
+-- each is matched to MusicBrainz (#5569). An album row with no tracks is
+-- not an album in the library.
+SELECT a.id, a.title, a.mbid, a.release_group_mbid, ar.name AS artist_name, ar.mbid AS artist_mbid
+  FROM albums a
+  JOIN artists ar ON ar.id = a.artist_id
+ WHERE a.created_at <= sqlc.arg(cutoff)::timestamptz
+   AND EXISTS (SELECT 1 FROM tracks t WHERE t.album_id = a.id)
+   AND NOT EXISTS (SELECT 1 FROM library_album_matches m WHERE m.album_id = a.id)
 ORDER BY a.created_at, a.id
 LIMIT sqlc.arg(max_rows);

+-- name: ListAlbumTracksToMatch :many
+-- What an album's tracks say about themselves, to check a match against.
+SELECT title, disc_number, track_number, duration_ms, mbid
+  FROM tracks
+ WHERE album_id = $1
+ ORDER BY disc_number NULLS FIRST, track_number NULLS LAST, title;
+
 -- name: SetAlbumReleaseGroup :exec
-UPDATE albums SET release_group_mbid = sqlc.arg(release_group_mbid)
- WHERE id = sqlc.arg(id) AND release_group_mbid IS NULL;
+-- Names a matched album's release group, filling only what nothing named.
+UPDATE albums SET release_group_mbid = sqlc.arg(release_group_mbid), updated_at = now()
+ WHERE id = sqlc.arg(id) AND (release_group_mbid IS NULL OR release_group_mbid = '');
+
+-- name: InsertLibraryWant :execrows
+-- Puts one matched album on the list as held, from just before its first
+-- lost track if it has one. A group already on the list is left as it is.
+INSERT INTO wants (release_group_mbid, artist_mbid, artist_name, title, primary_type, first_release_date,
+                   reason, album_id, wanted_at, fulfilled_at)
+VALUES (sqlc.arg(release_group_mbid), sqlc.arg(artist_mbid), sqlc.arg(artist_name), sqlc.arg(title),
+        sqlc.arg(primary_type), sqlc.arg(first_release_date), 'library', sqlc.arg(album_id)::uuid, now(),
+        COALESCE((SELECT MIN(t.missing_since) FROM tracks t WHERE t.album_id = sqlc.arg(album_id)::uuid)
+                 - interval '1 second', now()))
+ON CONFLICT (release_group_mbid) DO NOTHING;
+
+-- name: RecordAlbumMatch :exec
+INSERT INTO library_album_matches (album_id, outcome, method, release_group_mbid, detail)
+VALUES (sqlc.arg(album_id), sqlc.arg(outcome), sqlc.arg(method), sqlc.arg(release_group_mbid), sqlc.arg(detail))
+ON CONFLICT (album_id) DO UPDATE
+   SET outcome = EXCLUDED.outcome, method = EXCLUDED.method,
+       release_group_mbid = EXCLUDED.release_group_mbid, detail = EXCLUDED.detail, decided_at = now();

 -- name: FinishLibraryListing :one
-- The listing is done: count the albums from before the cutoff it could not
-- list, because nothing names them to MusicBrainz.
-UPDATE acquisition_settings s
+-- The listing is done: count the albums no match was found for.
+UPDATE acquisition_settings
   SET library_listed_at = now(),
-       library_unlisted = (
-         SELECT COUNT(*) FROM albums a
-          WHERE a.created_at <= s.library_listing_started_at
-            AND NOT EXISTS (SELECT 1 FROM wants w
-                             WHERE w.album_id = a.id
-                                OR (a.release_group_mbid <> '' AND w.release_group_mbid = a.release_group_mbid)))
- WHERE s.id = true AND s.library_listed_at IS NULL
+       library_unlisted = (SELECT COUNT(*) FROM library_album_matches WHERE outcome = 'unmatched')
+ WHERE id = true AND library_listed_at IS NULL
 RETURNING library_listed_at, library_unlisted;
+
+-- name: ListUnmatchedLibraryAlbums :many
+-- The albums the listing could not match, for the Held tab.
+SELECT m.album_id, a.title, ar.name AS artist_name, m.detail
+  FROM library_album_matches m
+  JOIN albums a ON a.id = m.album_id
+  JOIN artists ar ON ar.id = a.artist_id
+ WHERE m.outcome = 'unmatched'
+ ORDER BY ar.name, a.title
+ LIMIT sqlc.arg(max_rows);