bvandeusen/minstrel
1
0
Fork 0
Code Issues 1 Pull Requests Actions Packages Projects Releases 47 Wiki Activity

Missing files: detect them, stop offering them, and follow them when they move #121

Merged
bvandeusen merged 2 commits from dev into main 2026-08-06 20:40:39 -04:00
Conversation 0 Commits 2 Files Changed 25
+797 -49
22 changed files with 797 additions and 49 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Showing only changes of commit f6d1cf24f0 - Show all commits
internal/db/dbq/browse.sql.go
+17
View File
@@ -18,6 +18,7 @@ WHERE EXISTS (
FROM tracks FROM tracks
JOIN LATERAL regexp_split_to_table(coalesce(tracks.genre, ''), '[;,]') AS g(genre) ON true JOIN LATERAL regexp_split_to_table(coalesce(tracks.genre, ''), '[;,]') AS g(genre) ON true
WHERE tracks.album_id = albums.id WHERE tracks.album_id = albums.id
AND tracks.missing_since IS NULL
AND trim(g.genre) = trim($1::text) AND trim(g.genre) = trim($1::text)
) )
` `
@@ -97,6 +98,7 @@ WHERE EXISTS (
FROM tracks FROM tracks
JOIN LATERAL regexp_split_to_table(coalesce(tracks.genre, ''), '[;,]') AS g(genre) ON true JOIN LATERAL regexp_split_to_table(coalesce(tracks.genre, ''), '[;,]') AS g(genre) ON true
WHERE tracks.album_id = albums.id WHERE tracks.album_id = albums.id
AND tracks.missing_since IS NULL
AND trim(g.genre) = trim($1::text) AND trim(g.genre) = trim($1::text)
) )
ORDER BY albums.sort_title, albums.id ORDER BY albums.sort_title, albums.id
@@ -219,6 +221,7 @@ SELECT DISTINCT trim(g.genre) AS genre
FROM tracks FROM tracks
JOIN LATERAL regexp_split_to_table(coalesce(tracks.genre, ''), '[;,]') AS g(genre) ON true JOIN LATERAL regexp_split_to_table(coalesce(tracks.genre, ''), '[;,]') AS g(genre) ON true
WHERE tracks.album_id = $1 AND trim(g.genre) <> '' WHERE tracks.album_id = $1 AND trim(g.genre) <> ''
AND tracks.missing_since IS NULL
ORDER BY trim(g.genre) ORDER BY trim(g.genre)
` `
@@ -251,6 +254,7 @@ SELECT DISTINCT trim(g.genre) AS genre
FROM tracks FROM tracks
JOIN LATERAL regexp_split_to_table(coalesce(tracks.genre, ''), '[;,]') AS g(genre) ON true JOIN LATERAL regexp_split_to_table(coalesce(tracks.genre, ''), '[;,]') AS g(genre) ON true
WHERE tracks.artist_id = $1 AND trim(g.genre) <> '' WHERE tracks.artist_id = $1 AND trim(g.genre) <> ''
AND tracks.missing_since IS NULL
ORDER BY trim(g.genre) ORDER BY trim(g.genre)
` `
@@ -278,10 +282,12 @@ func (q *Queries) ListGenresForArtist(ctx context.Context, artistID pgtype.UUID)
} }
const listGenresWithCount = `-- name: ListGenresWithCount :many const listGenresWithCount = `-- name: ListGenresWithCount :many
SELECT trim(g.genre) AS genre, COUNT(DISTINCT tracks.id)::bigint AS track_count SELECT trim(g.genre) AS genre, COUNT(DISTINCT tracks.id)::bigint AS track_count
FROM tracks FROM tracks
JOIN LATERAL regexp_split_to_table(coalesce(tracks.genre, ''), '[;,]') AS g(genre) ON true JOIN LATERAL regexp_split_to_table(coalesce(tracks.genre, ''), '[;,]') AS g(genre) ON true
WHERE trim(g.genre) <> '' WHERE trim(g.genre) <> ''
AND tracks.missing_since IS NULL
GROUP BY trim(g.genre) GROUP BY trim(g.genre)
ORDER BY track_count DESC, trim(g.genre) ORDER BY track_count DESC, trim(g.genre)
` `
@@ -291,6 +297,17 @@ type ListGenresWithCountRow struct {
TrackCount int64 TrackCount int64
} }
// Every query in this file filters `tracks.missing_since IS NULL` (#2523).
// A row whose file has vanished keeps its genre forever — the scanner walks the
// filesystem, so it never revisits a path that no longer exists — which is how
// pre-#2499 welded genres survived a full re-scan and kept showing in the index.
// Browsing is a way of finding something to play, so a track that cannot play
// should not shape it.
//
// Year queries below join albums only and are deliberately left alone: an album
// is still a real release even if some of its tracks are gone. An album whose
// EVERY track is missing will linger on the year axis; that's a narrower case,
// tracked with the rest of the cleanup work.
// Genre browse index (#367). // Genre browse index (#367).
// //
// Genres live inline on tracks.genre as a delimited string, so this splits on // Genres live inline on tracks.genre as a delimited string, so this splits on
internal/db/dbq/discover.sql.go
+8 -4
View File
@@ -15,7 +15,8 @@ const listCrossUserLikedTracksForDiscover = `-- name: ListCrossUserLikedTracksFo
SELECT t.id, t.album_id, t.artist_id SELECT t.id, t.album_id, t.artist_id
FROM general_likes gl FROM general_likes gl
JOIN tracks t ON t.id = gl.track_id JOIN tracks t ON t.id = gl.track_id
WHERE gl.user_id != $1 WHERE t.missing_since IS NULL -- #2523: never offer a file that is gone
AND gl.user_id != $1
AND NOT EXISTS ( AND NOT EXISTS (
SELECT 1 FROM play_events pe SELECT 1 FROM play_events pe
WHERE pe.user_id = $1 WHERE pe.user_id = $1
@@ -95,7 +96,8 @@ dormant_artists AS (
SELECT t.id, t.album_id, t.artist_id SELECT t.id, t.album_id, t.artist_id
FROM tracks t FROM tracks t
JOIN dormant_artists da ON da.id = t.artist_id JOIN dormant_artists da ON da.id = t.artist_id
WHERE NOT EXISTS ( WHERE t.missing_since IS NULL -- #2523: never offer a file that is gone
AND NOT EXISTS (
SELECT 1 FROM play_events pe SELECT 1 FROM play_events pe
WHERE pe.user_id = $1 WHERE pe.user_id = $1
AND pe.track_id = t.id AND pe.track_id = t.id
@@ -159,7 +161,8 @@ func (q *Queries) ListDormantArtistTracksForDiscover(ctx context.Context, arg Li
const listRandomUnheardTracksForDiscover = `-- name: ListRandomUnheardTracksForDiscover :many const listRandomUnheardTracksForDiscover = `-- name: ListRandomUnheardTracksForDiscover :many
SELECT t.id, t.album_id, t.artist_id SELECT t.id, t.album_id, t.artist_id
FROM tracks t FROM tracks t
WHERE NOT EXISTS ( WHERE t.missing_since IS NULL -- #2523: never offer a file that is gone
AND NOT EXISTS (
SELECT 1 FROM play_events pe SELECT 1 FROM play_events pe
WHERE pe.user_id = $1 WHERE pe.user_id = $1
AND pe.track_id = t.id AND pe.track_id = t.id
@@ -217,7 +220,8 @@ SELECT t.id, t.album_id, t.artist_id
FROM tracks t FROM tracks t
JOIN LATERAL regexp_split_to_table(coalesce(t.genre, ''), '[;,]') AS g_split(g) ON true JOIN LATERAL regexp_split_to_table(coalesce(t.genre, ''), '[;,]') AS g_split(g) ON true
JOIN taste_profile_tags nt ON nt.user_id = $1 AND trim(g_split.g) = nt.tag JOIN taste_profile_tags nt ON nt.user_id = $1 AND trim(g_split.g) = nt.tag
WHERE nt.weight > 0 WHERE t.missing_since IS NULL -- #2523: never offer a file that is gone
AND nt.weight > 0
AND trim(g_split.g) <> '' AND trim(g_split.g) <> ''
AND NOT EXISTS ( AND NOT EXISTS (
SELECT 1 FROM play_events pe SELECT 1 FROM play_events pe
internal/db/dbq/events.sql.go
+2 -1
View File
@@ -261,7 +261,7 @@ func (q *Queries) InsertSkipEvent(ctx context.Context, arg InsertSkipEventParams
} }
const listRecentSessionTracks = `-- name: ListRecentSessionTracks :many const listRecentSessionTracks = `-- name: ListRecentSessionTracks :many
SELECT t.id, t.title, t.album_id, t.artist_id, t.track_number, t.disc_number, t.duration_ms, t.file_path, t.file_size, t.file_format, t.bitrate, t.mbid, t.genre, t.added_at, t.updated_at, t.tag_source, t.tag_sources_version, t.tag_read_version FROM tracks t SELECT t.id, t.title, t.album_id, t.artist_id, t.track_number, t.disc_number, t.duration_ms, t.file_path, t.file_size, t.file_format, t.bitrate, t.mbid, t.genre, t.added_at, t.updated_at, t.tag_source, t.tag_sources_version, t.tag_read_version, t.missing_since FROM tracks t
JOIN play_events pe ON pe.track_id = t.id JOIN play_events pe ON pe.track_id = t.id
WHERE pe.session_id = $1 WHERE pe.session_id = $1
AND pe.started_at < $2 AND pe.started_at < $2
@@ -306,6 +306,7 @@ func (q *Queries) ListRecentSessionTracks(ctx context.Context, arg ListRecentSes
&i.TagSource, &i.TagSource,
&i.TagSourcesVersion, &i.TagSourcesVersion,
&i.TagReadVersion, &i.TagReadVersion,
&i.MissingSince,
); err != nil { ); err != nil {
return nil, err return nil, err
} }
internal/db/dbq/history.sql.go
+2 -1
View File
@@ -14,7 +14,7 @@ import (
const listUserHistory = `-- name: ListUserHistory :many const listUserHistory = `-- name: ListUserHistory :many
SELECT pe.id AS event_id, SELECT pe.id AS event_id,
pe.started_at, pe.started_at,
t.id, t.title, t.album_id, t.artist_id, t.track_number, t.disc_number, t.duration_ms, t.file_path, t.file_size, t.file_format, t.bitrate, t.mbid, t.genre, t.added_at, t.updated_at, t.tag_source, t.tag_sources_version, t.tag_read_version, t.id, t.title, t.album_id, t.artist_id, t.track_number, t.disc_number, t.duration_ms, t.file_path, t.file_size, t.file_format, t.bitrate, t.mbid, t.genre, t.added_at, t.updated_at, t.tag_source, t.tag_sources_version, t.tag_read_version, t.missing_since,
albums.title AS album_title, albums.title AS album_title,
artists.name AS artist_name artists.name AS artist_name
FROM play_events pe FROM play_events pe
@@ -80,6 +80,7 @@ func (q *Queries) ListUserHistory(ctx context.Context, arg ListUserHistoryParams
&i.Track.TagSource, &i.Track.TagSource,
&i.Track.TagSourcesVersion, &i.Track.TagSourcesVersion,
&i.Track.TagReadVersion, &i.Track.TagReadVersion,
&i.Track.MissingSince,
&i.AlbumTitle, &i.AlbumTitle,
&i.ArtistName, &i.ArtistName,
); err != nil { ); err != nil {
internal/db/dbq/likes.sql.go
+2 -1
View File
@@ -259,7 +259,7 @@ func (q *Queries) ListLikedTrackIDs(ctx context.Context, userID pgtype.UUID) ([]
} }
const listLikedTrackRows = `-- name: ListLikedTrackRows :many const listLikedTrackRows = `-- name: ListLikedTrackRows :many
SELECT t.id, t.title, t.album_id, t.artist_id, t.track_number, t.disc_number, t.duration_ms, t.file_path, t.file_size, t.file_format, t.bitrate, t.mbid, t.genre, t.added_at, t.updated_at, t.tag_source, t.tag_sources_version, t.tag_read_version FROM tracks t SELECT t.id, t.title, t.album_id, t.artist_id, t.track_number, t.disc_number, t.duration_ms, t.file_path, t.file_size, t.file_format, t.bitrate, t.mbid, t.genre, t.added_at, t.updated_at, t.tag_source, t.tag_sources_version, t.tag_read_version, t.missing_since FROM tracks t
JOIN general_likes l ON l.track_id = t.id JOIN general_likes l ON l.track_id = t.id
WHERE l.user_id = $1 WHERE l.user_id = $1
ORDER BY l.liked_at DESC ORDER BY l.liked_at DESC
@@ -300,6 +300,7 @@ func (q *Queries) ListLikedTrackRows(ctx context.Context, arg ListLikedTrackRows
&i.TagSource, &i.TagSource,
&i.TagSourcesVersion, &i.TagSourcesVersion,
&i.TagReadVersion, &i.TagReadVersion,
&i.MissingSince,
); err != nil { ); err != nil {
return nil, err return nil, err
} }
internal/db/dbq/models.go
+1
View File
@@ -643,6 +643,7 @@ type Track struct {
TagSource *string TagSource *string
TagSourcesVersion int32 TagSourcesVersion int32
TagReadVersion int16 TagReadVersion int16
MissingSince pgtype.Timestamptz
} }
type TrackSimilarity struct { type TrackSimilarity struct {
internal/db/dbq/recommendation.sql.go
+13 -6
View File
@@ -208,7 +208,7 @@ WITH plays AS (
WHERE user_id = $2 AND was_skipped = false WHERE user_id = $2 AND was_skipped = false
GROUP BY track_id GROUP BY track_id
) )
SELECT t.id, t.title, t.album_id, t.artist_id, t.track_number, t.disc_number, t.duration_ms, t.file_path, t.file_size, t.file_format, t.bitrate, t.mbid, t.genre, t.added_at, t.updated_at, t.tag_source, t.tag_sources_version, t.tag_read_version, SELECT t.id, t.title, t.album_id, t.artist_id, t.track_number, t.disc_number, t.duration_ms, t.file_path, t.file_size, t.file_format, t.bitrate, t.mbid, t.genre, t.added_at, t.updated_at, t.tag_source, t.tag_sources_version, t.tag_read_version, t.missing_since,
albums.title AS album_title, albums.title AS album_title,
artists.name AS artist_name artists.name AS artist_name
FROM plays p FROM plays p
@@ -216,6 +216,7 @@ JOIN tracks t ON t.id = p.track_id
JOIN albums ON albums.id = t.album_id JOIN albums ON albums.id = t.album_id
JOIN artists ON artists.id = t.artist_id JOIN artists ON artists.id = t.artist_id
WHERE t.artist_id = $1 WHERE t.artist_id = $1
AND t.missing_since IS NULL -- #2523: never offer a file that is gone
AND NOT EXISTS ( AND NOT EXISTS (
SELECT 1 FROM lidarr_quarantine q SELECT 1 FROM lidarr_quarantine q
WHERE q.user_id = $2 AND q.track_id = t.id WHERE q.user_id = $2 AND q.track_id = t.id
@@ -268,6 +269,7 @@ func (q *Queries) ListMostPlayedTracksForArtist(ctx context.Context, arg ListMos
&i.Track.TagSource, &i.Track.TagSource,
&i.Track.TagSourcesVersion, &i.Track.TagSourcesVersion,
&i.Track.TagReadVersion, &i.Track.TagReadVersion,
&i.Track.MissingSince,
&i.AlbumTitle, &i.AlbumTitle,
&i.ArtistName, &i.ArtistName,
); err != nil { ); err != nil {
@@ -288,14 +290,15 @@ WITH plays AS (
WHERE user_id = $1 AND was_skipped = false WHERE user_id = $1 AND was_skipped = false
GROUP BY track_id GROUP BY track_id
) )
SELECT t.id, t.title, t.album_id, t.artist_id, t.track_number, t.disc_number, t.duration_ms, t.file_path, t.file_size, t.file_format, t.bitrate, t.mbid, t.genre, t.added_at, t.updated_at, t.tag_source, t.tag_sources_version, t.tag_read_version, SELECT t.id, t.title, t.album_id, t.artist_id, t.track_number, t.disc_number, t.duration_ms, t.file_path, t.file_size, t.file_format, t.bitrate, t.mbid, t.genre, t.added_at, t.updated_at, t.tag_source, t.tag_sources_version, t.tag_read_version, t.missing_since,
albums.title AS album_title, albums.title AS album_title,
artists.name AS artist_name artists.name AS artist_name
FROM plays p FROM plays p
JOIN tracks t ON t.id = p.track_id JOIN tracks t ON t.id = p.track_id
JOIN albums ON albums.id = t.album_id JOIN albums ON albums.id = t.album_id
JOIN artists ON artists.id = t.artist_id JOIN artists ON artists.id = t.artist_id
WHERE NOT EXISTS ( WHERE t.missing_since IS NULL -- #2523: never offer a file that is gone
AND NOT EXISTS (
SELECT 1 FROM lidarr_quarantine q SELECT 1 FROM lidarr_quarantine q
WHERE q.user_id = $1 AND q.track_id = t.id WHERE q.user_id = $1 AND q.track_id = t.id
) )
@@ -350,6 +353,7 @@ func (q *Queries) ListMostPlayedTracksForUser(ctx context.Context, arg ListMostP
&i.Track.TagSource, &i.Track.TagSource,
&i.Track.TagSourcesVersion, &i.Track.TagSourcesVersion,
&i.Track.TagReadVersion, &i.Track.TagReadVersion,
&i.Track.MissingSince,
&i.AlbumTitle, &i.AlbumTitle,
&i.ArtistName, &i.ArtistName,
); err != nil { ); err != nil {
@@ -687,7 +691,7 @@ func (q *Queries) ListRediscoverArtistsForUser(ctx context.Context, arg ListRedi
const loadRadioCandidates = `-- name: LoadRadioCandidates :many const loadRadioCandidates = `-- name: LoadRadioCandidates :many
SELECT SELECT
t.id, t.title, t.album_id, t.artist_id, t.track_number, t.disc_number, t.duration_ms, t.file_path, t.file_size, t.file_format, t.bitrate, t.mbid, t.genre, t.added_at, t.updated_at, t.tag_source, t.tag_sources_version, t.tag_read_version, t.id, t.title, t.album_id, t.artist_id, t.track_number, t.disc_number, t.duration_ms, t.file_path, t.file_size, t.file_format, t.bitrate, t.mbid, t.genre, t.added_at, t.updated_at, t.tag_source, t.tag_sources_version, t.tag_read_version, t.missing_since,
(l.user_id IS NOT NULL)::bool AS is_liked, (l.user_id IS NOT NULL)::bool AS is_liked,
pe.last_played_at::timestamptz AS last_played_at, pe.last_played_at::timestamptz AS last_played_at,
pe.play_count, pe.play_count,
@@ -705,6 +709,7 @@ LEFT JOIN LATERAL (
WHERE user_id = $1 AND track_id = t.id WHERE user_id = $1 AND track_id = t.id
) pe ON true ) pe ON true
WHERE t.id <> $2 WHERE t.id <> $2
AND t.missing_since IS NULL -- #2523: never offer a file that is gone
AND NOT EXISTS ( AND NOT EXISTS (
SELECT 1 FROM play_events SELECT 1 FROM play_events
WHERE user_id = $1 AND track_id = t.id WHERE user_id = $1 AND track_id = t.id
@@ -766,6 +771,7 @@ func (q *Queries) LoadRadioCandidates(ctx context.Context, arg LoadRadioCandidat
&i.Track.TagSource, &i.Track.TagSource,
&i.Track.TagSourcesVersion, &i.Track.TagSourcesVersion,
&i.Track.TagReadVersion, &i.Track.TagReadVersion,
&i.Track.MissingSince,
&i.IsLiked, &i.IsLiked,
&i.LastPlayedAt, &i.LastPlayedAt,
&i.PlayCount, &i.PlayCount,
@@ -898,7 +904,7 @@ random_fill AS (
LIMIT $9 LIMIT $9
) )
SELECT SELECT
t.id, t.title, t.album_id, t.artist_id, t.track_number, t.disc_number, t.duration_ms, t.file_path, t.file_size, t.file_format, t.bitrate, t.mbid, t.genre, t.added_at, t.updated_at, t.tag_source, t.tag_sources_version, t.tag_read_version, t.id, t.title, t.album_id, t.artist_id, t.track_number, t.disc_number, t.duration_ms, t.file_path, t.file_size, t.file_format, t.bitrate, t.mbid, t.genre, t.added_at, t.updated_at, t.tag_source, t.tag_sources_version, t.tag_read_version, t.missing_since,
(l.user_id IS NOT NULL)::bool AS is_liked, (l.user_id IS NOT NULL)::bool AS is_liked,
pe.last_played_at::timestamptz AS last_played_at, pe.last_played_at::timestamptz AS last_played_at,
pe.play_count, pe.play_count,
@@ -914,7 +920,7 @@ FROM (
UNION ALL SELECT track_id, sim_score FROM coplay_artists UNION ALL SELECT track_id, sim_score FROM coplay_artists
UNION ALL SELECT track_id, sim_score FROM random_fill UNION ALL SELECT track_id, sim_score FROM random_fill
) u ) u
JOIN tracks t ON t.id = u.track_id JOIN tracks t ON t.id = u.track_id AND t.missing_since IS NULL -- #2523: never offer a file that is gone
JOIN albums al ON al.id = t.album_id JOIN albums al ON al.id = t.album_id
LEFT JOIN general_likes l ON l.user_id = $1 AND l.track_id = t.id LEFT JOIN general_likes l ON l.user_id = $1 AND l.track_id = t.id
LEFT JOIN LATERAL ( LEFT JOIN LATERAL (
@@ -1008,6 +1014,7 @@ func (q *Queries) LoadRadioCandidatesV2(ctx context.Context, arg LoadRadioCandid
&i.Track.TagSource, &i.Track.TagSource,
&i.Track.TagSourcesVersion, &i.Track.TagSourcesVersion,
&i.Track.TagReadVersion, &i.Track.TagReadVersion,
&i.Track.MissingSince,
&i.IsLiked, &i.IsLiked,
&i.LastPlayedAt, &i.LastPlayedAt,
&i.PlayCount, &i.PlayCount,
internal/db/dbq/system_mixes.sql.go
+10 -5
View File
@@ -39,7 +39,8 @@ SELECT t.id, t.album_id, t.artist_id
JOIN affinity_artists aa ON aa.artist_id = t.artist_id JOIN affinity_artists aa ON aa.artist_id = t.artist_id
LEFT JOIN play_counts pc ON pc.track_id = t.id LEFT JOIN play_counts pc ON pc.track_id = t.id
LEFT JOIN skip_counts sc ON sc.track_id = t.id LEFT JOIN skip_counts sc ON sc.track_id = t.id
WHERE COALESCE(pc.c, 0) <= 2 WHERE t.missing_since IS NULL -- #2523: never offer a file that is gone
AND COALESCE(pc.c, 0) <= 2
AND COALESCE(sc.c, 0) < 2 AND COALESCE(sc.c, 0) < 2
AND NOT EXISTS ( AND NOT EXISTS (
SELECT 1 FROM lidarr_quarantine q SELECT 1 FROM lidarr_quarantine q
@@ -124,7 +125,8 @@ SELECT t.id, t.album_id, t.artist_id, alt.tier::int AS tier
FROM tracks t FROM tracks t
JOIN albums al ON al.id = t.album_id JOIN albums al ON al.id = t.album_id
JOIN albums_tiered alt ON alt.album_id = al.id JOIN albums_tiered alt ON alt.album_id = al.id
WHERE alt.tier IS NOT NULL WHERE t.missing_since IS NULL -- #2523: never offer a file that is gone
AND alt.tier IS NOT NULL
AND NOT EXISTS (SELECT 1 FROM attempted a WHERE a.track_id = t.id) AND NOT EXISTS (SELECT 1 FROM attempted a WHERE a.track_id = t.id)
AND NOT EXISTS ( AND NOT EXISTS (
SELECT 1 FROM lidarr_quarantine q SELECT 1 FROM lidarr_quarantine q
@@ -225,7 +227,8 @@ albums_tiered AS (
SELECT t.id, t.album_id, t.artist_id, alt.tier::int AS tier SELECT t.id, t.album_id, t.artist_id, alt.tier::int AS tier
FROM tracks t FROM tracks t
JOIN albums_tiered alt ON alt.album_id = t.album_id JOIN albums_tiered alt ON alt.album_id = t.album_id
WHERE NOT EXISTS ( WHERE t.missing_since IS NULL -- #2523: never offer a file that is gone
AND NOT EXISTS (
SELECT 1 FROM lidarr_quarantine q SELECT 1 FROM lidarr_quarantine q
WHERE q.user_id = $1 AND q.track_id = t.id WHERE q.user_id = $1 AND q.track_id = t.id
) )
@@ -303,7 +306,8 @@ WITH windowed AS (
SELECT t.id, t.album_id, t.artist_id SELECT t.id, t.album_id, t.artist_id
FROM tracks t FROM tracks t
JOIN windowed w ON w.track_id = t.id JOIN windowed w ON w.track_id = t.id
WHERE NOT EXISTS ( WHERE t.missing_since IS NULL -- #2523: never offer a file that is gone
AND NOT EXISTS (
SELECT 1 FROM lidarr_quarantine q SELECT 1 FROM lidarr_quarantine q
WHERE q.user_id = $1 AND q.track_id = t.id WHERE q.user_id = $1 AND q.track_id = t.id
) )
@@ -367,7 +371,8 @@ WITH stats AS (
SELECT t.id, t.album_id, t.artist_id SELECT t.id, t.album_id, t.artist_id
FROM tracks t FROM tracks t
JOIN stats s ON s.track_id = t.id JOIN stats s ON s.track_id = t.id
WHERE s.c >= 3 WHERE t.missing_since IS NULL -- #2523: never offer a file that is gone
AND s.c >= 3
AND s.last_at <= now() - interval '30 days' AND s.last_at <= now() - interval '30 days'
AND NOT EXISTS ( AND NOT EXISTS (
SELECT 1 FROM lidarr_quarantine q SELECT 1 FROM lidarr_quarantine q
internal/db/dbq/system_playlists.sql.go
+14 -6
View File
@@ -189,6 +189,7 @@ func (q *Queries) GetSystemPlaylistRun(ctx context.Context, userID pgtype.UUID)
const listActiveUsersForSystemPlaylists = `-- name: ListActiveUsersForSystemPlaylists :many const listActiveUsersForSystemPlaylists = `-- name: ListActiveUsersForSystemPlaylists :many
SELECT u.id FROM users u SELECT u.id FROM users u
WHERE EXISTS ( WHERE EXISTS (
SELECT 1 FROM play_events pe SELECT 1 FROM play_events pe
@@ -197,6 +198,13 @@ SELECT u.id FROM users u
) )
` `
// Track picks here join `tracks ... AND t.missing_since IS NULL` (#2523): a
// seed or For-You candidate has to be something that can actually play. Note
// this only affects newly GENERATED playlists — already-stored system
// playlists keep their rows until the next daily rebuild, which is why the
// shared ListPlaylistTracks read path is deliberately left unfiltered (it
// also serves user-curated playlists, where hiding a track the user added
// themselves would be wrong).
// M7 #352 slice 2: system-generated playlist queries. // M7 #352 slice 2: system-generated playlist queries.
// Active = had a play in the last 7 days. The cron iterates this list. // Active = had a play in the last 7 days. The cron iterates this list.
func (q *Queries) ListActiveUsersForSystemPlaylists(ctx context.Context) ([]pgtype.UUID, error) { func (q *Queries) ListActiveUsersForSystemPlaylists(ctx context.Context) ([]pgtype.UUID, error) {
@@ -298,7 +306,7 @@ recent7 AS (
COUNT(*) FILTER (WHERE pe.was_skipped = false) AS play_count, COUNT(*) FILTER (WHERE pe.was_skipped = false) AS play_count,
0 AS tier 0 AS tier
FROM play_events pe FROM play_events pe
JOIN tracks t ON t.id = pe.track_id JOIN tracks t ON t.id = pe.track_id AND t.missing_since IS NULL
WHERE pe.user_id = $1 WHERE pe.user_id = $1
AND pe.started_at > now() - INTERVAL '7 days' AND pe.started_at > now() - INTERVAL '7 days'
AND t.artist_id IS NOT NULL AND t.artist_id IS NOT NULL
@@ -309,7 +317,7 @@ recent30 AS (
COUNT(*) FILTER (WHERE pe.was_skipped = false) AS play_count, COUNT(*) FILTER (WHERE pe.was_skipped = false) AS play_count,
1 AS tier 1 AS tier
FROM play_events pe FROM play_events pe
JOIN tracks t ON t.id = pe.track_id JOIN tracks t ON t.id = pe.track_id AND t.missing_since IS NULL
WHERE pe.user_id = $1 WHERE pe.user_id = $1
AND pe.started_at > now() - INTERVAL '30 days' AND pe.started_at > now() - INTERVAL '30 days'
AND t.artist_id IS NOT NULL AND t.artist_id IS NOT NULL
@@ -320,7 +328,7 @@ alltime AS (
COUNT(*) FILTER (WHERE pe.was_skipped = false) AS play_count, COUNT(*) FILTER (WHERE pe.was_skipped = false) AS play_count,
2 AS tier 2 AS tier
FROM play_events pe FROM play_events pe
JOIN tracks t ON t.id = pe.track_id JOIN tracks t ON t.id = pe.track_id AND t.missing_since IS NULL
WHERE pe.user_id = $1 WHERE pe.user_id = $1
AND t.artist_id IS NOT NULL AND t.artist_id IS NOT NULL
GROUP BY t.artist_id GROUP BY t.artist_id
@@ -432,7 +440,7 @@ const pickTopPlayedTrackForArtistByUser = `-- name: PickTopPlayedTrackForArtistB
SELECT COALESCE( SELECT COALESCE(
(SELECT t.id (SELECT t.id
FROM play_events pe FROM play_events pe
JOIN tracks t ON t.id = pe.track_id JOIN tracks t ON t.id = pe.track_id AND t.missing_since IS NULL
WHERE pe.user_id = $1 WHERE pe.user_id = $1
AND t.artist_id = $2 AND t.artist_id = $2
AND pe.started_at > now() - INTERVAL '7 days' AND pe.started_at > now() - INTERVAL '7 days'
@@ -472,7 +480,7 @@ const pickTopPlayedTracksForUser = `-- name: PickTopPlayedTracksForUser :many
WITH recent AS ( WITH recent AS (
SELECT t.id, COUNT(*) AS c, 0 AS tier SELECT t.id, COUNT(*) AS c, 0 AS tier
FROM play_events pe FROM play_events pe
JOIN tracks t ON t.id = pe.track_id JOIN tracks t ON t.id = pe.track_id AND t.missing_since IS NULL
WHERE pe.user_id = $1 WHERE pe.user_id = $1
AND pe.started_at > now() - INTERVAL '30 days' AND pe.started_at > now() - INTERVAL '30 days'
AND pe.was_skipped = false AND pe.was_skipped = false
@@ -481,7 +489,7 @@ WITH recent AS (
alltime AS ( alltime AS (
SELECT t.id, COUNT(*) AS c, 1 AS tier SELECT t.id, COUNT(*) AS c, 1 AS tier
FROM play_events pe FROM play_events pe
JOIN tracks t ON t.id = pe.track_id JOIN tracks t ON t.id = pe.track_id AND t.missing_since IS NULL
WHERE pe.user_id = $1 WHERE pe.user_id = $1
AND pe.was_skipped = false AND pe.was_skipped = false
GROUP BY t.id GROUP BY t.id
internal/db/dbq/tracks.sql.go
+92 -8
View File
@@ -11,6 +11,24 @@ import (
"github.com/jackc/pgx/v5/pgtype" "github.com/jackc/pgx/v5/pgtype"
) )
const clearTracksMissing = `-- name: ClearTracksMissing :execrows
UPDATE tracks
SET missing_since = NULL
WHERE id = ANY($1::uuid[])
AND missing_since IS NOT NULL
`
// Clears the mark on rows whose file is back. Runs independently of the mtime
// skip check, so a file that reappears unchanged is un-marked even though the
// scanner skips re-reading its tags.
func (q *Queries) ClearTracksMissing(ctx context.Context, ids []pgtype.UUID) (int64, error) {
result, err := q.db.Exec(ctx, clearTracksMissing, ids)
if err != nil {
return 0, err
}
return result.RowsAffected(), nil
}
const countTracksByAlbum = `-- name: CountTracksByAlbum :one const countTracksByAlbum = `-- name: CountTracksByAlbum :one
SELECT count(*) FROM tracks WHERE album_id = $1 SELECT count(*) FROM tracks WHERE album_id = $1
` `
@@ -90,7 +108,7 @@ func (q *Queries) DeleteTrack(ctx context.Context, id pgtype.UUID) (DeleteTrackR
} }
const getTrackByID = `-- name: GetTrackByID :one const getTrackByID = `-- name: GetTrackByID :one
SELECT id, title, album_id, artist_id, track_number, disc_number, duration_ms, file_path, file_size, file_format, bitrate, mbid, genre, added_at, updated_at, tag_source, tag_sources_version, tag_read_version FROM tracks WHERE id = $1 SELECT id, title, album_id, artist_id, track_number, disc_number, duration_ms, file_path, file_size, file_format, bitrate, mbid, genre, added_at, updated_at, tag_source, tag_sources_version, tag_read_version, missing_since FROM tracks WHERE id = $1
` `
func (q *Queries) GetTrackByID(ctx context.Context, id pgtype.UUID) (Track, error) { func (q *Queries) GetTrackByID(ctx context.Context, id pgtype.UUID) (Track, error) {
@@ -115,12 +133,13 @@ func (q *Queries) GetTrackByID(ctx context.Context, id pgtype.UUID) (Track, erro
&i.TagSource, &i.TagSource,
&i.TagSourcesVersion, &i.TagSourcesVersion,
&i.TagReadVersion, &i.TagReadVersion,
&i.MissingSince,
) )
return i, err return i, err
} }
const getTrackByPath = `-- name: GetTrackByPath :one const getTrackByPath = `-- name: GetTrackByPath :one
SELECT id, title, album_id, artist_id, track_number, disc_number, duration_ms, file_path, file_size, file_format, bitrate, mbid, genre, added_at, updated_at, tag_source, tag_sources_version, tag_read_version FROM tracks WHERE file_path = $1 SELECT id, title, album_id, artist_id, track_number, disc_number, duration_ms, file_path, file_size, file_format, bitrate, mbid, genre, added_at, updated_at, tag_source, tag_sources_version, tag_read_version, missing_since FROM tracks WHERE file_path = $1
` `
func (q *Queries) GetTrackByPath(ctx context.Context, filePath string) (Track, error) { func (q *Queries) GetTrackByPath(ctx context.Context, filePath string) (Track, error) {
@@ -145,12 +164,13 @@ func (q *Queries) GetTrackByPath(ctx context.Context, filePath string) (Track, e
&i.TagSource, &i.TagSource,
&i.TagSourcesVersion, &i.TagSourcesVersion,
&i.TagReadVersion, &i.TagReadVersion,
&i.MissingSince,
) )
return i, err return i, err
} }
const getTracksByIDs = `-- name: GetTracksByIDs :many const getTracksByIDs = `-- name: GetTracksByIDs :many
SELECT id, title, album_id, artist_id, track_number, disc_number, duration_ms, file_path, file_size, file_format, bitrate, mbid, genre, added_at, updated_at, tag_source, tag_sources_version, tag_read_version FROM tracks WHERE id = ANY($1::uuid[]) SELECT id, title, album_id, artist_id, track_number, disc_number, duration_ms, file_path, file_size, file_format, bitrate, mbid, genre, added_at, updated_at, tag_source, tag_sources_version, tag_read_version, missing_since FROM tracks WHERE id = ANY($1::uuid[])
` `
// Batched lookup used by /api/library/sync to hydrate upsert payloads // Batched lookup used by /api/library/sync to hydrate upsert payloads
@@ -183,6 +203,7 @@ func (q *Queries) GetTracksByIDs(ctx context.Context, dollar_1 []pgtype.UUID) ([
&i.TagSource, &i.TagSource,
&i.TagSourcesVersion, &i.TagSourcesVersion,
&i.TagReadVersion, &i.TagReadVersion,
&i.MissingSince,
); err != nil { ); err != nil {
return nil, err return nil, err
} }
@@ -195,7 +216,7 @@ func (q *Queries) GetTracksByIDs(ctx context.Context, dollar_1 []pgtype.UUID) ([
} }
const listArtistTracksForUser = `-- name: ListArtistTracksForUser :many const listArtistTracksForUser = `-- name: ListArtistTracksForUser :many
SELECT t.id, t.title, t.album_id, t.artist_id, t.track_number, t.disc_number, t.duration_ms, t.file_path, t.file_size, t.file_format, t.bitrate, t.mbid, t.genre, t.added_at, t.updated_at, t.tag_source, t.tag_sources_version, t.tag_read_version, SELECT t.id, t.title, t.album_id, t.artist_id, t.track_number, t.disc_number, t.duration_ms, t.file_path, t.file_size, t.file_format, t.bitrate, t.mbid, t.genre, t.added_at, t.updated_at, t.tag_source, t.tag_sources_version, t.tag_read_version, t.missing_since,
albums.title AS album_title, albums.title AS album_title,
artists.name AS artist_name artists.name AS artist_name
FROM tracks t FROM tracks t
@@ -254,6 +275,7 @@ func (q *Queries) ListArtistTracksForUser(ctx context.Context, arg ListArtistTra
&i.Track.TagSource, &i.Track.TagSource,
&i.Track.TagSourcesVersion, &i.Track.TagSourcesVersion,
&i.Track.TagReadVersion, &i.Track.TagReadVersion,
&i.Track.MissingSince,
&i.AlbumTitle, &i.AlbumTitle,
&i.ArtistName, &i.ArtistName,
); err != nil { ); err != nil {
@@ -268,7 +290,7 @@ func (q *Queries) ListArtistTracksForUser(ctx context.Context, arg ListArtistTra
} }
const listRandomTracksForUser = `-- name: ListRandomTracksForUser :many const listRandomTracksForUser = `-- name: ListRandomTracksForUser :many
SELECT t.id, t.title, t.album_id, t.artist_id, t.track_number, t.disc_number, t.duration_ms, t.file_path, t.file_size, t.file_format, t.bitrate, t.mbid, t.genre, t.added_at, t.updated_at, t.tag_source, t.tag_sources_version, t.tag_read_version, SELECT t.id, t.title, t.album_id, t.artist_id, t.track_number, t.disc_number, t.duration_ms, t.file_path, t.file_size, t.file_format, t.bitrate, t.mbid, t.genre, t.added_at, t.updated_at, t.tag_source, t.tag_sources_version, t.tag_read_version, t.missing_since,
albums.title AS album_title, albums.title AS album_title,
artists.name AS artist_name artists.name AS artist_name
FROM tracks t FROM tracks t
@@ -324,6 +346,7 @@ func (q *Queries) ListRandomTracksForUser(ctx context.Context, arg ListRandomTra
&i.Track.TagSource, &i.Track.TagSource,
&i.Track.TagSourcesVersion, &i.Track.TagSourcesVersion,
&i.Track.TagReadVersion, &i.Track.TagReadVersion,
&i.Track.MissingSince,
&i.AlbumTitle, &i.AlbumTitle,
&i.ArtistName, &i.ArtistName,
); err != nil { ); err != nil {
@@ -337,8 +360,43 @@ func (q *Queries) ListRandomTracksForUser(ctx context.Context, arg ListRandomTra
return items, nil return items, nil
} }
const listTrackPathsForReconcile = `-- name: ListTrackPathsForReconcile :many
SELECT id, file_path, missing_since FROM tracks
`
type ListTrackPathsForReconcileRow struct {
ID pgtype.UUID
FilePath string
MissingSince pgtype.Timestamptz
}
// Every row's path + current missing mark, for the scanner's reconcile pass
// (#2523). Deliberately unfiltered and unpaged: reconcile has to compare the
// WHOLE table against what the walk saw, and a filtered subset would let rows
// outside it drift forever. Three narrow columns keep it cheap even on a
// library of a few hundred thousand tracks.
func (q *Queries) ListTrackPathsForReconcile(ctx context.Context) ([]ListTrackPathsForReconcileRow, error) {
rows, err := q.db.Query(ctx, listTrackPathsForReconcile)
if err != nil {
return nil, err
}
defer rows.Close()
var items []ListTrackPathsForReconcileRow
for rows.Next() {
var i ListTrackPathsForReconcileRow
if err := rows.Scan(&i.ID, &i.FilePath, &i.MissingSince); err != nil {
return nil, err
}
items = append(items, i)
}
if err := rows.Err(); err != nil {
return nil, err
}
return items, nil
}
const listTracksByAlbum = `-- name: ListTracksByAlbum :many const listTracksByAlbum = `-- name: ListTracksByAlbum :many
SELECT id, title, album_id, artist_id, track_number, disc_number, duration_ms, file_path, file_size, file_format, bitrate, mbid, genre, added_at, updated_at, tag_source, tag_sources_version, tag_read_version FROM tracks SELECT id, title, album_id, artist_id, track_number, disc_number, duration_ms, file_path, file_size, file_format, bitrate, mbid, genre, added_at, updated_at, tag_source, tag_sources_version, tag_read_version, missing_since FROM tracks
WHERE album_id = $1 WHERE album_id = $1
AND NOT EXISTS ( AND NOT EXISTS (
SELECT 1 FROM lidarr_quarantine q SELECT 1 FROM lidarr_quarantine q
@@ -383,6 +441,7 @@ func (q *Queries) ListTracksByAlbum(ctx context.Context, arg ListTracksByAlbumPa
&i.TagSource, &i.TagSource,
&i.TagSourcesVersion, &i.TagSourcesVersion,
&i.TagReadVersion, &i.TagReadVersion,
&i.MissingSince,
); err != nil { ); err != nil {
return nil, err return nil, err
} }
@@ -429,8 +488,31 @@ func (q *Queries) ListTracksMissingMbidWithPath(ctx context.Context, limit int32
return items, nil return items, nil
} }
const markTracksMissing = `-- name: MarkTracksMissing :execrows
UPDATE tracks
SET missing_since = now()
WHERE id = ANY($1::uuid[])
AND missing_since IS NULL
`
// Marks rows whose file the walk did not see. `missing_since IS NULL` in the
// predicate makes this idempotent: a row already marked keeps its ORIGINAL
// timestamp, so "how long has it been gone" survives repeated scans. Losing
// that would make any age-based cleanup policy meaningless.
//
// updated_at is deliberately NOT touched. It tracks content changes and gates
// the scanner's mtime skip; moving it here would make a returning file look
// newer than its own mtime and stop its tags being re-read.
func (q *Queries) MarkTracksMissing(ctx context.Context, ids []pgtype.UUID) (int64, error) {
result, err := q.db.Exec(ctx, markTracksMissing, ids)
if err != nil {
return 0, err
}
return result.RowsAffected(), nil
}
const searchTracks = `-- name: SearchTracks :many const searchTracks = `-- name: SearchTracks :many
SELECT id, title, album_id, artist_id, track_number, disc_number, duration_ms, file_path, file_size, file_format, bitrate, mbid, genre, added_at, updated_at, tag_source, tag_sources_version, tag_read_version FROM tracks SELECT id, title, album_id, artist_id, track_number, disc_number, duration_ms, file_path, file_size, file_format, bitrate, mbid, genre, added_at, updated_at, tag_source, tag_sources_version, tag_read_version, missing_since FROM tracks
WHERE title ILIKE '%' || $1::text || '%' WHERE title ILIKE '%' || $1::text || '%'
AND NOT EXISTS ( AND NOT EXISTS (
SELECT 1 FROM lidarr_quarantine q SELECT 1 FROM lidarr_quarantine q
@@ -482,6 +564,7 @@ func (q *Queries) SearchTracks(ctx context.Context, arg SearchTracksParams) ([]T
&i.TagSource, &i.TagSource,
&i.TagSourcesVersion, &i.TagSourcesVersion,
&i.TagReadVersion, &i.TagReadVersion,
&i.MissingSince,
); err != nil { ); err != nil {
return nil, err return nil, err
} }
@@ -533,7 +616,7 @@ ON CONFLICT (file_path) DO UPDATE SET
-- next scan can short-circuit them again (#2499). -- next scan can short-circuit them again (#2499).
tag_read_version = EXCLUDED.tag_read_version, tag_read_version = EXCLUDED.tag_read_version,
updated_at = now() updated_at = now()
RETURNING id, title, album_id, artist_id, track_number, disc_number, duration_ms, file_path, file_size, file_format, bitrate, mbid, genre, added_at, updated_at, tag_source, tag_sources_version, tag_read_version RETURNING id, title, album_id, artist_id, track_number, disc_number, duration_ms, file_path, file_size, file_format, bitrate, mbid, genre, added_at, updated_at, tag_source, tag_sources_version, tag_read_version, missing_since
` `
type UpsertTrackParams struct { type UpsertTrackParams struct {
@@ -589,6 +672,7 @@ func (q *Queries) UpsertTrack(ctx context.Context, arg UpsertTrackParams) (Track
&i.TagSource, &i.TagSource,
&i.TagSourcesVersion, &i.TagSourcesVersion,
&i.TagReadVersion, &i.TagReadVersion,
&i.MissingSince,
) )
return i, err return i, err
} }
internal/db/migrations/0055_track_missing_since.down.sql
+4
View File
@@ -0,0 +1,4 @@
DROP INDEX IF EXISTS tracks_missing_since_idx;
ALTER TABLE tracks
DROP COLUMN missing_since;
internal/db/migrations/0055_track_missing_since.up.sql
+25
View File
@@ -0,0 +1,25 @@
-- Marks a track whose file the scanner could no longer find (#2523).
--
-- NULL means present. A timestamp means the file was absent as of that scan,
-- and is the point from which "how long has this been gone" is measured — which
-- is what a later cleanup pass needs in order to require a grace period rather
-- than deleting on a single missed stat.
--
-- Deliberately a nullable timestamp rather than a boolean: "missing" is not a
-- state we want to act on immediately, and the age is the only thing that makes
-- an automated deletion safe to reason about.
--
-- No default and no backfill. Existing rows start NULL (present) and the next
-- full scan sets the mark where it belongs — a migration cannot check the
-- filesystem, and guessing here would mark the whole library on a server whose
-- media volume happens to be detached at upgrade time.
ALTER TABLE tracks
ADD COLUMN missing_since timestamptz;
-- Partial index: the only query that filters on this column positively is the
-- admin "what's missing" list, which is a small set. Playback and browse
-- queries filter `missing_since IS NULL`, which matches nearly every row and is
-- better served by a sequential scan than an index lookup.
CREATE INDEX tracks_missing_since_idx
ON tracks (missing_since)
WHERE missing_since IS NOT NULL;
internal/db/queries/browse.sql
+17
View File
@@ -1,3 +1,15 @@
-- Every query in this file filters `tracks.missing_since IS NULL` (#2523).
-- A row whose file has vanished keeps its genre forever — the scanner walks the
-- filesystem, so it never revisits a path that no longer exists — which is how
-- pre-#2499 welded genres survived a full re-scan and kept showing in the index.
-- Browsing is a way of finding something to play, so a track that cannot play
-- should not shape it.
--
-- Year queries below join albums only and are deliberately left alone: an album
-- is still a real release even if some of its tracks are gone. An album whose
-- EVERY track is missing will linger on the year axis; that's a narrower case,
-- tracked with the rest of the cleanup work.
-- name: ListGenresWithCount :many -- name: ListGenresWithCount :many
-- Genre browse index (#367). -- Genre browse index (#367).
-- --
@@ -22,6 +34,7 @@ SELECT trim(g.genre) AS genre, COUNT(DISTINCT tracks.id)::bigint AS track_count
FROM tracks FROM tracks
JOIN LATERAL regexp_split_to_table(coalesce(tracks.genre, ''), '[;,]') AS g(genre) ON true JOIN LATERAL regexp_split_to_table(coalesce(tracks.genre, ''), '[;,]') AS g(genre) ON true
WHERE trim(g.genre) <> '' WHERE trim(g.genre) <> ''
AND tracks.missing_since IS NULL
GROUP BY trim(g.genre) GROUP BY trim(g.genre)
-- Ordered by the expression, not the output alias: `ORDER BY genre` is -- Ordered by the expression, not the output alias: `ORDER BY genre` is
-- ambiguous between the alias and tracks.genre, and sqlc rejects it. -- ambiguous between the alias and tracks.genre, and sqlc rejects it.
@@ -41,6 +54,7 @@ WHERE EXISTS (
FROM tracks FROM tracks
JOIN LATERAL regexp_split_to_table(coalesce(tracks.genre, ''), '[;,]') AS g(genre) ON true JOIN LATERAL regexp_split_to_table(coalesce(tracks.genre, ''), '[;,]') AS g(genre) ON true
WHERE tracks.album_id = albums.id WHERE tracks.album_id = albums.id
AND tracks.missing_since IS NULL
AND trim(g.genre) = trim(sqlc.arg(genre)::text) AND trim(g.genre) = trim(sqlc.arg(genre)::text)
) )
ORDER BY albums.sort_title, albums.id ORDER BY albums.sort_title, albums.id
@@ -56,6 +70,7 @@ WHERE EXISTS (
FROM tracks FROM tracks
JOIN LATERAL regexp_split_to_table(coalesce(tracks.genre, ''), '[;,]') AS g(genre) ON true JOIN LATERAL regexp_split_to_table(coalesce(tracks.genre, ''), '[;,]') AS g(genre) ON true
WHERE tracks.album_id = albums.id WHERE tracks.album_id = albums.id
AND tracks.missing_since IS NULL
AND trim(g.genre) = trim(sqlc.arg(genre)::text) AND trim(g.genre) = trim(sqlc.arg(genre)::text)
); );
@@ -96,6 +111,7 @@ SELECT DISTINCT trim(g.genre) AS genre
FROM tracks FROM tracks
JOIN LATERAL regexp_split_to_table(coalesce(tracks.genre, ''), '[;,]') AS g(genre) ON true JOIN LATERAL regexp_split_to_table(coalesce(tracks.genre, ''), '[;,]') AS g(genre) ON true
WHERE tracks.album_id = $1 AND trim(g.genre) <> '' WHERE tracks.album_id = $1 AND trim(g.genre) <> ''
AND tracks.missing_since IS NULL
ORDER BY trim(g.genre); ORDER BY trim(g.genre);
-- name: ListGenresForArtist :many -- name: ListGenresForArtist :many
@@ -106,4 +122,5 @@ SELECT DISTINCT trim(g.genre) AS genre
FROM tracks FROM tracks
JOIN LATERAL regexp_split_to_table(coalesce(tracks.genre, ''), '[;,]') AS g(genre) ON true JOIN LATERAL regexp_split_to_table(coalesce(tracks.genre, ''), '[;,]') AS g(genre) ON true
WHERE tracks.artist_id = $1 AND trim(g.genre) <> '' WHERE tracks.artist_id = $1 AND trim(g.genre) <> ''
AND tracks.missing_since IS NULL
ORDER BY trim(g.genre); ORDER BY trim(g.genre);
internal/db/queries/discover.sql
+8 -4
View File
@@ -29,7 +29,8 @@ dormant_artists AS (
SELECT t.id, t.album_id, t.artist_id SELECT t.id, t.album_id, t.artist_id
FROM tracks t FROM tracks t
JOIN dormant_artists da ON da.id = t.artist_id JOIN dormant_artists da ON da.id = t.artist_id
WHERE NOT EXISTS ( WHERE t.missing_since IS NULL -- #2523: never offer a file that is gone
AND NOT EXISTS (
SELECT 1 FROM play_events pe SELECT 1 FROM play_events pe
WHERE pe.user_id = $1 WHERE pe.user_id = $1
AND pe.track_id = t.id AND pe.track_id = t.id
@@ -60,7 +61,8 @@ SELECT t.id, t.album_id, t.artist_id
SELECT t.id, t.album_id, t.artist_id SELECT t.id, t.album_id, t.artist_id
FROM general_likes gl FROM general_likes gl
JOIN tracks t ON t.id = gl.track_id JOIN tracks t ON t.id = gl.track_id
WHERE gl.user_id != $1 WHERE t.missing_since IS NULL -- #2523: never offer a file that is gone
AND gl.user_id != $1
AND NOT EXISTS ( AND NOT EXISTS (
SELECT 1 FROM play_events pe SELECT 1 FROM play_events pe
WHERE pe.user_id = $1 WHERE pe.user_id = $1
@@ -86,7 +88,8 @@ SELECT t.id, t.album_id, t.artist_id
-- $1 = user_id, $2 = date string for md5 ordering. -- $1 = user_id, $2 = date string for md5 ordering.
SELECT t.id, t.album_id, t.artist_id SELECT t.id, t.album_id, t.artist_id
FROM tracks t FROM tracks t
WHERE NOT EXISTS ( WHERE t.missing_since IS NULL -- #2523: never offer a file that is gone
AND NOT EXISTS (
SELECT 1 FROM play_events pe SELECT 1 FROM play_events pe
WHERE pe.user_id = $1 WHERE pe.user_id = $1
AND pe.track_id = t.id AND pe.track_id = t.id
@@ -117,7 +120,8 @@ SELECT t.id, t.album_id, t.artist_id
FROM tracks t FROM tracks t
JOIN LATERAL regexp_split_to_table(coalesce(t.genre, ''), '[;,]') AS g_split(g) ON true JOIN LATERAL regexp_split_to_table(coalesce(t.genre, ''), '[;,]') AS g_split(g) ON true
JOIN taste_profile_tags nt ON nt.user_id = $1 AND trim(g_split.g) = nt.tag JOIN taste_profile_tags nt ON nt.user_id = $1 AND trim(g_split.g) = nt.tag
WHERE nt.weight > 0 WHERE t.missing_since IS NULL -- #2523: never offer a file that is gone
AND nt.weight > 0
AND trim(g_split.g) <> '' AND trim(g_split.g) <> ''
AND NOT EXISTS ( AND NOT EXISTS (
SELECT 1 FROM play_events pe SELECT 1 FROM play_events pe
internal/db/queries/recommendation.sql
+5 -2
View File
@@ -24,6 +24,7 @@ LEFT JOIN LATERAL (
WHERE user_id = $1 AND track_id = t.id WHERE user_id = $1 AND track_id = t.id
) pe ON true ) pe ON true
WHERE t.id <> $2 WHERE t.id <> $2
AND t.missing_since IS NULL -- #2523: never offer a file that is gone
AND NOT EXISTS ( AND NOT EXISTS (
SELECT 1 FROM play_events SELECT 1 FROM play_events
WHERE user_id = $1 AND track_id = t.id WHERE user_id = $1 AND track_id = t.id
@@ -177,7 +178,7 @@ FROM (
UNION ALL SELECT track_id, sim_score FROM coplay_artists UNION ALL SELECT track_id, sim_score FROM coplay_artists
UNION ALL SELECT track_id, sim_score FROM random_fill UNION ALL SELECT track_id, sim_score FROM random_fill
) u ) u
JOIN tracks t ON t.id = u.track_id JOIN tracks t ON t.id = u.track_id AND t.missing_since IS NULL -- #2523: never offer a file that is gone
JOIN albums al ON al.id = t.album_id JOIN albums al ON al.id = t.album_id
LEFT JOIN general_likes l ON l.user_id = $1 AND l.track_id = t.id LEFT JOIN general_likes l ON l.user_id = $1 AND l.track_id = t.id
LEFT JOIN LATERAL ( LEFT JOIN LATERAL (
@@ -382,7 +383,8 @@ FROM plays p
JOIN tracks t ON t.id = p.track_id JOIN tracks t ON t.id = p.track_id
JOIN albums ON albums.id = t.album_id JOIN albums ON albums.id = t.album_id
JOIN artists ON artists.id = t.artist_id JOIN artists ON artists.id = t.artist_id
WHERE NOT EXISTS ( WHERE t.missing_since IS NULL -- #2523: never offer a file that is gone
AND NOT EXISTS (
SELECT 1 FROM lidarr_quarantine q SELECT 1 FROM lidarr_quarantine q
WHERE q.user_id = $1 AND q.track_id = t.id WHERE q.user_id = $1 AND q.track_id = t.id
) )
@@ -408,6 +410,7 @@ JOIN tracks t ON t.id = p.track_id
JOIN albums ON albums.id = t.album_id JOIN albums ON albums.id = t.album_id
JOIN artists ON artists.id = t.artist_id JOIN artists ON artists.id = t.artist_id
WHERE t.artist_id = sqlc.arg(artist_id) WHERE t.artist_id = sqlc.arg(artist_id)
AND t.missing_since IS NULL -- #2523: never offer a file that is gone
AND NOT EXISTS ( AND NOT EXISTS (
SELECT 1 FROM lidarr_quarantine q SELECT 1 FROM lidarr_quarantine q
WHERE q.user_id = sqlc.arg(user_id) AND q.track_id = t.id WHERE q.user_id = sqlc.arg(user_id) AND q.track_id = t.id
internal/db/queries/system_mixes.sql
+10 -5
View File
@@ -40,7 +40,8 @@ SELECT t.id, t.album_id, t.artist_id
JOIN affinity_artists aa ON aa.artist_id = t.artist_id JOIN affinity_artists aa ON aa.artist_id = t.artist_id
LEFT JOIN play_counts pc ON pc.track_id = t.id LEFT JOIN play_counts pc ON pc.track_id = t.id
LEFT JOIN skip_counts sc ON sc.track_id = t.id LEFT JOIN skip_counts sc ON sc.track_id = t.id
WHERE COALESCE(pc.c, 0) <= 2 WHERE t.missing_since IS NULL -- #2523: never offer a file that is gone
AND COALESCE(pc.c, 0) <= 2
AND COALESCE(sc.c, 0) < 2 AND COALESCE(sc.c, 0) < 2
AND NOT EXISTS ( AND NOT EXISTS (
SELECT 1 FROM lidarr_quarantine q SELECT 1 FROM lidarr_quarantine q
@@ -70,7 +71,8 @@ WITH stats AS (
SELECT t.id, t.album_id, t.artist_id SELECT t.id, t.album_id, t.artist_id
FROM tracks t FROM tracks t
JOIN stats s ON s.track_id = t.id JOIN stats s ON s.track_id = t.id
WHERE s.c >= 3 WHERE t.missing_since IS NULL -- #2523: never offer a file that is gone
AND s.c >= 3
AND s.last_at <= now() - interval '30 days' AND s.last_at <= now() - interval '30 days'
AND NOT EXISTS ( AND NOT EXISTS (
SELECT 1 FROM lidarr_quarantine q SELECT 1 FROM lidarr_quarantine q
@@ -149,7 +151,8 @@ albums_tiered AS (
SELECT t.id, t.album_id, t.artist_id, alt.tier::int AS tier SELECT t.id, t.album_id, t.artist_id, alt.tier::int AS tier
FROM tracks t FROM tracks t
JOIN albums_tiered alt ON alt.album_id = t.album_id JOIN albums_tiered alt ON alt.album_id = t.album_id
WHERE NOT EXISTS ( WHERE t.missing_since IS NULL -- #2523: never offer a file that is gone
AND NOT EXISTS (
SELECT 1 FROM lidarr_quarantine q SELECT 1 FROM lidarr_quarantine q
WHERE q.user_id = $1 AND q.track_id = t.id WHERE q.user_id = $1 AND q.track_id = t.id
) )
@@ -187,7 +190,8 @@ WITH windowed AS (
SELECT t.id, t.album_id, t.artist_id SELECT t.id, t.album_id, t.artist_id
FROM tracks t FROM tracks t
JOIN windowed w ON w.track_id = t.id JOIN windowed w ON w.track_id = t.id
WHERE NOT EXISTS ( WHERE t.missing_since IS NULL -- #2523: never offer a file that is gone
AND NOT EXISTS (
SELECT 1 FROM lidarr_quarantine q SELECT 1 FROM lidarr_quarantine q
WHERE q.user_id = $1 AND q.track_id = t.id WHERE q.user_id = $1 AND q.track_id = t.id
) )
@@ -240,7 +244,8 @@ SELECT t.id, t.album_id, t.artist_id, alt.tier::int AS tier
FROM tracks t FROM tracks t
JOIN albums al ON al.id = t.album_id JOIN albums al ON al.id = t.album_id
JOIN albums_tiered alt ON alt.album_id = al.id JOIN albums_tiered alt ON alt.album_id = al.id
WHERE alt.tier IS NOT NULL WHERE t.missing_since IS NULL -- #2523: never offer a file that is gone
AND alt.tier IS NOT NULL
AND NOT EXISTS (SELECT 1 FROM attempted a WHERE a.track_id = t.id) AND NOT EXISTS (SELECT 1 FROM attempted a WHERE a.track_id = t.id)
AND NOT EXISTS ( AND NOT EXISTS (
SELECT 1 FROM lidarr_quarantine q SELECT 1 FROM lidarr_quarantine q
internal/db/queries/system_playlists.sql
+14 -6
View File
@@ -1,3 +1,11 @@
-- Track picks here join `tracks ... AND t.missing_since IS NULL` (#2523): a
-- seed or For-You candidate has to be something that can actually play. Note
-- this only affects newly GENERATED playlists — already-stored system
-- playlists keep their rows until the next daily rebuild, which is why the
-- shared ListPlaylistTracks read path is deliberately left unfiltered (it
-- also serves user-curated playlists, where hiding a track the user added
-- themselves would be wrong).
-- M7 #352 slice 2: system-generated playlist queries. -- M7 #352 slice 2: system-generated playlist queries.
-- name: ListActiveUsersForSystemPlaylists :many -- name: ListActiveUsersForSystemPlaylists :many
@@ -72,7 +80,7 @@ recent7 AS (
COUNT(*) FILTER (WHERE pe.was_skipped = false) AS play_count, COUNT(*) FILTER (WHERE pe.was_skipped = false) AS play_count,
0 AS tier 0 AS tier
FROM play_events pe FROM play_events pe
JOIN tracks t ON t.id = pe.track_id JOIN tracks t ON t.id = pe.track_id AND t.missing_since IS NULL
WHERE pe.user_id = $1 WHERE pe.user_id = $1
AND pe.started_at > now() - INTERVAL '7 days' AND pe.started_at > now() - INTERVAL '7 days'
AND t.artist_id IS NOT NULL AND t.artist_id IS NOT NULL
@@ -83,7 +91,7 @@ recent30 AS (
COUNT(*) FILTER (WHERE pe.was_skipped = false) AS play_count, COUNT(*) FILTER (WHERE pe.was_skipped = false) AS play_count,
1 AS tier 1 AS tier
FROM play_events pe FROM play_events pe
JOIN tracks t ON t.id = pe.track_id JOIN tracks t ON t.id = pe.track_id AND t.missing_since IS NULL
WHERE pe.user_id = $1 WHERE pe.user_id = $1
AND pe.started_at > now() - INTERVAL '30 days' AND pe.started_at > now() - INTERVAL '30 days'
AND t.artist_id IS NOT NULL AND t.artist_id IS NOT NULL
@@ -94,7 +102,7 @@ alltime AS (
COUNT(*) FILTER (WHERE pe.was_skipped = false) AS play_count, COUNT(*) FILTER (WHERE pe.was_skipped = false) AS play_count,
2 AS tier 2 AS tier
FROM play_events pe FROM play_events pe
JOIN tracks t ON t.id = pe.track_id JOIN tracks t ON t.id = pe.track_id AND t.missing_since IS NULL
WHERE pe.user_id = $1 WHERE pe.user_id = $1
AND t.artist_id IS NOT NULL AND t.artist_id IS NOT NULL
GROUP BY t.artist_id GROUP BY t.artist_id
@@ -139,7 +147,7 @@ SELECT c.artist_id,
WITH recent AS ( WITH recent AS (
SELECT t.id, COUNT(*) AS c, 0 AS tier SELECT t.id, COUNT(*) AS c, 0 AS tier
FROM play_events pe FROM play_events pe
JOIN tracks t ON t.id = pe.track_id JOIN tracks t ON t.id = pe.track_id AND t.missing_since IS NULL
WHERE pe.user_id = $1 WHERE pe.user_id = $1
AND pe.started_at > now() - INTERVAL '30 days' AND pe.started_at > now() - INTERVAL '30 days'
AND pe.was_skipped = false AND pe.was_skipped = false
@@ -148,7 +156,7 @@ WITH recent AS (
alltime AS ( alltime AS (
SELECT t.id, COUNT(*) AS c, 1 AS tier SELECT t.id, COUNT(*) AS c, 1 AS tier
FROM play_events pe FROM play_events pe
JOIN tracks t ON t.id = pe.track_id JOIN tracks t ON t.id = pe.track_id AND t.missing_since IS NULL
WHERE pe.user_id = $1 WHERE pe.user_id = $1
AND pe.was_skipped = false AND pe.was_skipped = false
GROUP BY t.id GROUP BY t.id
@@ -181,7 +189,7 @@ SELECT id
SELECT COALESCE( SELECT COALESCE(
(SELECT t.id (SELECT t.id
FROM play_events pe FROM play_events pe
JOIN tracks t ON t.id = pe.track_id JOIN tracks t ON t.id = pe.track_id AND t.missing_since IS NULL
WHERE pe.user_id = $1 WHERE pe.user_id = $1
AND t.artist_id = $2 AND t.artist_id = $2
AND pe.started_at > now() - INTERVAL '7 days' AND pe.started_at > now() - INTERVAL '7 days'
internal/db/queries/tracks.sql
+31
View File
@@ -137,3 +137,34 @@ RETURNING id, album_id, artist_id, file_path, mbid;
-- Batched lookup used by /api/library/sync to hydrate upsert payloads -- Batched lookup used by /api/library/sync to hydrate upsert payloads
-- (#357). Mirror of GetArtistsByIDs. -- (#357). Mirror of GetArtistsByIDs.
SELECT * FROM tracks WHERE id = ANY($1::uuid[]); SELECT * FROM tracks WHERE id = ANY($1::uuid[]);
-- name: ListTrackPathsForReconcile :many
-- Every row's path + current missing mark, for the scanner's reconcile pass
-- (#2523). Deliberately unfiltered and unpaged: reconcile has to compare the
-- WHOLE table against what the walk saw, and a filtered subset would let rows
-- outside it drift forever. Three narrow columns keep it cheap even on a
-- library of a few hundred thousand tracks.
SELECT id, file_path, missing_since FROM tracks;
-- name: MarkTracksMissing :execrows
-- Marks rows whose file the walk did not see. `missing_since IS NULL` in the
-- predicate makes this idempotent: a row already marked keeps its ORIGINAL
-- timestamp, so "how long has it been gone" survives repeated scans. Losing
-- that would make any age-based cleanup policy meaningless.
--
-- updated_at is deliberately NOT touched. It tracks content changes and gates
-- the scanner's mtime skip; moving it here would make a returning file look
-- newer than its own mtime and stop its tags being re-read.
UPDATE tracks
SET missing_since = now()
WHERE id = ANY(sqlc.arg(ids)::uuid[])
AND missing_since IS NULL;
-- name: ClearTracksMissing :execrows
-- Clears the mark on rows whose file is back. Runs independently of the mtime
-- skip check, so a file that reappears unchanged is un-marked even though the
-- scanner skips re-reading its tags.
UPDATE tracks
SET missing_since = NULL
WHERE id = ANY(sqlc.arg(ids)::uuid[])
AND missing_since IS NOT NULL;
internal/library/reconcile.go
+155
View File
@@ -0,0 +1,155 @@
package library
import (
"context"
"errors"
"fmt"
"os"
"github.com/jackc/pgx/v5/pgtype"
"git.fabledsword.com/bvandeusen/minstrel/internal/db/dbq"
)
// trackReconciler is the slice of dbq.Queries reconcileMissing needs. Narrowed
// to an interface so the guard logic — which is the part that can do damage —
// is unit-testable against a fake without a database.
type trackReconciler interface {
ListTrackPathsForReconcile(ctx context.Context) ([]dbq.ListTrackPathsForReconcileRow, error)
MarkTracksMissing(ctx context.Context, ids []pgtype.UUID) (int64, error)
ClearTracksMissing(ctx context.Context, ids []pgtype.UUID) (int64, error)
}
// Reconcile marks tracks whose files have disappeared (#2523).
//
// Why this exists: nothing in Minstrel used to notice a deleted file. The walk
// only visits paths that exist, so a row whose file is gone was never scanned,
// never errored, never counted — permanently invisible. The watcher ignores
// removals by design (see classifyEvent), and the safety-net scan is the same
// walk, so it covers additions only. Rows accumulated forever, kept being
// offered to recommendations, and failed at playback.
//
// Why it MARKS rather than deletes: a missing file is a claim about the
// filesystem, and the filesystem lies transiently — an unmounted volume, a
// network-storage blip, a container that started before its media mount
// attached. Every other sweep in this codebase (internal/gc) resolves a truth
// *inside* the database and is safe to run blind. This one isn't, so the
// destructive step is deliberately not here. Marking is reversible: the next
// good scan clears it.
// missingMarkMaxFraction caps how much of the library one reconcile may newly
// mark missing. A partially-attached mount is the failure this defends against:
// the roots resolve, the walk succeeds, and it legitimately sees only part of
// the library — evidence indistinguishable from a mass deletion.
//
// A quarter is deliberately conservative. A genuine bulk deletion trips it and
// gets logged rather than applied, which needs a second scan (or operator
// action) to take effect. That's the right trade: the cost of over-refusing is
// a stale row and a log line, and the cost of over-marking is a chunk of the
// library silently vanishing from every mix.
const missingMarkMaxFraction = 0.25
// reconcileMissing diffs the paths the walk saw against every row in the table.
// Rows not seen get marked; rows seen that carry a mark get cleared.
//
// seen must come from a COMPLETE walk of every configured root. Callers with a
// partial view must not call this.
func (s *Scanner) reconcileMissing(
ctx context.Context, q trackReconciler, seen map[string]struct{}, stats *Stats,
) error {
if err := s.verifyRootsPresent(); err != nil {
return err
}
// Roots resolved but the walk found nothing. Either the library is genuinely
// empty — in which case there is nothing to reconcile — or the mount is
// hollow. Both mean: don't act.
if len(seen) == 0 {
return errors.New("walk saw no audio files; refusing to reconcile")
}
rows, err := q.ListTrackPathsForReconcile(ctx)
if err != nil {
return fmt.Errorf("list track paths: %w", err)
}
if len(rows) == 0 {
return nil
}
var toMark, toClear []pgtype.UUID
for _, row := range rows {
_, present := seen[row.FilePath]
switch {
case !present && !row.MissingSince.Valid:
toMark = append(toMark, row.ID)
case present && row.MissingSince.Valid:
toClear = append(toClear, row.ID)
}
}
// Clear before marking, and unconditionally. Restoring a file is never the
// dangerous direction, so it must not be blocked by the guard below —
// otherwise a library that tripped the cap once could never recover its
// marks even after the mount came back.
if len(toClear) > 0 {
n, err := q.ClearTracksMissing(ctx, toClear)
if err != nil {
return fmt.Errorf("clear missing marks: %w", err)
}
stats.Restored = int(n)
s.logger.Info("library scan: files returned", "count", n)
}
if len(toMark) == 0 {
return nil
}
if fraction := float64(len(toMark)) / float64(len(rows)); fraction > missingMarkMaxFraction {
return fmt.Errorf(
"refusing to mark %d of %d tracks missing (%.0f%% > %.0f%% cap): "+
"this looks like an unavailable mount rather than a deletion",
len(toMark), len(rows), fraction*100, missingMarkMaxFraction*100,
)
}
n, err := q.MarkTracksMissing(ctx, toMark)
if err != nil {
return fmt.Errorf("mark tracks missing: %w", err)
}
stats.Missing = int(n)
// Warn, not Info: every one of these is a library entry the operator
// probably didn't intend to lose, and the only place it surfaces today is
// this line.
s.logger.Warn("library scan: tracks marked missing (files not found)",
"count", n, "library_total", len(rows))
return nil
}
// verifyRootsPresent is the first and most important guard. If a configured root
// doesn't resolve to a readable directory, the walk beneath it found nothing and
// every row under it would look deleted. An unmounted media volume is the
// obvious case, and it is common enough — a container restart racing its volume
// mount does exactly this.
func (s *Scanner) verifyRootsPresent() error {
if len(s.paths) == 0 {
return errors.New("no scan roots configured")
}
for _, root := range s.paths {
info, err := os.Stat(root)
if err != nil {
return fmt.Errorf("scan root %q unavailable: %w", root, err)
}
if !info.IsDir() {
return fmt.Errorf("scan root %q is not a directory", root)
}
entries, err := os.ReadDir(root)
if err != nil {
return fmt.Errorf("scan root %q unreadable: %w", root, err)
}
// An empty root is the signature of a mount point with nothing mounted
// on it. `os.Stat` succeeds on the bare directory, so this is the only
// cheap way to tell the two apart.
if len(entries) == 0 {
return fmt.Errorf("scan root %q is empty; refusing to reconcile", root)
}
}
return nil
}
internal/library/reconcile_test.go
+326
View File
@@ -0,0 +1,326 @@
package library
import (
"context"
"errors"
"fmt"
"io"
"log/slog"
"os"
"path/filepath"
"testing"
"github.com/jackc/pgx/v5/pgtype"
"git.fabledsword.com/bvandeusen/minstrel/internal/db/dbq"
)
// fakeReconciler records what reconcileMissing decided to do, so the guards can
// be tested without a database. The guards are the whole point of this pass —
// they are what stands between an unmounted volume and the library disappearing
// from every mix — so they get tested directly rather than via integration.
type fakeReconciler struct {
rows []dbq.ListTrackPathsForReconcileRow
marked []pgtype.UUID
cleared []pgtype.UUID
listErr error
markErr error
clearErr error
}
func (f *fakeReconciler) ListTrackPathsForReconcile(context.Context) ([]dbq.ListTrackPathsForReconcileRow, error) {
return f.rows, f.listErr
}
func (f *fakeReconciler) MarkTracksMissing(_ context.Context, ids []pgtype.UUID) (int64, error) {
if f.markErr != nil {
return 0, f.markErr
}
f.marked = append(f.marked, ids...)
return int64(len(ids)), nil
}
func (f *fakeReconciler) ClearTracksMissing(_ context.Context, ids []pgtype.UUID) (int64, error) {
if f.clearErr != nil {
return 0, f.clearErr
}
f.cleared = append(f.cleared, ids...)
return int64(len(ids)), nil
}
// Compile-time proof the real queries still satisfy what reconcile needs — the
// interface exists to narrow dbq.Queries, not to diverge from it.
var _ trackReconciler = (*dbq.Queries)(nil)
func testUUID(n byte) pgtype.UUID {
var u pgtype.UUID
u.Bytes[15] = n
u.Valid = true
return u
}
func markedAt() pgtype.Timestamptz {
return pgtype.Timestamptz{Valid: true}
}
func row(n byte, path string, missing bool) dbq.ListTrackPathsForReconcileRow {
r := dbq.ListTrackPathsForReconcileRow{ID: testUUID(n), FilePath: path}
if missing {
r.MissingSince = markedAt()
}
return r
}
// populatedRoot returns a directory containing one file, so verifyRootsPresent
// treats it as a real, mounted library root.
func populatedRoot(t *testing.T) string {
t.Helper()
dir := t.TempDir()
if err := os.WriteFile(filepath.Join(dir, "a.mp3"), []byte("x"), 0o600); err != nil {
t.Fatal(err)
}
return dir
}
func testScanner(t *testing.T, roots ...string) *Scanner {
t.Helper()
return &Scanner{
logger: slog.New(slog.NewTextHandler(io.Discard, nil)),
paths: roots,
}
}
func TestReconcileMissing_MarksRowsTheWalkDidNotSee(t *testing.T) {
root := populatedRoot(t)
s := testScanner(t, root)
// 10 rows with 2 absent — 20%, deliberately under missingMarkMaxFraction so
// this exercises marking rather than the cap. (An earlier version of this
// test used 2-of-4 and was really testing the guard by accident.)
rows := make([]dbq.ListTrackPathsForReconcileRow, 0, 10)
seen := map[string]struct{}{}
for i := 0; i < 10; i++ {
p := fmt.Sprintf("/music/track-%02d.mp3", i)
rows = append(rows, row(byte(i), p, false))
if i >= 2 {
seen[p] = struct{}{}
}
}
q := &fakeReconciler{rows: rows}
var stats Stats
if err := s.reconcileMissing(context.Background(), q, seen, &stats); err != nil {
t.Fatalf("reconcile: %v", err)
}
if len(q.marked) != 2 {
t.Fatalf("marked %d rows, want 2", len(q.marked))
}
if q.marked[0] != testUUID(0) || q.marked[1] != testUUID(1) {
t.Errorf("marked the wrong rows: %v", q.marked)
}
if stats.Missing != 2 {
t.Errorf("stats.Missing = %d, want 2", stats.Missing)
}
if len(q.cleared) != 0 {
t.Errorf("cleared %d rows, want 0", len(q.cleared))
}
}
func TestReconcileMissing_ClearsRowsWhoseFileReturned(t *testing.T) {
root := populatedRoot(t)
s := testScanner(t, root)
q := &fakeReconciler{rows: []dbq.ListTrackPathsForReconcileRow{
row(1, "/music/back.mp3", true),
row(2, "/music/still-here.mp3", false),
}}
seen := map[string]struct{}{
"/music/back.mp3": {},
"/music/still-here.mp3": {},
}
var stats Stats
if err := s.reconcileMissing(context.Background(), q, seen, &stats); err != nil {
t.Fatalf("reconcile: %v", err)
}
if len(q.cleared) != 1 || q.cleared[0] != testUUID(1) {
t.Fatalf("cleared = %v, want just row 1", q.cleared)
}
if stats.Restored != 1 {
t.Errorf("stats.Restored = %d, want 1", stats.Restored)
}
if len(q.marked) != 0 {
t.Errorf("marked %d rows, want 0", len(q.marked))
}
}
// An already-marked row must not be re-marked: the timestamp is the "how long
// has this been gone" clock that any future cleanup policy depends on.
func TestReconcileMissing_DoesNotRemarkAlreadyMissingRows(t *testing.T) {
root := populatedRoot(t)
s := testScanner(t, root)
q := &fakeReconciler{rows: []dbq.ListTrackPathsForReconcileRow{
row(1, "/music/long-gone.mp3", true),
row(2, "/music/present.mp3", false),
}}
seen := map[string]struct{}{"/music/present.mp3": {}}
var stats Stats
if err := s.reconcileMissing(context.Background(), q, seen, &stats); err != nil {
t.Fatalf("reconcile: %v", err)
}
if len(q.marked) != 0 {
t.Errorf("re-marked an already-missing row: %v", q.marked)
}
if len(q.cleared) != 0 {
t.Errorf("cleared = %v, want none", q.cleared)
}
}
// The guard that matters most. A half-attached mount makes the walk succeed
// while seeing only part of the library — evidence indistinguishable from a mass
// deletion, so reconcile must refuse rather than guess.
func TestReconcileMissing_RefusesWhenTooMuchWouldBeMarked(t *testing.T) {
root := populatedRoot(t)
s := testScanner(t, root)
rows := make([]dbq.ListTrackPathsForReconcileRow, 0, 100)
seen := map[string]struct{}{}
for i := 0; i < 100; i++ {
p := fmt.Sprintf("/music/track-%03d.mp3", i)
rows = append(rows, row(byte(i), p, false))
// Only 60 of 100 present -> 40% would be marked, over the 25% cap.
if i < 60 {
seen[p] = struct{}{}
}
}
q := &fakeReconciler{rows: rows}
var stats Stats
err := s.reconcileMissing(context.Background(), q, seen, &stats)
if err == nil {
t.Fatal("expected reconcile to refuse, got nil error")
}
if len(q.marked) != 0 {
t.Errorf("marked %d rows despite refusing", len(q.marked))
}
if stats.Missing != 0 {
t.Errorf("stats.Missing = %d, want 0", stats.Missing)
}
}
// Restoring is never the dangerous direction, so it must survive the cap —
// otherwise a library that tripped the cap once could never clear its marks
// even after the volume came back.
func TestReconcileMissing_ClearsEvenWhenMarkCapTrips(t *testing.T) {
root := populatedRoot(t)
s := testScanner(t, root)
rows := []dbq.ListTrackPathsForReconcileRow{row(1, "/music/back.mp3", true)}
seen := map[string]struct{}{"/music/back.mp3": {}}
// Add enough absent rows to blow the cap.
for i := 2; i < 10; i++ {
rows = append(rows, row(byte(i), fmt.Sprintf("/music/absent-%02d.mp3", i), false))
}
q := &fakeReconciler{rows: rows}
var stats Stats
if err := s.reconcileMissing(context.Background(), q, seen, &stats); err == nil {
t.Fatal("expected the mark cap to trip")
}
if len(q.cleared) != 1 {
t.Errorf("cleared %d rows, want 1 — restores must not be blocked by the cap", len(q.cleared))
}
if stats.Restored != 1 {
t.Errorf("stats.Restored = %d, want 1", stats.Restored)
}
}
func TestReconcileMissing_RefusesOnEmptyWalk(t *testing.T) {
root := populatedRoot(t)
s := testScanner(t, root)
q := &fakeReconciler{rows: []dbq.ListTrackPathsForReconcileRow{
row(1, "/music/a.mp3", false),
}}
var stats Stats
if err := s.reconcileMissing(context.Background(), q, map[string]struct{}{}, &stats); err == nil {
t.Fatal("expected refusal when the walk saw no files")
}
if len(q.marked) != 0 {
t.Errorf("marked rows on an empty walk: %v", q.marked)
}
}
// The unmounted-volume case: the configured root doesn't exist at all.
func TestReconcileMissing_RefusesWhenRootMissing(t *testing.T) {
s := testScanner(t, filepath.Join(t.TempDir(), "not-mounted"))
q := &fakeReconciler{rows: []dbq.ListTrackPathsForReconcileRow{
row(1, "/music/a.mp3", false),
}}
var stats Stats
if err := s.reconcileMissing(context.Background(), q, map[string]struct{}{"/x": {}}, &stats); err == nil {
t.Fatal("expected refusal when a scan root is absent")
}
if len(q.marked) != 0 {
t.Errorf("marked rows with an absent root: %v", q.marked)
}
}
// A mount point that exists but has nothing mounted on it: os.Stat succeeds on
// the bare directory, which is why emptiness is checked separately.
func TestReconcileMissing_RefusesWhenRootEmpty(t *testing.T) {
s := testScanner(t, t.TempDir())
q := &fakeReconciler{rows: []dbq.ListTrackPathsForReconcileRow{
row(1, "/music/a.mp3", false),
}}
var stats Stats
if err := s.reconcileMissing(context.Background(), q, map[string]struct{}{"/x": {}}, &stats); err == nil {
t.Fatal("expected refusal when a scan root is empty")
}
}
// Several roots, one detached. Marking must not proceed on partial evidence just
// because the other roots looked fine.
func TestReconcileMissing_RefusesWhenAnyRootMissing(t *testing.T) {
good := populatedRoot(t)
s := testScanner(t, good, filepath.Join(t.TempDir(), "detached"))
q := &fakeReconciler{rows: []dbq.ListTrackPathsForReconcileRow{
row(1, "/music/a.mp3", false),
}}
var stats Stats
if err := s.reconcileMissing(context.Background(), q, map[string]struct{}{"/x": {}}, &stats); err == nil {
t.Fatal("expected refusal when one of several roots is absent")
}
}
func TestReconcileMissing_NoRowsIsNotAnError(t *testing.T) {
root := populatedRoot(t)
s := testScanner(t, root)
q := &fakeReconciler{}
var stats Stats
if err := s.reconcileMissing(context.Background(), q, map[string]struct{}{"/x": {}}, &stats); err != nil {
t.Fatalf("empty library should reconcile cleanly, got %v", err)
}
}
func TestReconcileMissing_PropagatesListError(t *testing.T) {
root := populatedRoot(t)
s := testScanner(t, root)
sentinel := errors.New("boom")
q := &fakeReconciler{listErr: sentinel}
var stats Stats
err := s.reconcileMissing(context.Background(), q, map[string]struct{}{"/x": {}}, &stats)
if !errors.Is(err, sentinel) {
t.Fatalf("err = %v, want it to wrap %v", err, sentinel)
}
}
func TestVerifyRootsPresent_NoRootsConfigured(t *testing.T) {
s := testScanner(t)
if err := s.verifyRootsPresent(); err == nil {
t.Fatal("expected an error with no scan roots configured")
}
}
internal/library/scanner.go
+36
View File
@@ -60,6 +60,11 @@ type Stats struct {
Updated int `json:"updated"` Updated int `json:"updated"`
Skipped int `json:"skipped"` Skipped int `json:"skipped"`
Errored int `json:"errored"` Errored int `json:"errored"`
// Missing / Restored come from the reconcile pass, not the walk (#2523):
// rows whose file the walk didn't find, and rows whose file came back.
// Only a full Scan sets these — see reconcileMissing.
Missing int `json:"missing"`
Restored int `json:"restored"`
} }
type Scanner struct { type Scanner struct {
@@ -76,6 +81,12 @@ func New(pool *pgxpool.Pool, logger *slog.Logger, paths []string) *Scanner {
// newer than the existing row's updated_at. Walk errors and per-file errors // newer than the existing row's updated_at. Walk errors and per-file errors
// are logged + counted; the scan keeps going. // are logged + counted; the scan keeps going.
// //
// It then reconciles: rows whose file the walk never saw get marked missing,
// and rows whose file has come back get un-marked (#2523). Only a FULL scan may
// do this — the walk's set of seen paths is the evidence, and a partial
// (watcher-driven) scan has no basis for concluding anything about files it
// didn't look at. That's why ScanFiles does not reconcile.
//
// progressCb (may be nil) receives the current Stats snapshot after each // progressCb (may be nil) receives the current Stats snapshot after each
// processed file. Used by the orchestrator to drive partial-tally writes. // processed file. Used by the orchestrator to drive partial-tally writes.
func (s *Scanner) Scan(ctx context.Context, progressCb func(Stats)) (Stats, error) { func (s *Scanner) Scan(ctx context.Context, progressCb func(Stats)) (Stats, error) {
@@ -83,6 +94,12 @@ func (s *Scanner) Scan(ctx context.Context, progressCb func(Stats)) (Stats, erro
q := dbq.New(s.pool) q := dbq.New(s.pool)
start := time.Now() start := time.Now()
// Every audio path the walk visited. Reconcile diffs this against the table,
// so it costs no extra filesystem I/O — the walk already established which
// files exist. ~100 bytes/path, so a 250k-track library is ~25MB, which is
// worth it to avoid a second stat pass over the whole library.
seen := make(map[string]struct{}, 8192)
for _, root := range s.paths { for _, root := range s.paths {
if err := filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error { if err := filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error {
if ctx.Err() != nil { if ctx.Err() != nil {
@@ -102,6 +119,11 @@ func (s *Scanner) Scan(ctx context.Context, progressCb func(Stats)) (Stats, erro
if !audioExtensions[strings.ToLower(filepath.Ext(path))] { if !audioExtensions[strings.ToLower(filepath.Ext(path))] {
return nil return nil
} }
// Recorded before scanFile so a file that exists but fails to parse
// still counts as present. It's a broken file, not a missing one,
// and marking it missing would hide it from the operator behind the
// wrong explanation.
seen[path] = struct{}{}
if _, _, err := s.scanFile(ctx, q, path, &stats); err != nil { if _, _, err := s.scanFile(ctx, q, path, &stats); err != nil {
s.logger.Warn("library scan file error", "path", path, "err", err) s.logger.Warn("library scan file error", "path", path, "err", err)
stats.Errored++ stats.Errored++
@@ -115,12 +137,26 @@ func (s *Scanner) Scan(ctx context.Context, progressCb func(Stats)) (Stats, erro
} }
} }
// Reconcile only after a COMPLETE walk. A cancelled scan has a partial
// `seen` set, which would mark everything it hadn't reached yet.
if err := ctx.Err(); err != nil {
return stats, err
}
if err := s.reconcileMissing(ctx, q, seen, &stats); err != nil {
// Not fatal: the walk's results are already persisted and useful. The
// guards deliberately refuse to act on ambiguous evidence, and that
// refusal arrives here as an error.
s.logger.Warn("library scan: reconcile skipped", "err", err)
}
s.logger.Info("library scan complete", s.logger.Info("library scan complete",
"scanned", stats.Scanned, "scanned", stats.Scanned,
"added", stats.Added, "added", stats.Added,
"updated", stats.Updated, "updated", stats.Updated,
"skipped", stats.Skipped, "skipped", stats.Skipped,
"errored", stats.Errored, "errored", stats.Errored,
"missing", stats.Missing,
"restored", stats.Restored,
"duration_ms", time.Since(start).Milliseconds(), "duration_ms", time.Since(start).Milliseconds(),
) )
if err := ctx.Err(); err != nil { if err := ctx.Err(); err != nil {
internal/library/scanrun.go
+5
View File
@@ -24,6 +24,11 @@ type LibraryStageTallies struct {
Updated int `json:"updated"` Updated int `json:"updated"`
Skipped int `json:"skipped"` Skipped int `json:"skipped"`
Errored int `json:"errored"` Errored int `json:"errored"`
// Reconcile results (#2523). Surfaced in the scan record because a track
// disappearing from the library is something the operator should be able to
// see happened, rather than discovering it when a mix comes up short.
Missing int `json:"missing"`
Restored int `json:"restored"`
} }
// MBIDBackfillStageTallies wires BackfillMBIDsResult into the scan_runs jsonb column. // MBIDBackfillStageTallies wires BackfillMBIDsResult into the scan_runs jsonb column.