From 7226dab9ff9db7af195de74255be2d23e14c61b3 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Mon, 13 Jul 2026 18:16:04 -0400 Subject: [PATCH] feat(discover): taste-targeted novelty bucket + rebalance toward discovery MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The 2-week metrics review found Discover beating the manual baseline on skip rate — which for a discovery surface means it plays it safe. On a single-user server it's effectively dormant + crude-random, and the per-user taste profile (taste_profile_tags, #796) went unused (#1254 gap). Add a fourth Discover candidate bucket, ListTasteUnheardTracksForDiscover: unheard / non-liked / non-quarantined tracks ranked by summed taste-tag weight over tracks.genre (split like the radio tag_overlap arm), md5 tiebreak. Its picks are stamped pick_kind = 'taste_unheard' (migration 0041 widens the CHECK on playlist_tracks + play_events, rule #36). Rebalance the slot allocation 40/30/30 → taste_unheard 35 / dormant 30 / cross_user 20 / random 15, and lead the interleave with taste_unheard so a track shared with another bucket keeps the taste stamp and the targeted-novelty arm stays measurable. Metrics label "Taste-matched" + order entry added to the single server-side pickKindLabels map, so web and Android surface the new breakdown row with no client change. Cold start (empty taste_profile_tags) yields an empty taste bucket that redistributes to the survivors, so Discover still fills. Scribe #1488. Companion review outcomes: Songs-like starvation already fixed (#1255); For You v2 ratified as-is (fresh-injection cost disproven). Co-Authored-By: Claude Opus 4.8 (1M context) --- internal/api/me_recommendation_metrics.go | 21 +-- .../api/me_recommendation_metrics_test.go | 2 +- internal/db/dbq/discover.sql.go | 66 ++++++++++ .../0041_discover_taste_unheard.down.sql | 27 ++++ .../0041_discover_taste_unheard.up.sql | 32 +++++ internal/db/queries/discover.sql | 34 +++++ internal/playlists/discover.go | 124 ++++++++++++------ internal/playlists/system.go | 21 +-- 8 files changed, 267 insertions(+), 60 deletions(-) create mode 100644 internal/db/migrations/0041_discover_taste_unheard.down.sql create mode 100644 internal/db/migrations/0041_discover_taste_unheard.up.sql diff --git a/internal/api/me_recommendation_metrics.go b/internal/api/me_recommendation_metrics.go index 75761b05..7aa02bb1 100644 --- a/internal/api/me_recommendation_metrics.go +++ b/internal/api/me_recommendation_metrics.go @@ -171,18 +171,19 @@ func (h *handlers) handleGetRecommendationMetrics(w http.ResponseWriter, r *http } // pickKindLabels is the display vocabulary for play_events.pick_kind -// values (mirrors the CHECK in migration 0039). Every system mix that +// values (mirrors the CHECK in migration 0041). Every system mix that // stamps provenance gets its breakdown from this one map — adding a // stamping mix needs no metrics change. var pickKindLabels = map[string]string{ - "taste": "Taste picks", - "fresh": "Fresh picks", - "dormant": "Dormant artists", - "cross_user": "Liked by others", - "random": "Random unheard", - "tier1": "Tier 1 (exact)", - "tier2": "Tier 2 (relaxed)", - "tier3": "Tier 3 (stretched)", + "taste": "Taste picks", + "fresh": "Fresh picks", + "dormant": "Dormant artists", + "taste_unheard": "Taste-matched", + "cross_user": "Liked by others", + "random": "Random unheard", + "tier1": "Tier 1 (exact)", + "tier2": "Tier 2 (relaxed)", + "tier3": "Tier 3 (stretched)", } // pickKindOrder fixes breakdown row order; unattributed ("", i.e. NULL @@ -192,7 +193,7 @@ var pickKindLabels = map[string]string{ // of silently shrinking. The DB CHECK gates pick_kind to exactly this // vocabulary, so iterating the list is exhaustive. var pickKindOrder = []string{ - "taste", "fresh", "dormant", "cross_user", "random", + "taste", "fresh", "dormant", "taste_unheard", "cross_user", "random", "tier1", "tier2", "tier3", "", } diff --git a/internal/api/me_recommendation_metrics_test.go b/internal/api/me_recommendation_metrics_test.go index de717b79..a5a6946e 100644 --- a/internal/api/me_recommendation_metrics_test.go +++ b/internal/api/me_recommendation_metrics_test.go @@ -171,7 +171,7 @@ func TestBucketMetricsResponse_ForYouBreakdown(t *testing.T) { func TestBucketMetricsResponse_DiscoverBucketBreakdown(t *testing.T) { // Provenance is standard (#1270): Discover's bucket stamps surface as // a breakdown exactly like For You's taste/fresh — this is what makes - // the 40/30/30 allocation judgeable instead of a guess. + // the bucket allocation judgeable instead of a guess. src := func(s string) *string { return &s } kind := func(s string) *string { return &s } rows := []dbq.RecommendationSourceMetricsForUserRow{ diff --git a/internal/db/dbq/discover.sql.go b/internal/db/dbq/discover.sql.go index f4f8266a..69161ec1 100644 --- a/internal/db/dbq/discover.sql.go +++ b/internal/db/dbq/discover.sql.go @@ -211,3 +211,69 @@ func (q *Queries) ListRandomUnheardTracksForDiscover(ctx context.Context, arg Li } return items, nil } + +const listTasteUnheardTracksForDiscover = `-- name: ListTasteUnheardTracksForDiscover :many +SELECT t.id, t.album_id, t.artist_id + FROM tracks t + JOIN LATERAL regexp_split_to_table(coalesce(t.genre, ''), '[;,]') AS g_split(g) ON true + JOIN taste_profile_tags nt ON nt.user_id = $1 AND trim(g_split.g) = nt.tag + WHERE nt.weight > 0 + AND trim(g_split.g) <> '' + AND NOT EXISTS ( + SELECT 1 FROM play_events pe + WHERE pe.user_id = $1 + AND pe.track_id = t.id + AND pe.was_skipped = false + ) + AND NOT EXISTS ( + SELECT 1 FROM general_likes gl + WHERE gl.user_id = $1 AND gl.track_id = t.id + ) + AND NOT EXISTS ( + SELECT 1 FROM lidarr_quarantine q + WHERE q.user_id = $1 AND q.track_id = t.id + ) + GROUP BY t.id, t.album_id, t.artist_id + ORDER BY SUM(nt.weight) DESC, md5(t.id::text || $2::text) + LIMIT 120 +` + +type ListTasteUnheardTracksForDiscoverParams struct { + UserID pgtype.UUID + Column2 string +} + +type ListTasteUnheardTracksForDiscoverRow struct { + ID pgtype.UUID + AlbumID pgtype.UUID + ArtistID pgtype.UUID +} + +// Taste-targeted novelty: unheard tracks whose genres overlap the user's +// taste-profile tags (taste_profile_tags, #796), ranked by summed tag +// weight — "new to you, but your vibe" rather than the crude random arm. +// Genres live inline on tracks.genre as a delimited string, split the +// same way the radio tag_overlap arm does (regexp_split_to_table on +// [;,]). Same exclusion filters as the other buckets. Returns nothing +// when the user has no taste tags yet (cold start), so the caller +// redistributes its slots to the other buckets. Stamped 'taste_unheard'. +// $1 = user_id, $2 = date string for md5 tiebreak ordering. +func (q *Queries) ListTasteUnheardTracksForDiscover(ctx context.Context, arg ListTasteUnheardTracksForDiscoverParams) ([]ListTasteUnheardTracksForDiscoverRow, error) { + rows, err := q.db.Query(ctx, listTasteUnheardTracksForDiscover, arg.UserID, arg.Column2) + if err != nil { + return nil, err + } + defer rows.Close() + var items []ListTasteUnheardTracksForDiscoverRow + for rows.Next() { + var i ListTasteUnheardTracksForDiscoverRow + if err := rows.Scan(&i.ID, &i.AlbumID, &i.ArtistID); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} diff --git a/internal/db/migrations/0041_discover_taste_unheard.down.sql b/internal/db/migrations/0041_discover_taste_unheard.down.sql new file mode 100644 index 00000000..2bc620a2 --- /dev/null +++ b/internal/db/migrations/0041_discover_taste_unheard.down.sql @@ -0,0 +1,27 @@ +-- Reverse 0041_discover_taste_unheard.up.sql. +-- +-- Null out any rows stamped with the new value before shrinking the +-- CHECK back to the 0039 vocabulary, or the ADD CONSTRAINT would fail +-- on existing 'taste_unheard' rows (mirrors 0039's down). +UPDATE play_events SET pick_kind = NULL WHERE pick_kind = 'taste_unheard'; +UPDATE playlist_tracks SET pick_kind = NULL WHERE pick_kind = 'taste_unheard'; + +ALTER TABLE playlist_tracks + DROP CONSTRAINT playlist_tracks_pick_kind_check; +ALTER TABLE playlist_tracks + ADD CONSTRAINT playlist_tracks_pick_kind_check + CHECK (pick_kind IN ( + 'taste', 'fresh', + 'dormant', 'cross_user', 'random', + 'tier1', 'tier2', 'tier3' + )); + +ALTER TABLE play_events + DROP CONSTRAINT play_events_pick_kind_check; +ALTER TABLE play_events + ADD CONSTRAINT play_events_pick_kind_check + CHECK (pick_kind IN ( + 'taste', 'fresh', + 'dormant', 'cross_user', 'random', + 'tier1', 'tier2', 'tier3' + )); diff --git a/internal/db/migrations/0041_discover_taste_unheard.up.sql b/internal/db/migrations/0041_discover_taste_unheard.up.sql new file mode 100644 index 00000000..c2cee8bd --- /dev/null +++ b/internal/db/migrations/0041_discover_taste_unheard.up.sql @@ -0,0 +1,32 @@ +-- Discover taste-targeted novelty bucket (milestone #127, Scribe #1252/#1488). +-- +-- The 2-week metrics review found Discover beating the manual baseline on +-- skip rate — which for a discovery surface means it is playing it safe: +-- on a single-user server it is effectively dormant + crude-random, and +-- the per-user taste profile (n_tags, #796) went unused. This adds a +-- fourth Discover candidate bucket that ranks unheard tracks by the +-- user's taste-profile tag weights ("novelty that fits your vibe"), so +-- its picks need a new pick_kind provenance value. +-- +-- Postgres CHECK whitelists can't be altered in place: DROP + re-ADD with +-- the expanded list, in the same migration (family rule — new CHECK-enum +-- value needs a same-change migration). +ALTER TABLE playlist_tracks + DROP CONSTRAINT playlist_tracks_pick_kind_check; +ALTER TABLE playlist_tracks + ADD CONSTRAINT playlist_tracks_pick_kind_check + CHECK (pick_kind IN ( + 'taste', 'fresh', + 'dormant', 'cross_user', 'random', 'taste_unheard', + 'tier1', 'tier2', 'tier3' + )); + +ALTER TABLE play_events + DROP CONSTRAINT play_events_pick_kind_check; +ALTER TABLE play_events + ADD CONSTRAINT play_events_pick_kind_check + CHECK (pick_kind IN ( + 'taste', 'fresh', + 'dormant', 'cross_user', 'random', 'taste_unheard', + 'tier1', 'tier2', 'tier3' + )); diff --git a/internal/db/queries/discover.sql b/internal/db/queries/discover.sql index 3f3762d4..ed44c378 100644 --- a/internal/db/queries/discover.sql +++ b/internal/db/queries/discover.sql @@ -102,3 +102,37 @@ SELECT t.id, t.album_id, t.artist_id ) ORDER BY md5(t.id::text || $2::text) LIMIT 200; + +-- name: ListTasteUnheardTracksForDiscover :many +-- Taste-targeted novelty: unheard tracks whose genres overlap the user's +-- taste-profile tags (taste_profile_tags, #796), ranked by summed tag +-- weight — "new to you, but your vibe" rather than the crude random arm. +-- Genres live inline on tracks.genre as a delimited string, split the +-- same way the radio tag_overlap arm does (regexp_split_to_table on +-- [;,]). Same exclusion filters as the other buckets. Returns nothing +-- when the user has no taste tags yet (cold start), so the caller +-- redistributes its slots to the other buckets. Stamped 'taste_unheard'. +-- $1 = user_id, $2 = date string for md5 tiebreak ordering. +SELECT t.id, t.album_id, t.artist_id + FROM tracks t + JOIN LATERAL regexp_split_to_table(coalesce(t.genre, ''), '[;,]') AS g_split(g) ON true + JOIN taste_profile_tags nt ON nt.user_id = $1 AND trim(g_split.g) = nt.tag + WHERE nt.weight > 0 + AND trim(g_split.g) <> '' + AND NOT EXISTS ( + SELECT 1 FROM play_events pe + WHERE pe.user_id = $1 + AND pe.track_id = t.id + AND pe.was_skipped = false + ) + AND NOT EXISTS ( + SELECT 1 FROM general_likes gl + WHERE gl.user_id = $1 AND gl.track_id = t.id + ) + AND NOT EXISTS ( + SELECT 1 FROM lidarr_quarantine q + WHERE q.user_id = $1 AND q.track_id = t.id + ) + GROUP BY t.id, t.album_id, t.artist_id + ORDER BY SUM(nt.weight) DESC, md5(t.id::text || $2::text) + LIMIT 120; diff --git a/internal/playlists/discover.go b/internal/playlists/discover.go index 809e33fb..f70c658d 100644 --- a/internal/playlists/discover.go +++ b/internal/playlists/discover.go @@ -10,10 +10,17 @@ import ( ) const ( - discoverTotalSlots = 100 - discoverDormantSlots = 40 - discoverCrossUserSlots = 30 - discoverRandomSlots = 30 + discoverTotalSlots = 100 + // Bucket allocation, pushed toward *targeted* novelty (Scribe #1488): + // the taste-matched-unheard arm gets the plurality, dormant stays a + // familiar anchor, and the crude random arm is trimmed to a + // serendipity sliver. On a single-user server cross_user is empty and + // its slots redistribute, so taste_unheard becomes the dominant real + // arm — Discover actually discovers instead of re-warming known artists. + discoverTasteUnheardSlots = 35 + discoverDormantSlots = 30 + discoverCrossUserSlots = 20 + discoverRandomSlots = 15 discoverMaxTracksPerAlbum = 2 discoverMaxTracksPerArtist = 3 ) @@ -21,9 +28,10 @@ const ( // discoverTrack is the common shape used by the bucket allocator. The // three sqlc-generated row types collapse into this internal struct so // downstream functions don't need to be generic over them. PickKind is -// the originating bucket (dormant/cross_user/random, #1270), stamped in -// the row adapters so it survives the interleave — bucket identity is -// what makes the 40/30/30 allocation measurable in the metrics. +// the originating bucket (taste_unheard/dormant/cross_user/random, +// #1270/#1488), stamped in the row adapters so it survives the +// interleave — bucket identity is what makes the slot allocation +// measurable in the metrics. type discoverTrack struct { ID pgtype.UUID AlbumID pgtype.UUID @@ -32,8 +40,9 @@ type discoverTrack struct { } // buildDiscoverCandidates assembles the Discover playlist track list. -// Pulls from three buckets, applies per-album/per-artist caps, then -// redistributes any deficit equally across the remaining buckets. +// Pulls from four buckets (taste_unheard / dormant / cross_user / +// random), applies per-album/per-artist caps, then redistributes any +// deficit equally across the remaining buckets. // // Returns up to discoverTotalSlots track IDs in the order they should // appear in the playlist (round-robin interleaved across buckets). @@ -43,6 +52,57 @@ type discoverTrack struct { // the redistribution algorithm rolls the deficit into the surviving // buckets so one broken bucket can't silently kill the whole playlist. func buildDiscoverCandidates(ctx context.Context, q *dbq.Queries, logger *slog.Logger, userID pgtype.UUID, dateStr string) ([]rankedCandidate, error) { + pools := loadDiscoverPools(ctx, q, logger, userID, dateStr) + + // Allocate slots with redistribution. Order fixes both the allocation + // index mapping and (below) the interleave/dedup priority. + allocations := redistributeSlots([]bucketRequest{ + {want: discoverTasteUnheardSlots, available: len(pools.taste)}, + {want: discoverDormantSlots, available: len(pools.dormant)}, + {want: discoverCrossUserSlots, available: len(pools.crossUser)}, + {want: discoverRandomSlots, available: len(pools.random)}, + }) + + // Round-robin interleave so the playlist doesn't front-load one + // flavour. taste_unheard leads so a track shared with another bucket + // keeps the taste stamp — the targeted-novelty arm stays measurable. + out := interleaveBuckets( + pools.taste[:allocations[0]], + pools.dormant[:allocations[1]], + pools.crossUser[:allocations[2]], + pools.random[:allocations[3]], + ) + + // Convert to rankedCandidate (the type insertSystemPlaylist accepts). + // Score is unused for Discover — the daily-deterministic md5 ordering + // already gave us the ranking. + ranked := make([]rankedCandidate, 0, len(out)) + for _, t := range out { + ranked = append(ranked, rankedCandidate{TrackID: t.ID, PickKind: t.PickKind}) + } + return ranked, nil +} + +// discoverPools holds the four Discover candidate buckets after adapting +// + per-album/per-artist capping. Bucket query failures degrade to an +// empty pool (logged) so one broken bucket can't kill the whole playlist; +// the slot redistribution rolls its deficit into the survivors. +type discoverPools struct { + taste []discoverTrack + dormant []discoverTrack + crossUser []discoverTrack + random []discoverTrack +} + +func loadDiscoverPools(ctx context.Context, q *dbq.Queries, logger *slog.Logger, userID pgtype.UUID, dateStr string) discoverPools { + tasteRows, err := q.ListTasteUnheardTracksForDiscover(ctx, dbq.ListTasteUnheardTracksForDiscoverParams{ + UserID: userID, Column2: dateStr, + }) + if err != nil { + logger.Warn("discover: taste-unheard bucket failed; continuing with empty pool", + "user_id", uuidStringPL(userID), "err", err) + tasteRows = nil + } dormantRows, err := q.ListDormantArtistTracksForDiscover(ctx, dbq.ListDormantArtistTracksForDiscoverParams{ UserID: userID, Column2: dateStr, }) @@ -67,37 +127,23 @@ func buildDiscoverCandidates(ctx context.Context, q *dbq.Queries, logger *slog.L "user_id", uuidStringPL(userID), "err", err) randomRows = nil } - - // Adapt sqlc-generated rows into the internal struct, then apply - // per-album / per-artist caps. - dormantPool := capByAlbumAndArtist(dormantRowsToTracks(dormantRows)) - crossUserPool := capByAlbumAndArtist(crossUserRowsToTracks(crossUserRows)) - randomPool := capByAlbumAndArtist(randomRowsToTracks(randomRows)) - - // Allocate slots with redistribution. - allocations := redistributeSlots([]bucketRequest{ - {want: discoverDormantSlots, available: len(dormantPool)}, - {want: discoverCrossUserSlots, available: len(crossUserPool)}, - {want: discoverRandomSlots, available: len(randomPool)}, - }) - - // Take the head of each bucket's capped pool. - dormantTake := dormantPool[:allocations[0]] - crossUserTake := crossUserPool[:allocations[1]] - randomTake := randomPool[:allocations[2]] - - // Round-robin interleave so the playlist doesn't front-load one - // flavour. - out := interleaveBuckets(dormantTake, crossUserTake, randomTake) - - // Convert to rankedCandidate (the type insertSystemPlaylist accepts). - // Score is unused for Discover — the daily-deterministic md5 ordering - // already gave us the ranking. - ranked := make([]rankedCandidate, 0, len(out)) - for _, t := range out { - ranked = append(ranked, rankedCandidate{TrackID: t.ID, PickKind: t.PickKind}) + return discoverPools{ + taste: capByAlbumAndArtist(tasteUnheardRowsToTracks(tasteRows)), + dormant: capByAlbumAndArtist(dormantRowsToTracks(dormantRows)), + crossUser: capByAlbumAndArtist(crossUserRowsToTracks(crossUserRows)), + random: capByAlbumAndArtist(randomRowsToTracks(randomRows)), } - return ranked, nil +} + +func tasteUnheardRowsToTracks(rows []dbq.ListTasteUnheardTracksForDiscoverRow) []discoverTrack { + out := make([]discoverTrack, len(rows)) + for i, r := range rows { + out[i] = discoverTrack{ + ID: r.ID, AlbumID: r.AlbumID, ArtistID: r.ArtistID, + PickKind: pickKindTasteUnheard, + } + } + return out } func dormantRowsToTracks(rows []dbq.ListDormantArtistTracksForDiscoverRow) []discoverTrack { diff --git a/internal/playlists/system.go b/internal/playlists/system.go index 5103c14f..b6b7f155 100644 --- a/internal/playlists/system.go +++ b/internal/playlists/system.go @@ -130,17 +130,18 @@ type rankedCandidate struct { // Pick kinds, persisted on playlist_tracks.pick_kind so plays can // attribute skips to the population that sourced the track — For You's // "taste engine missing" vs "freshness tax", Discover's three buckets -// (#1270). Values mirror the CHECK in migration 0039; the tier1-3 -// values there land with the tiered-mix rebuilds. +// (#1270). Values mirror the CHECK in migration 0041; taste_unheard is +// Discover's taste-targeted-novelty arm (#1488). const ( - pickKindTaste = "taste" - pickKindFresh = "fresh" - pickKindDormant = "dormant" - pickKindCrossUser = "cross_user" - pickKindRandom = "random" - pickKindTier1 = "tier1" - pickKindTier2 = "tier2" - pickKindTier3 = "tier3" + pickKindTaste = "taste" + pickKindFresh = "fresh" + pickKindDormant = "dormant" + pickKindTasteUnheard = "taste_unheard" + pickKindCrossUser = "cross_user" + pickKindRandom = "random" + pickKindTier1 = "tier1" + pickKindTier2 = "tier2" + pickKindTier3 = "tier3" ) // pickKindForSeedTier maps PickSeedArtists' seed-pool fallback tier