From 0bfd51a1498db3eaabbcc2dcf915780768540679 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Thu, 7 May 2026 10:34:39 -0400 Subject: [PATCH] feat(server/playlists): For-You head+tail + diversity caps MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two improvements to the system playlist builder: 1. Per-artist (<=3) and per-album (<=2) caps applied to the pickTopN truncation step, using the same numeric caps Discover already enforces. Both For-You and Songs-like-X benefit. Same skewed candidate pool no longer collapses to "10 tracks from the same artist" — the playlist always carries at least 9 distinct artists in 25 slots. 2. New pickHeadAndTail function for For-You: 20 top-similarity tracks + 5 sampled from the tail (positions 2*headN onward of the score-sorted, cap-applied pool). Tail sampling uses tieBreakHash for daily determinism — same user same day still sees the same playlist, but the daily refresh feels less stuck-in-a-rut. Tail tracks are still similarity-related (they passed the similarity candidate filter) so the user should enjoy them, just from artists they wouldn't have surfaced via strict top-N ranking. Songs-like-X keeps the simple pickTopN call — the seed-artist context already provides the "you'll like this" framing without needing a tail injection. Refactors pickTopN internals: now sorts candidates first via scoreAndSortCandidates, applies the cap on []Candidate via capCandidatesByAlbumAndArtist, and truncates. Removes the now- dead stableSortByScoreThenHash helper (only used in old pickTopN). The cap helper mirrors capByAlbumAndArtist in discover.go but operates on recommendation.Candidate so it sees Track.AlbumID / Track.ArtistID directly. Tests cover the cap helper truth table, head+tail split with small/large pools, buffer-zone exclusion, daily determinism, and cross-day tail variance. Co-Authored-By: Claude Sonnet 4.6 --- internal/playlists/foryou_test.go | 233 ++++++++++++++++++++++++++++++ internal/playlists/system.go | 176 +++++++++++++++++++--- 2 files changed, 387 insertions(+), 22 deletions(-) create mode 100644 internal/playlists/foryou_test.go diff --git a/internal/playlists/foryou_test.go b/internal/playlists/foryou_test.go new file mode 100644 index 00000000..578876db --- /dev/null +++ b/internal/playlists/foryou_test.go @@ -0,0 +1,233 @@ +package playlists + +import ( + "testing" + "time" + + "github.com/jackc/pgx/v5/pgtype" + + "git.fabledsword.com/bvandeusen/minstrel/internal/db/dbq" + "git.fabledsword.com/bvandeusen/minstrel/internal/recommendation" +) + +// makeCand constructs a Candidate with distinct track/album/artist IDs +// and a SimilarityScore that drives distinguishable scores when passed +// through recommendation.Score. Using SimilarityScore alone is sufficient +// because systemMixWeights.SimilarityWeight = 1.5 (non-zero), so +// different similarity values produce different scores. +func makeCand(trackN, albumN, artistN int, similarity float64) recommendation.Candidate { + var tID, aID, arID pgtype.UUID + tID.Valid, aID.Valid, arID.Valid = true, true, true + tID.Bytes[15] = byte(trackN) + aID.Bytes[15] = byte(albumN) + arID.Bytes[15] = byte(artistN) + return recommendation.Candidate{ + Track: dbq.Track{ID: tID, AlbumID: aID, ArtistID: arID}, + Inputs: recommendation.ScoringInputs{ + SimilarityScore: similarity, + }, + } +} + +func TestCapCandidatesByAlbumAndArtist_AlbumCap(t *testing.T) { + // discoverMaxTracksPerAlbum = 2: third track on album 10 must be dropped. + in := []recommendation.Candidate{ + makeCand(1, 10, 100, 1.0), + makeCand(2, 10, 101, 0.9), + makeCand(3, 10, 102, 0.8), // 3rd from album 10 → drop + makeCand(4, 11, 103, 0.7), + } + got := capCandidatesByAlbumAndArtist(in) + if len(got) != 3 { + t.Errorf("len = %d, want 3 (album 10 capped at 2)", len(got)) + } +} + +func TestCapCandidatesByAlbumAndArtist_ArtistCap(t *testing.T) { + // discoverMaxTracksPerArtist = 3: fourth track by artist 100 must be dropped. + in := []recommendation.Candidate{ + makeCand(1, 10, 100, 1.0), + makeCand(2, 11, 100, 0.9), + makeCand(3, 12, 100, 0.8), + makeCand(4, 13, 100, 0.7), // 4th by artist 100 → drop + makeCand(5, 14, 101, 0.6), + } + got := capCandidatesByAlbumAndArtist(in) + if len(got) != 4 { + t.Errorf("len = %d, want 4 (artist 100 capped at 3)", len(got)) + } +} + +func TestCapCandidatesByAlbumAndArtist_PreservesOrder(t *testing.T) { + // All distinct albums and artists: all kept, original order preserved. + in := []recommendation.Candidate{ + makeCand(3, 10, 100, 0.5), + makeCand(1, 11, 101, 0.9), + makeCand(2, 12, 102, 0.7), + } + got := capCandidatesByAlbumAndArtist(in) + if len(got) != 3 { + t.Fatalf("len = %d, want 3", len(got)) + } + for i, want := range []byte{3, 1, 2} { + if got[i].Track.ID.Bytes[15] != want { + t.Errorf("got[%d].ID = %d, want %d", i, got[i].Track.ID.Bytes[15], want) + } + } +} + +func TestCapCandidatesByAlbumAndArtist_Empty(t *testing.T) { + got := capCandidatesByAlbumAndArtist(nil) + if len(got) != 0 { + t.Errorf("len = %d, want 0", len(got)) + } +} + +func TestPickHeadAndTail_SmallPool(t *testing.T) { + // Pool < headN+tailN → return up to total entries, no tail split. + in := []recommendation.Candidate{ + makeCand(1, 10, 100, 1.0), + makeCand(2, 11, 101, 0.9), + makeCand(3, 12, 102, 0.8), + } + got := pickHeadAndTail(in, "2026-05-07", time.Now(), 5, 2) + if len(got) != 3 { + t.Errorf("len = %d, want 3 (pool too small for head/tail split)", len(got)) + } +} + +func TestPickHeadAndTail_ExactlyTotal(t *testing.T) { + // Pool == headN+tailN: no tail to sample from, returns all. + in := make([]recommendation.Candidate, 0, 7) + for i := 0; i < 7; i++ { + in = append(in, makeCand(i+1, i+1, i+1, float64(7-i)/10.0)) + } + got := pickHeadAndTail(in, "2026-05-07", time.Now(), 5, 2) + if len(got) != 7 { + t.Errorf("len = %d, want 7 (pool == total)", len(got)) + } +} + +func TestPickHeadAndTail_HeadAndTailSplit(t *testing.T) { + // Pool of 100 distinct (album, artist) pairs; no caps trim. + // With headN=20, tailN=5, expect 25 entries total. + in := make([]recommendation.Candidate, 0, 100) + for i := 0; i < 100; i++ { + in = append(in, makeCand(i+1, i+1, i+1, float64(100-i)/100.0)) + } + got := pickHeadAndTail(in, "2026-05-07", time.Now(), 20, 5) + if len(got) != 25 { + t.Errorf("len = %d, want 25 (20 head + 5 tail)", len(got)) + } +} + +func TestPickHeadAndTail_Determinism(t *testing.T) { + // Same inputs, same dateStr → identical output both times. + in := make([]recommendation.Candidate, 0, 100) + for i := 0; i < 100; i++ { + in = append(in, makeCand(i+1, i+1, i+1, float64(100-i)/100.0)) + } + now := time.Now() + got1 := pickHeadAndTail(in, "2026-05-07", now, 20, 5) + got2 := pickHeadAndTail(in, "2026-05-07", now, 20, 5) + if len(got1) != len(got2) { + t.Fatalf("len mismatch: %d vs %d", len(got1), len(got2)) + } + for i := range got1 { + if got1[i].TrackID.Bytes[15] != got2[i].TrackID.Bytes[15] { + t.Errorf("position %d differs across calls (not deterministic within day)", i) + break + } + } +} + +func TestPickHeadAndTail_HeadStable_TailVariesAcrossDays(t *testing.T) { + // Head (positions 0..headN-1) must match across different date strings. + // Tail (positions headN..headN+tailN-1) should differ for different dates + // because tieBreakHash incorporates the date. + in := make([]recommendation.Candidate, 0, 100) + for i := 0; i < 100; i++ { + in = append(in, makeCand(i+1, i+1, i+1, float64(100-i)/100.0)) + } + now := time.Now() + day1 := pickHeadAndTail(in, "2026-05-07", now, 20, 5) + day2 := pickHeadAndTail(in, "2026-05-08", now, 20, 5) + if len(day1) != 25 || len(day2) != 25 { + t.Fatalf("len mismatch: day1=%d day2=%d", len(day1), len(day2)) + } + + // Head (score-sorted, deterministic): positions 0..19 must match. + for i := 0; i < 20; i++ { + if day1[i].TrackID.Bytes[15] != day2[i].TrackID.Bytes[15] { + t.Errorf("head position %d differs across date strings (should be score-stable)", i) + } + } + + // Tail (tieBreakHash order): at least one position should differ. + tailDiffers := false + for i := 20; i < 25; i++ { + if day1[i].TrackID.Bytes[15] != day2[i].TrackID.Bytes[15] { + tailDiffers = true + break + } + } + if !tailDiffers { + t.Errorf("tail identical across different date strings (tieBreakHash not using dateStr)") + } +} + +func TestPickHeadAndTail_TailFromBeyond2xHeadN(t *testing.T) { + // With headN=5 and a 50-candidate pool, tailStart = 2*5 = 10. + // Tail samples come from positions >=10, not from positions 5..9. + // Verify: the first 5 positions contain tracks from the top-5 by score, + // and positions 5..9 (the "buffer zone") do not appear in the tail. + // + // Tracks 1..50 have similarity 0.99..0.01 (descending), so + // after scoring the sorted order is tracks 1,2,3,...,50. + // Top 5 head = tracks 1..5. + // Buffer zone (positions 5..9) = tracks 6..10. + // Tail pool = tracks 11..50; tail sample = 3 from that pool. + in := make([]recommendation.Candidate, 0, 50) + for i := 0; i < 50; i++ { + // trackN = i+1, score descends: track 1 scores highest. + sim := float64(50-i) / 50.0 + in = append(in, makeCand(i+1, i+1, i+1, sim)) + } + got := pickHeadAndTail(in, "2026-05-07", time.Now(), 5, 3) + if len(got) != 8 { + t.Fatalf("len = %d, want 8 (5 head + 3 tail)", len(got)) + } + + // Head tracks must be tracks 1..5 (highest scorers). + for i := 0; i < 5; i++ { + if got[i].TrackID.Bytes[15] != byte(i+1) { + t.Errorf("head[%d] = track %d, want track %d", + i, got[i].TrackID.Bytes[15], i+1) + } + } + + // Buffer-zone tracks 6..10 must NOT appear in the tail positions 5..7. + bufferSet := map[byte]bool{6: true, 7: true, 8: true, 9: true, 10: true} + for i := 5; i < 8; i++ { + if bufferSet[got[i].TrackID.Bytes[15]] { + t.Errorf("tail[%d] = track %d (buffer zone); should come from positions >=10", + i-5, got[i].TrackID.Bytes[15]) + } + } +} + +func TestPickTopN_DiversityCap(t *testing.T) { + // Verify pickTopN now enforces the diversity cap. + // 5 tracks from the same artist: only 3 should survive the cap. + in := []recommendation.Candidate{ + makeCand(1, 10, 100, 1.0), + makeCand(2, 11, 100, 0.9), + makeCand(3, 12, 100, 0.8), + makeCand(4, 13, 100, 0.7), // 4th by artist → dropped by cap + makeCand(5, 14, 100, 0.6), // 5th by artist → dropped by cap + } + got := pickTopN(in, "2026-05-07", time.Now(), 25) + if len(got) != 3 { + t.Errorf("len = %d, want 3 (artist 100 capped at 3)", len(got)) + } +} diff --git a/internal/playlists/system.go b/internal/playlists/system.go index 268beb30..483ff2d3 100644 --- a/internal/playlists/system.go +++ b/internal/playlists/system.go @@ -65,17 +65,6 @@ type rankedCandidate struct { Score float64 } -// stableSortByScoreThenHash sorts candidates in-place by score DESC, -// breaking ties with tieBreakHash(track_id, dateStr). -func stableSortByScoreThenHash(cands []rankedCandidate, dateStr string) { - sort.SliceStable(cands, func(i, j int) bool { - if cands[i].Score != cands[j].Score { - return cands[i].Score > cands[j].Score - } - return tieBreakHash(cands[i].TrackID, dateStr) < tieBreakHash(cands[j].TrackID, dateStr) - }) -} - const systemMixLength = 25 // systemMixWeights are the fixed scoring weights used by the cron worker. @@ -95,6 +84,70 @@ var systemMixWeights = recommendation.ScoringWeights{ // recommendation.Score fully deterministic. func noopRNG() float64 { return 0 } +// forYouHeadN is the number of top-scored tracks that anchor the For-You +// playlist. forYouTailN is the number of diversity picks sampled from the +// tail of the score-sorted pool (positions 2*forYouHeadN onward), injected +// after the head to give users a daily-deterministic surprise without +// compromising quality. Songs-like-X keeps simple pickTopN (the seed-artist +// context already frames the "you'll like this" promise). +const ( + forYouHeadN = 20 + forYouTailN = 5 +) + +// scoreAndSortCandidates scores every candidate with recommendation.Score +// and returns a new slice sorted by score DESC (ties broken by +// tieBreakHash). Pure — no truncation, no cap. +func scoreAndSortCandidates(cands []recommendation.Candidate, dateStr string, now time.Time) []recommendation.Candidate { + type scored struct { + c recommendation.Candidate + score float64 + } + pairs := make([]scored, len(cands)) + for i, c := range cands { + pairs[i] = scored{c: c, score: recommendation.Score(c.Inputs, systemMixWeights, now, noopRNG)} + } + sort.SliceStable(pairs, func(i, j int) bool { + if pairs[i].score != pairs[j].score { + return pairs[i].score > pairs[j].score + } + return tieBreakHash(pairs[i].c.Track.ID, dateStr) < tieBreakHash(pairs[j].c.Track.ID, dateStr) + }) + out := make([]recommendation.Candidate, len(pairs)) + for i, p := range pairs { + out[i] = p.c + } + return out +} + +// capCandidatesByAlbumAndArtist trims a candidate list so no single +// album appears more than discoverMaxTracksPerAlbum times and no +// single artist appears more than discoverMaxTracksPerArtist times. +// Mirrors capByAlbumAndArtist (which operates on []discoverTrack); +// here the input/output is []recommendation.Candidate so For-You and +// Songs-like-X can apply the same diversity caps as Discover. +// +// Preserves input order. Reuses the discoverMax* constants — +// playlists across all three system variants get consistent +// diversity behavior. +func capCandidatesByAlbumAndArtist(cands []recommendation.Candidate) []recommendation.Candidate { + albumCount := map[pgtype.UUID]int{} + artistCount := map[pgtype.UUID]int{} + out := make([]recommendation.Candidate, 0, len(cands)) + for _, c := range cands { + if albumCount[c.Track.AlbumID] >= discoverMaxTracksPerAlbum { + continue + } + if artistCount[c.Track.ArtistID] >= discoverMaxTracksPerArtist { + continue + } + albumCount[c.Track.AlbumID]++ + artistCount[c.Track.ArtistID]++ + out = append(out, c) + } + return out +} + // BuildSystemPlaylists builds the user's daily system mixes (one For-You + // up to 3 Songs-like-{seed} mixes). Atomic-replace inside one tx; // concurrency-guarded via system_playlist_runs.in_flight; deterministic @@ -160,7 +213,12 @@ func BuildSystemPlaylists(ctx context.Context, pool *pgxpool.Pool, logger *slog. recommendation.DefaultCandidateSourceLimits(), ) if cerr == nil { - forYouTracks = pickTopN(cands, dateStr, now, systemMixLength) + // For-You uses head+tail composition: forYouHeadN top-similarity + // tracks + forYouTailN tail-sampled to inject daily-deterministic + // surprise. Songs-like-X keeps pickTopN (top-25 with caps) since + // the seed-artist context already provides the "you'll like this" + // framing. + forYouTracks = pickHeadAndTail(cands, dateStr, now, forYouHeadN, forYouTailN) } else { logger.Warn("system playlist: for-you candidates load failed; skipping", "user_id", uuidStringPL(userID), "err", cerr) @@ -311,19 +369,93 @@ func BuildSystemPlaylists(ctx context.Context, pool *pgxpool.Pool, logger *slog. return nil } -// pickTopN ranks candidates with recommendation.Score, sorts by score-DESC -// breaking ties via tieBreakHash(track_id, dateStr), then truncates to N. +// pickTopN sorts candidates by score DESC, applies per-album (<=2) / +// per-artist (<=3) diversity caps matching Discover's behavior, then +// truncates to n. Used by Songs-like-X (and as the fallback inside +// pickHeadAndTail for small pools). func pickTopN(cands []recommendation.Candidate, dateStr string, now time.Time, n int) []rankedCandidate { - ranked := make([]rankedCandidate, 0, len(cands)) - for _, c := range cands { - s := recommendation.Score(c.Inputs, systemMixWeights, now, noopRNG) - ranked = append(ranked, rankedCandidate{TrackID: c.Track.ID, Score: s}) + sorted := scoreAndSortCandidates(cands, dateStr, now) + capped := capCandidatesByAlbumAndArtist(sorted) + if len(capped) > n { + capped = capped[:n] } - stableSortByScoreThenHash(ranked, dateStr) - if len(ranked) > n { - ranked = ranked[:n] + out := make([]rankedCandidate, len(capped)) + for i, c := range capped { + out[i] = rankedCandidate{ + TrackID: c.Track.ID, + Score: recommendation.Score(c.Inputs, systemMixWeights, now, noopRNG), + } } - return ranked + return out +} + +// pickHeadAndTail picks headN from the score-sorted head plus tailN from +// positions 2*headN onward (the tail), with the tail sampled +// daily-deterministically via tieBreakHash. Caps applied before the +// head/tail split. Used by For-You only. +// +// The "tail" — candidates ranked beyond 2*headN — is still similarity- +// related (every candidate passed the similarity filter) but isn't among +// the obvious top hits. Sampling from there gives the user variety they'll +// probably enjoy without resorting to genuinely random unrelated tracks. +// +// Falls back to standard pickTopN behavior when the candidate pool is too +// small to support a meaningful head/tail split (capped pool <= +// headN+tailN, or no candidates at or beyond position 2*headN). +func pickHeadAndTail(cands []recommendation.Candidate, dateStr string, now time.Time, headN, tailN int) []rankedCandidate { + sorted := scoreAndSortCandidates(cands, dateStr, now) + capped := capCandidatesByAlbumAndArtist(sorted) + + total := headN + tailN + if len(capped) <= total { + // Pool too small for a head/tail split — return up to total entries. + if len(capped) < total { + total = len(capped) + } + out := make([]rankedCandidate, total) + for i := 0; i < total; i++ { + out[i] = rankedCandidate{ + TrackID: capped[i].Track.ID, + Score: recommendation.Score(capped[i].Inputs, systemMixWeights, now, noopRNG), + } + } + return out + } + + head := capped[:headN] + + tailStart := 2 * headN + if tailStart >= len(capped) { + tailStart = headN + } + + // Defensive copy so that sorting the tail pool does not mutate capped. + tailPool := append([]recommendation.Candidate{}, capped[tailStart:]...) + + // Sort tail pool by tieBreakHash (daily-deterministic), take tailN. + // Sample is stable across requests within a day but varies across days. + sort.SliceStable(tailPool, func(i, j int) bool { + return tieBreakHash(tailPool[i].Track.ID, dateStr) < tieBreakHash(tailPool[j].Track.ID, dateStr) + }) + tail := tailPool + if len(tail) > tailN { + tail = tail[:tailN] + } + + // Combine: head order preserved (score-sorted), tail in tieBreakHash + // order. "First similar, then surprise" reads naturally in playback. + combined := make([]recommendation.Candidate, 0, len(head)+len(tail)) + combined = append(combined, head...) + combined = append(combined, tail...) + + out := make([]rankedCandidate, len(combined)) + for i, c := range combined { + out[i] = rankedCandidate{ + TrackID: c.Track.ID, + Score: recommendation.Score(c.Inputs, systemMixWeights, now, noopRNG), + } + } + return out } // insertSystemPlaylist inserts a kind='system' playlists row plus its