From 60533073adceeb83f8b1efa32c03adab3ce573f0 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Thu, 2 Jul 2026 18:00:40 -0400 Subject: [PATCH] feat(metrics): bucketed surface families + manual-plays baseline (#1248, milestone 127) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The recommendation metrics table was observable but not actionable: raw source strings (album: one-offs) drowned the stable surfaces, and manual plays were excluded so skip rates had no control group. - SQL: include NULL-source rows (the baseline) and carry completion_n so family merges can weight avg_completion correctly. - Handler buckets raw sources into stable families (radio: → Radio, album:/artist: → direct plays, etc.) grouped by surface intent: go-to / discovery / direct — each band judged against its job, since discovery mixes are expected to skip hotter. Families under 20 plays are flagged low-confidence, not hidden. - Settings card renders the baseline row and per-surface deltas in percentage points vs baseline (worse-than-baseline deltas in danger color), intent hint copy per group, low-data rows dimmed. - Pure-unit test for the bucketing/merge; DB test updated to the new contract (baseline included, radio: collapse). Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01TsF3cNoKrqCYsU78cXC8U6 --- internal/api/me_recommendation_metrics.go | 198 +++++++++++++++--- .../api/me_recommendation_metrics_test.go | 100 +++++++-- internal/db/dbq/recommendation_metrics.sql.go | 18 +- .../db/queries/recommendation_metrics.sql | 16 +- web/src/lib/api/metrics.ts | 37 ++-- web/src/routes/settings/+page.svelte | 123 ++++++++--- web/src/routes/settings/Appearance.test.ts | 5 +- web/src/routes/settings/settings.test.ts | 5 +- 8 files changed, 391 insertions(+), 111 deletions(-) diff --git a/internal/api/me_recommendation_metrics.go b/internal/api/me_recommendation_metrics.go index 05a15a75..6da64398 100644 --- a/internal/api/me_recommendation_metrics.go +++ b/internal/api/me_recommendation_metrics.go @@ -2,7 +2,9 @@ package api import ( "net/http" + "sort" "strconv" + "strings" "git.fabledsword.com/bvandeusen/minstrel/internal/apierror" "git.fabledsword.com/bvandeusen/minstrel/internal/db/dbq" @@ -11,28 +13,135 @@ import ( const ( recMetricsDefaultDays = 30 recMetricsMaxDays = 365 + + // recMetricsLowVolume marks a family as low-confidence rather than + // hiding it: with fewer plays than this a skip rate is anecdote, not + // signal, but silently dropping the row would misread as "surface + // unused". The web renders low-confidence rows dimmed. + recMetricsLowVolume = 20 ) -// recommendationMetric is one recommendation surface's outcomes. -type recommendationMetric struct { - Source string `json:"source"` // 'for_you' | 'discover' | mixes - Plays int64 `json:"plays"` // plays launched from this surface +// Surface intents (milestone #127): each family is judged against its +// job, not one global bar — discovery mixes are EXPECTED to run higher +// skip rates than the go-to surfaces. +const ( + intentGoTo = "go_to" + intentDiscovery = "discovery" + intentDirect = "direct" +) + +// surfaceMetric is one bucketed surface family's outcomes. +type surfaceMetric struct { + Key string `json:"key"` // stable family key ("for_you", "radio", …) + Label string `json:"label"` // display label + Plays int64 `json:"plays"` // plays launched from this family Skips int64 `json:"skips"` // of those, marked skipped SkipRate float64 `json:"skip_rate"` // skips / plays, [0,1] AvgCompletion float64 `json:"avg_completion"` // mean completion ratio, [0,1] + LowConfidence bool `json:"low_confidence"` // plays < recMetricsLowVolume +} + +// surfaceGroup is one intent band of surface families. +type surfaceGroup struct { + Intent string `json:"intent"` // go_to | discovery | direct + Label string `json:"label"` + Surfaces []surfaceMetric `json:"surfaces"` } type recommendationMetricsResp struct { - WindowDays int `json:"window_days"` - Sources []recommendationMetric `json:"sources"` + WindowDays int `json:"window_days"` + // Baseline is the control group: plays the user picked manually + // (source IS NULL). Surfaces are judged as deltas against it; nil + // when the window holds no manual plays. + Baseline *surfaceMetric `json:"baseline"` + Groups []surfaceGroup `json:"groups"` +} + +// recFamily is the bucketing target for a raw play_events.source value. +type recFamily struct { + key string + label string + intent string +} + +// bucketRecSource maps a raw client-stamped source string to its stable +// family. One-off sources (album:, radio:) collapse into +// their family so the table stays readable at any library size. +func bucketRecSource(src string) recFamily { + switch { + case src == "for_you": + return recFamily{"for_you", "For You", intentGoTo} + case src == "songs_like_artist": + return recFamily{"songs_like_artist", "Songs like…", intentGoTo} + case src == "radio" || strings.HasPrefix(src, "radio:"): + return recFamily{"radio", "Radio", intentGoTo} + case src == "discover": + return recFamily{"discover", "Discover", intentDiscovery} + case src == "deep_cuts": + return recFamily{"deep_cuts", "Deep cuts", intentDiscovery} + case src == "rediscover": + return recFamily{"rediscover", "Rediscover", intentDiscovery} + case src == "new_for_you": + return recFamily{"new_for_you", "New for you", intentDiscovery} + case src == "on_this_day": + return recFamily{"on_this_day", "On this day", intentDiscovery} + case src == "first_listens": + return recFamily{"first_listens", "First listens", intentDiscovery} + case strings.HasPrefix(src, "album:"): + return recFamily{"direct_album", "Album plays", intentDirect} + case strings.HasPrefix(src, "artist:"): + return recFamily{"direct_artist", "Artist plays", intentDirect} + case strings.HasPrefix(src, "offline:"): + return recFamily{"offline", "Offline pools", intentDirect} + case strings.HasPrefix(src, "home:"): + return recFamily{"home", "Home sections", intentDirect} + case src == "history": + return recFamily{"history", "History", intentDirect} + default: + return recFamily{"other", "Other", intentDirect} + } +} + +// familyAccum merges raw source rows into one family, carrying the +// completion sample count so the merged average stays play-weighted. +type familyAccum struct { + fam recFamily + plays int64 + skips int64 + completionN int64 + // completionSum is avg*count re-expanded, so merging N raw rows + // reduces to a single weighted division at the end. + completionSum float64 +} + +func (a *familyAccum) add(row dbq.RecommendationSourceMetricsForUserRow) { + a.plays += row.Plays + a.skips += row.Skips + a.completionN += row.CompletionN + a.completionSum += row.AvgCompletion * float64(row.CompletionN) +} + +func (a *familyAccum) metric() surfaceMetric { + m := surfaceMetric{ + Key: a.fam.key, + Label: a.fam.label, + Plays: a.plays, + Skips: a.skips, + LowConfidence: a.plays < recMetricsLowVolume, + } + if a.plays > 0 { + m.SkipRate = float64(a.skips) / float64(a.plays) + } + if a.completionN > 0 { + m.AvgCompletion = a.completionSum / float64(a.completionN) + } + return m } // handleGetRecommendationMetrics implements GET /api/me/recommendation-metrics. -// Per-source play outcomes (plays / skips / skip-rate / avg-completion) for the -// caller over the last `days` (default 30, capped at 365), so the operator can -// see which recommendation surfaces are landing and tune the taste weights. -// Only plays tagged with a system-playlist source count; library/radio plays -// (no source) are excluded. +// Bucketed per-surface-family outcomes for the caller over the last `days` +// (default 30, capped at 365), grouped by surface intent and anchored by the +// manual-plays baseline so the numbers are judgeable, not just observable. func (h *handlers) handleGetRecommendationMetrics(w http.ResponseWriter, r *http.Request) { caller, ok := requireUser(w, r) if !ok { @@ -51,28 +160,59 @@ func (h *handlers) handleGetRecommendationMetrics(w http.ResponseWriter, r *http return } - out := recommendationMetricsResp{ - WindowDays: days, - Sources: make([]recommendationMetric, 0, len(rows)), - } + writeJSON(w, http.StatusOK, bucketMetricsResponse(days, rows)) +} + +// bucketMetricsResponse folds the raw per-source rows into the grouped, +// baseline-anchored response shape. Split from the handler for pure-unit +// testability. +func bucketMetricsResponse( + days int, rows []dbq.RecommendationSourceMetricsForUserRow, +) recommendationMetricsResp { + baseline := &familyAccum{fam: recFamily{"manual", "Manual library plays", ""}} + families := map[string]*familyAccum{} for _, row := range rows { - source := "" - if row.Source != nil { - source = *row.Source + if row.Source == nil || *row.Source == "" { + baseline.add(row) + continue } - var skipRate float64 - if row.Plays > 0 { - skipRate = float64(row.Skips) / float64(row.Plays) + fam := bucketRecSource(*row.Source) + acc, exists := families[fam.key] + if !exists { + acc = &familyAccum{fam: fam} + families[fam.key] = acc } - out.Sources = append(out.Sources, recommendationMetric{ - Source: source, - Plays: row.Plays, - Skips: row.Skips, - SkipRate: skipRate, - AvgCompletion: row.AvgCompletion, - }) + acc.add(row) } - writeJSON(w, http.StatusOK, out) + + resp := recommendationMetricsResp{WindowDays: days, Groups: []surfaceGroup{}} + if baseline.plays > 0 { + m := baseline.metric() + resp.Baseline = &m + } + for _, g := range []struct{ intent, label string }{ + {intentGoTo, "Go-to surfaces"}, + {intentDiscovery, "Discovery mixes"}, + {intentDirect, "Direct plays"}, + } { + group := surfaceGroup{Intent: g.intent, Label: g.label} + for _, acc := range families { + if acc.fam.intent == g.intent { + group.Surfaces = append(group.Surfaces, acc.metric()) + } + } + if len(group.Surfaces) == 0 { + continue + } + sort.Slice(group.Surfaces, func(i, j int) bool { + if group.Surfaces[i].Plays != group.Surfaces[j].Plays { + return group.Surfaces[i].Plays > group.Surfaces[j].Plays + } + return group.Surfaces[i].Key < group.Surfaces[j].Key + }) + resp.Groups = append(resp.Groups, group) + } + return resp } // parseMetricsDays reads the `days` query param (default 30, capped at 365). diff --git a/internal/api/me_recommendation_metrics_test.go b/internal/api/me_recommendation_metrics_test.go index 2a1af4c3..1c1f49dc 100644 --- a/internal/api/me_recommendation_metrics_test.go +++ b/internal/api/me_recommendation_metrics_test.go @@ -11,6 +11,8 @@ import ( "github.com/go-chi/chi/v5" "github.com/jackc/pgx/v5/pgtype" + + "git.fabledsword.com/bvandeusen/minstrel/internal/db/dbq" ) func newMetricsRouter(h *handlers) chi.Router { @@ -20,7 +22,7 @@ func newMetricsRouter(h *handlers) chi.Router { } // seedSourcedPlay inserts a play_event with an explicit source + completion + -// skip flag. A nil source inserts NULL (library/radio play). +// skip flag. A nil source inserts NULL (manual library play → baseline). func seedSourcedPlay( t *testing.T, h *handlers, userID, trackID, sessionID pgtype.UUID, source *string, completion float64, skipped bool, @@ -45,7 +47,77 @@ func TestRecommendationMetrics_NoSession401(t *testing.T) { } } -func TestRecommendationMetrics_AggregatesBySourceExcludingNull(t *testing.T) { +// findSurface returns the named family from any group, or nil. +func findSurface(resp recommendationMetricsResp, key string) *surfaceMetric { + for _, g := range resp.Groups { + for i := range g.Surfaces { + if g.Surfaces[i].Key == key { + return &g.Surfaces[i] + } + } + } + return nil +} + +func TestBucketMetricsResponse_FamiliesGroupsBaseline(t *testing.T) { + src := func(s string) *string { return &s } + rows := []dbq.RecommendationSourceMetricsForUserRow{ + {Source: src("for_you"), Plays: 3, Skips: 1, CompletionN: 3, AvgCompletion: 2.0 / 3}, + // Two radio sessions collapse into one "radio" family; weighted + // completion = (0.2*1 + 0.8*1) / 2 = 0.5. + {Source: src("radio:aaaa"), Plays: 1, Skips: 1, CompletionN: 1, AvgCompletion: 0.2}, + {Source: src("radio:bbbb"), Plays: 1, Skips: 0, CompletionN: 1, AvgCompletion: 0.8}, + {Source: src("album:cccc"), Plays: 1, Skips: 0, CompletionN: 0, AvgCompletion: 0}, + {Source: src("discover"), Plays: 1, Skips: 0, CompletionN: 1, AvgCompletion: 0.8}, + // NULL source = manual plays → baseline, not a group row. + {Source: nil, Plays: 25, Skips: 5, CompletionN: 20, AvgCompletion: 0.9}, + } + resp := bucketMetricsResponse(recMetricsDefaultDays, rows) + + if resp.Baseline == nil { + t.Fatal("baseline missing") + } + if resp.Baseline.Plays != 25 || resp.Baseline.SkipRate != 0.2 { + t.Errorf("baseline = %+v, want plays=25 skip_rate=0.2", resp.Baseline) + } + if resp.Baseline.LowConfidence { + t.Error("baseline with 25 plays should not be low-confidence") + } + + radio := findSurface(resp, "radio") + if radio == nil { + t.Fatal("radio family missing") + } + if radio.Plays != 2 || radio.Skips != 1 { + t.Errorf("radio plays/skips = %d/%d, want 2/1", radio.Plays, radio.Skips) + } + if radio.AvgCompletion < 0.49 || radio.AvgCompletion > 0.51 { + t.Errorf("radio avg_completion = %.3f, want 0.5 (play-weighted merge)", radio.AvgCompletion) + } + if !radio.LowConfidence { + t.Error("radio with 2 plays should be low-confidence") + } + + if s := findSurface(resp, "direct_album"); s == nil || s.Plays != 1 { + t.Errorf("direct_album = %+v, want plays=1", s) + } + if s := findSurface(resp, ""); s != nil { + t.Error("NULL source must not appear as a surface family") + } + + // Group ordering is intent-banded: go_to before discovery before direct. + wantOrder := []string{intentGoTo, intentDiscovery, intentDirect} + if len(resp.Groups) != len(wantOrder) { + t.Fatalf("groups = %d, want %d", len(resp.Groups), len(wantOrder)) + } + for i, g := range resp.Groups { + if g.Intent != wantOrder[i] { + t.Errorf("group[%d].intent = %s, want %s", i, g.Intent, wantOrder[i]) + } + } +} + +func TestRecommendationMetrics_BucketsWithBaseline(t *testing.T) { if os.Getenv("MINSTREL_TEST_DATABASE_URL") == "" { t.Skip("MINSTREL_TEST_DATABASE_URL not set") } @@ -57,14 +129,14 @@ func TestRecommendationMetrics_AggregatesBySourceExcludingNull(t *testing.T) { session := seedPlaySession(t, pool, user.ID, time.Now()) forYou := "for_you" - discover := "discover" + radioA := "radio:11111111-1111-1111-1111-111111111111" // for_you: 3 plays, 1 skipped; completions 1.0, 0.95, 0.05 → mean 0.6667. seedSourcedPlay(t, h, user.ID, tk.ID, session, &forYou, 1.0, false) seedSourcedPlay(t, h, user.ID, tk.ID, session, &forYou, 0.95, false) seedSourcedPlay(t, h, user.ID, tk.ID, session, &forYou, 0.05, true) - // discover: 1 play. - seedSourcedPlay(t, h, user.ID, tk.ID, session, &discover, 0.8, false) - // library play (NULL source) — must be excluded. + // A radio session play collapses into the "radio" family. + seedSourcedPlay(t, h, user.ID, tk.ID, session, &radioA, 0.8, false) + // Manual play (NULL source) — the baseline row. seedSourcedPlay(t, h, user.ID, tk.ID, session, nil, 1.0, false) req := httptest.NewRequest(http.MethodGet, "/api/me/recommendation-metrics", nil) @@ -82,15 +154,11 @@ func TestRecommendationMetrics_AggregatesBySourceExcludingNull(t *testing.T) { if resp.WindowDays != recMetricsDefaultDays { t.Errorf("window_days = %d, want %d", resp.WindowDays, recMetricsDefaultDays) } - bySource := map[string]recommendationMetric{} - for _, m := range resp.Sources { - bySource[m.Source] = m + if resp.Baseline == nil || resp.Baseline.Plays != 1 { + t.Fatalf("baseline = %+v, want plays=1", resp.Baseline) } - if _, present := bySource[""]; present { - t.Error("NULL-source (library) plays should be excluded") - } - fy, ok := bySource["for_you"] - if !ok { + fy := findSurface(resp, "for_you") + if fy == nil { t.Fatal("for_you metrics missing") } if fy.Plays != 3 || fy.Skips != 1 { @@ -102,7 +170,7 @@ func TestRecommendationMetrics_AggregatesBySourceExcludingNull(t *testing.T) { if fy.AvgCompletion < 0.66 || fy.AvgCompletion > 0.67 { t.Errorf("for_you avg_completion = %.4f, want ~0.6667", fy.AvgCompletion) } - if d, ok := bySource["discover"]; !ok || d.Plays != 1 || d.Skips != 0 { - t.Errorf("discover metrics = %+v, want plays=1 skips=0", d) + if radio := findSurface(resp, "radio"); radio == nil || radio.Plays != 1 { + t.Errorf("radio family = %+v, want plays=1 (collapsed from radio:)", radio) } } diff --git a/internal/db/dbq/recommendation_metrics.sql.go b/internal/db/dbq/recommendation_metrics.sql.go index db1f6e77..acace5c9 100644 --- a/internal/db/dbq/recommendation_metrics.sql.go +++ b/internal/db/dbq/recommendation_metrics.sql.go @@ -17,12 +17,10 @@ SELECT pe.source, count(*)::bigint AS plays, count(*) FILTER (WHERE pe.was_skipped)::bigint AS skips, - COALESCE( - avg(pe.completion_ratio) FILTER (WHERE pe.completion_ratio IS NOT NULL), - 0)::float8 AS avg_completion + count(pe.completion_ratio)::bigint AS completion_n, + COALESCE(avg(pe.completion_ratio), 0)::float8 AS avg_completion FROM play_events pe WHERE pe.user_id = $1 - AND pe.source IS NOT NULL AND pe.started_at > now() - ($2::float8 * INTERVAL '1 day') GROUP BY pe.source ORDER BY plays DESC @@ -37,16 +35,21 @@ type RecommendationSourceMetricsForUserRow struct { Source *string Plays int64 Skips int64 + CompletionN int64 AvgCompletion float64 } // Recommendation observability (#796 phase 4). Per-source play outcomes so the // operator can see whether each recommendation surface is landing and tune the // taste weights. Source is stamped on play_events when a play is launched from -// a system-playlist surface ('for_you' | 'discover' | the discovery mixes); -// NULL for library / radio / user-playlist plays, which are excluded here. +// a recommendation surface; NULL means the user picked the track manually — +// those rows are INCLUDED here as the baseline control group the surfaces are +// judged against (milestone #127: delta-vs-baseline is what makes the numbers +// actionable). Raw source strings are bucketed into stable surface families in +// the Go handler; completion_n is carried so family merges can weight +// avg_completion correctly. // $1 user_id, $2 window_days. plays/skips are counts; avg_completion is the -// mean completion ratio over plays that recorded one (0 when none did). +// mean completion ratio over the completion_n plays that recorded one. func (q *Queries) RecommendationSourceMetricsForUser(ctx context.Context, arg RecommendationSourceMetricsForUserParams) ([]RecommendationSourceMetricsForUserRow, error) { rows, err := q.db.Query(ctx, recommendationSourceMetricsForUser, arg.UserID, arg.Column2) if err != nil { @@ -60,6 +63,7 @@ func (q *Queries) RecommendationSourceMetricsForUser(ctx context.Context, arg Re &i.Source, &i.Plays, &i.Skips, + &i.CompletionN, &i.AvgCompletion, ); err != nil { return nil, err diff --git a/internal/db/queries/recommendation_metrics.sql b/internal/db/queries/recommendation_metrics.sql index c7d04159..2d2f4ebd 100644 --- a/internal/db/queries/recommendation_metrics.sql +++ b/internal/db/queries/recommendation_metrics.sql @@ -1,22 +1,24 @@ -- Recommendation observability (#796 phase 4). Per-source play outcomes so the -- operator can see whether each recommendation surface is landing and tune the -- taste weights. Source is stamped on play_events when a play is launched from --- a system-playlist surface ('for_you' | 'discover' | the discovery mixes); --- NULL for library / radio / user-playlist plays, which are excluded here. +-- a recommendation surface; NULL means the user picked the track manually — +-- those rows are INCLUDED here as the baseline control group the surfaces are +-- judged against (milestone #127: delta-vs-baseline is what makes the numbers +-- actionable). Raw source strings are bucketed into stable surface families in +-- the Go handler; completion_n is carried so family merges can weight +-- avg_completion correctly. -- name: RecommendationSourceMetricsForUser :many -- $1 user_id, $2 window_days. plays/skips are counts; avg_completion is the --- mean completion ratio over plays that recorded one (0 when none did). +-- mean completion ratio over the completion_n plays that recorded one. SELECT pe.source, count(*)::bigint AS plays, count(*) FILTER (WHERE pe.was_skipped)::bigint AS skips, - COALESCE( - avg(pe.completion_ratio) FILTER (WHERE pe.completion_ratio IS NOT NULL), - 0)::float8 AS avg_completion + count(pe.completion_ratio)::bigint AS completion_n, + COALESCE(avg(pe.completion_ratio), 0)::float8 AS avg_completion FROM play_events pe WHERE pe.user_id = $1 - AND pe.source IS NOT NULL AND pe.started_at > now() - ($2::float8 * INTERVAL '1 day') GROUP BY pe.source ORDER BY plays DESC; diff --git a/web/src/lib/api/metrics.ts b/web/src/lib/api/metrics.ts index be1fb87d..9a21df3b 100644 --- a/web/src/lib/api/metrics.ts +++ b/web/src/lib/api/metrics.ts @@ -1,18 +1,31 @@ import { createQuery } from '@tanstack/svelte-query'; import { api } from './client'; -// Mirrors internal/api/me_recommendation_metrics.go. -export type RecommendationMetric = { - source: string; +// Mirrors internal/api/me_recommendation_metrics.go: raw play sources are +// bucketed server-side into stable surface families, grouped by intent, and +// anchored by the manual-plays baseline (milestone #127). +export type SurfaceMetric = { + key: string; + label: string; plays: number; skips: number; skip_rate: number; avg_completion: number; + low_confidence: boolean; +}; + +export type SurfaceIntent = 'go_to' | 'discovery' | 'direct'; + +export type SurfaceGroup = { + intent: SurfaceIntent; + label: string; + surfaces: SurfaceMetric[]; }; export type RecommendationMetrics = { window_days: number; - sources: RecommendationMetric[]; + baseline: SurfaceMetric | null; + groups: SurfaceGroup[]; }; export function getRecommendationMetrics(): Promise { @@ -28,19 +41,3 @@ export function createRecommendationMetricsQuery() { staleTime: 60_000 }); } - -// Friendly labels for the system-playlist source keys (play_events.source). -const SOURCE_LABELS: Record = { - for_you: 'For You', - discover: 'Discover', - deep_cuts: 'Deep cuts', - rediscover: 'Rediscover', - new_for_you: 'New for you', - on_this_day: 'On this day', - first_listens: 'First listens', - songs_like_artist: 'Songs like…' -}; - -export function sourceLabel(source: string): string { - return SOURCE_LABELS[source] ?? source; -} diff --git a/web/src/routes/settings/+page.svelte b/web/src/routes/settings/+page.svelte index eef29ee9..b32970fa 100644 --- a/web/src/routes/settings/+page.svelte +++ b/web/src/routes/settings/+page.svelte @@ -10,8 +10,9 @@ } from '$lib/api/listenbrainz'; import { createRecommendationMetricsQuery, - sourceLabel, - type RecommendationMetrics + type RecommendationMetrics, + type SurfaceIntent, + type SurfaceMetric } from '$lib/api/metrics'; import { theme, setTheme, type ThemePreference } from '$lib/stores/theme.svelte'; import { player, setCrossfade } from '$lib/player/store.svelte'; @@ -30,6 +31,34 @@ const status = createLBStatusQuery() as CreateQueryResult; const metrics = createRecommendationMetricsQuery() as CreateQueryResult; + + // Per-intent expectation copy: each surface band is judged against its + // job — discovery mixes are supposed to run hotter skip rates. + const intentHints: Record = { + go_to: 'Everyday surfaces — these should beat your own picking.', + discovery: 'Exploration surfaces — higher skip rates here are expected and healthy.', + direct: 'Music you chose yourself — naturally close to the baseline.' + }; + + function pct(v: number): string { + return `${(v * 100).toFixed(0)}%`; + } + + // Delta in percentage points vs the baseline, signed ("+12" / "−5"). + function deltaPts(value: number, baseline: number): string { + const pts = Math.round((value - baseline) * 100); + return pts > 0 ? `+${pts}` : `${pts}`; + } + + // A surface's skip delta is "worse" when it skips more than the + // baseline; completion delta is "worse" when it completes less. + function skipDeltaClass(m: SurfaceMetric, baseline: SurfaceMetric): string { + return m.skip_rate > baseline.skip_rate ? 'text-danger' : 'text-text-secondary'; + } + + function completionDeltaClass(m: SurfaceMetric, baseline: SurfaceMetric): string { + return m.avg_completion < baseline.avg_completion ? 'text-danger' : 'text-text-secondary'; + } const tokenMutation = createTokenMutation(queryClient); const enabledMutation = createEnabledMutation(queryClient); @@ -278,37 +307,79 @@

Recommendation metrics

How plays launched from each recommendation surface land, over the last - {$metrics.data?.window_days ?? 30} days. Lower skip rate and higher average - completion mean the surface is hitting. + {$metrics.data?.window_days ?? 30} days — compared against the baseline of + music you picked yourself. Deltas are percentage points vs that baseline.

{#if $metrics.isPending}

Loading…

{:else if $metrics.isError}

Couldn't load metrics.

- {:else if $metrics.data && $metrics.data.sources.length > 0} - - - - - - - - - - - {#each $metrics.data.sources as m (m.source)} - - - - - - - {/each} - -
SurfacePlaysSkip rateAvg completion
{sourceLabel(m.source)}{m.plays}{(m.skip_rate * 100).toFixed(0)}%{(m.avg_completion * 100).toFixed(0)}%
+ {:else if $metrics.data && ($metrics.data.groups.length > 0 || $metrics.data.baseline)} + {@const baseline = $metrics.data.baseline} + {#if baseline} +

+ Baseline — manual plays: + + {baseline.plays} plays · {pct(baseline.skip_rate)} skip · + {pct(baseline.avg_completion)} completion + +

+ {:else} +

+ No manual plays in this window yet, so deltas are hidden — raw rates only. +

+ {/if} + {#each $metrics.data.groups as group (group.intent)} +
+

{group.label}

+

{intentHints[group.intent]}

+ + + + + + + + + + + {#each group.surfaces as m (m.key)} + + + + + + + {/each} + +
SurfacePlaysSkip rateAvg completion
+ {m.label}{#if m.low_confidence}· low data{/if} + {m.plays} + {pct(m.skip_rate)} + {#if baseline} + + {deltaPts(m.skip_rate, baseline.skip_rate)} + + {/if} + + {pct(m.avg_completion)} + {#if baseline} + + {deltaPts(m.avg_completion, baseline.avg_completion)} + + {/if} +
+
+ {/each} {:else}

- No recommendation plays yet. Play something from For You, Discover, or a mix. + No plays recorded yet. Play something from For You, Discover, or a mix.

{/if} diff --git a/web/src/routes/settings/Appearance.test.ts b/web/src/routes/settings/Appearance.test.ts index d254e97b..67407e35 100644 --- a/web/src/routes/settings/Appearance.test.ts +++ b/web/src/routes/settings/Appearance.test.ts @@ -28,11 +28,10 @@ vi.mock('$lib/api/listenbrainz', () => { vi.mock('$lib/api/metrics', () => ({ createRecommendationMetricsQuery: () => ({ subscribe: (run: (v: unknown) => void) => { - run({ isPending: false, isError: false, data: { window_days: 30, sources: [] } }); + run({ isPending: false, isError: false, data: { window_days: 30, baseline: null, groups: [] } }); return () => {}; } - }), - sourceLabel: (s: string) => s + }) })); beforeEach(() => { diff --git a/web/src/routes/settings/settings.test.ts b/web/src/routes/settings/settings.test.ts index ae913c07..1347c0d2 100644 --- a/web/src/routes/settings/settings.test.ts +++ b/web/src/routes/settings/settings.test.ts @@ -23,11 +23,10 @@ vi.mock('$lib/api/me', () => ({ vi.mock('$lib/api/metrics', () => ({ createRecommendationMetricsQuery: () => ({ subscribe: (run: (v: unknown) => void) => { - run({ isPending: false, isError: false, data: { window_days: 30, sources: [] } }); + run({ isPending: false, isError: false, data: { window_days: 30, baseline: null, groups: [] } }); return () => {}; } - }), - sourceLabel: (s: string) => s + }) })); import SettingsPage from './+page.svelte';