From 1a7515e6ea3018fc4107d2b94885e873765622e2 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Fri, 12 Jun 2026 00:28:30 -0400 Subject: [PATCH] =?UTF-8?q?feat(taste):=20phase=204=20=E2=80=94=20recommen?= =?UTF-8?q?dation=20observability=20(#796)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Per-source play outcomes so the operator can see whether each recommendation surface is landing and tune the now-operator-tunable taste weights. Server: - query RecommendationSourceMetricsForUser: groups the user's play_events by source (system-playlist surface), reporting plays / skips / avg completion over a window; NULL-source (library/radio) plays excluded. - GET /api/me/recommendation-metrics?days=30 (default 30, capped 365) → {window_days, sources:[{source, plays, skips, skip_rate, avg_completion}]}. - handler test: 401 unauth; per-source aggregation + NULL-source exclusion + skip_rate / avg_completion math. Web: - lib/api/metrics.ts: query + friendly source labels. - settings page gains a "Recommendation metrics" card (table of surface / plays / skip rate / avg completion), with loading/error/empty states. - settings tests mock the new query (manual subscribe-store, hoisting-safe). Note: You-might-like plays aren't source-tagged (it's a Home row, not a system playlist), so this covers For-You / Discover / the mixes. Tagging YML plays would be a client follow-up. Co-Authored-By: Claude Opus 4.8 (1M context) --- internal/api/api.go | 1 + internal/api/me_recommendation_metrics.go | 94 +++++++++++++++ .../api/me_recommendation_metrics_test.go | 108 ++++++++++++++++++ internal/db/dbq/recommendation_metrics.sql.go | 73 ++++++++++++ .../db/queries/recommendation_metrics.sql | 22 ++++ web/src/lib/api/metrics.ts | 46 ++++++++ web/src/routes/settings/+page.svelte | 46 ++++++++ web/src/routes/settings/Appearance.test.ts | 10 ++ web/src/routes/settings/settings.test.ts | 10 ++ 9 files changed, 410 insertions(+) create mode 100644 internal/api/me_recommendation_metrics.go create mode 100644 internal/api/me_recommendation_metrics_test.go create mode 100644 internal/db/dbq/recommendation_metrics.sql.go create mode 100644 internal/db/queries/recommendation_metrics.sql create mode 100644 web/src/lib/api/metrics.ts diff --git a/internal/api/api.go b/internal/api/api.go index ff425bb9..4639cca2 100644 --- a/internal/api/api.go +++ b/internal/api/api.go @@ -74,6 +74,7 @@ func Mount(r chi.Router, pool *pgxpool.Pool, logger *slog.Logger, events *playev authed.Post("/auth/logout", h.handleLogout) authed.Get("/me", h.handleGetMe) authed.Get("/me/system-playlists-status", h.handleGetSystemPlaylistsStatus) + authed.Get("/me/recommendation-metrics", h.handleGetRecommendationMetrics) authed.Get("/me/listenbrainz", h.handleGetListenBrainz) authed.Put("/me/listenbrainz", h.handlePutListenBrainz) authed.Get("/me/history", h.handleGetMyHistory) diff --git a/internal/api/me_recommendation_metrics.go b/internal/api/me_recommendation_metrics.go new file mode 100644 index 00000000..05a15a75 --- /dev/null +++ b/internal/api/me_recommendation_metrics.go @@ -0,0 +1,94 @@ +package api + +import ( + "net/http" + "strconv" + + "git.fabledsword.com/bvandeusen/minstrel/internal/apierror" + "git.fabledsword.com/bvandeusen/minstrel/internal/db/dbq" +) + +const ( + recMetricsDefaultDays = 30 + recMetricsMaxDays = 365 +) + +// recommendationMetric is one recommendation surface's outcomes. +type recommendationMetric struct { + Source string `json:"source"` // 'for_you' | 'discover' | mixes + Plays int64 `json:"plays"` // plays launched from this surface + Skips int64 `json:"skips"` // of those, marked skipped + SkipRate float64 `json:"skip_rate"` // skips / plays, [0,1] + AvgCompletion float64 `json:"avg_completion"` // mean completion ratio, [0,1] +} + +type recommendationMetricsResp struct { + WindowDays int `json:"window_days"` + Sources []recommendationMetric `json:"sources"` +} + +// handleGetRecommendationMetrics implements GET /api/me/recommendation-metrics. +// Per-source play outcomes (plays / skips / skip-rate / avg-completion) for the +// caller over the last `days` (default 30, capped at 365), so the operator can +// see which recommendation surfaces are landing and tune the taste weights. +// Only plays tagged with a system-playlist source count; library/radio plays +// (no source) are excluded. +func (h *handlers) handleGetRecommendationMetrics(w http.ResponseWriter, r *http.Request) { + caller, ok := requireUser(w, r) + if !ok { + return + } + days, ok := parseMetricsDays(w, r) + if !ok { + return + } + + rows, err := dbq.New(h.pool).RecommendationSourceMetricsForUser(r.Context(), + dbq.RecommendationSourceMetricsForUserParams{UserID: caller.ID, Column2: float64(days)}) + if err != nil { + h.logger.Error("api: recommendation metrics", "err", err) + writeErr(w, apierror.InternalMsg("lookup failed", err)) + return + } + + out := recommendationMetricsResp{ + WindowDays: days, + Sources: make([]recommendationMetric, 0, len(rows)), + } + for _, row := range rows { + source := "" + if row.Source != nil { + source = *row.Source + } + var skipRate float64 + if row.Plays > 0 { + skipRate = float64(row.Skips) / float64(row.Plays) + } + out.Sources = append(out.Sources, recommendationMetric{ + Source: source, + Plays: row.Plays, + Skips: row.Skips, + SkipRate: skipRate, + AvgCompletion: row.AvgCompletion, + }) + } + writeJSON(w, http.StatusOK, out) +} + +// parseMetricsDays reads the `days` query param (default 30, capped at 365). +// Writes a 400 and returns ok=false on a malformed value. +func parseMetricsDays(w http.ResponseWriter, r *http.Request) (int, bool) { + v := r.URL.Query().Get("days") + if v == "" { + return recMetricsDefaultDays, true + } + n, err := strconv.Atoi(v) + if err != nil || n < 1 { + writeErr(w, apierror.BadRequest("bad_request", "invalid days")) + return 0, false + } + if n > recMetricsMaxDays { + n = recMetricsMaxDays + } + return n, true +} diff --git a/internal/api/me_recommendation_metrics_test.go b/internal/api/me_recommendation_metrics_test.go new file mode 100644 index 00000000..2a1af4c3 --- /dev/null +++ b/internal/api/me_recommendation_metrics_test.go @@ -0,0 +1,108 @@ +package api + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "os" + "testing" + "time" + + "github.com/go-chi/chi/v5" + "github.com/jackc/pgx/v5/pgtype" +) + +func newMetricsRouter(h *handlers) chi.Router { + r := chi.NewRouter() + r.Get("/api/me/recommendation-metrics", h.handleGetRecommendationMetrics) + return r +} + +// seedSourcedPlay inserts a play_event with an explicit source + completion + +// skip flag. A nil source inserts NULL (library/radio play). +func seedSourcedPlay( + t *testing.T, h *handlers, userID, trackID, sessionID pgtype.UUID, + source *string, completion float64, skipped bool, +) { + t.Helper() + if _, err := h.pool.Exec(context.Background(), + `INSERT INTO play_events + (user_id, track_id, session_id, started_at, source, completion_ratio, was_skipped) + VALUES ($1, $2, $3, now(), $4, $5, $6)`, + userID, trackID, sessionID, source, completion, skipped); err != nil { + t.Fatalf("seed sourced play: %v", err) + } +} + +func TestRecommendationMetrics_NoSession401(t *testing.T) { + h, _ := testHandlers(t) + req := httptest.NewRequest(http.MethodGet, "/api/me/recommendation-metrics", nil) + rec := httptest.NewRecorder() + newMetricsRouter(h).ServeHTTP(rec, req) + if rec.Code != http.StatusUnauthorized { + t.Errorf("status = %d, want 401", rec.Code) + } +} + +func TestRecommendationMetrics_AggregatesBySourceExcludingNull(t *testing.T) { + if os.Getenv("MINSTREL_TEST_DATABASE_URL") == "" { + t.Skip("MINSTREL_TEST_DATABASE_URL not set") + } + h, pool := testHandlers(t) + user := seedUser(t, pool, "metrics", "pw", false) + artist := seedArtist(t, pool, "MetricArtist") + album := seedAlbum(t, pool, artist.ID, "MetricAlbum", 2020) + tk := seedTrack(t, pool, album.ID, artist.ID, "MetricTrack", 1, 200000) + session := seedPlaySession(t, pool, user.ID, time.Now()) + + forYou := "for_you" + discover := "discover" + // for_you: 3 plays, 1 skipped; completions 1.0, 0.95, 0.05 → mean 0.6667. + seedSourcedPlay(t, h, user.ID, tk.ID, session, &forYou, 1.0, false) + seedSourcedPlay(t, h, user.ID, tk.ID, session, &forYou, 0.95, false) + seedSourcedPlay(t, h, user.ID, tk.ID, session, &forYou, 0.05, true) + // discover: 1 play. + seedSourcedPlay(t, h, user.ID, tk.ID, session, &discover, 0.8, false) + // library play (NULL source) — must be excluded. + seedSourcedPlay(t, h, user.ID, tk.ID, session, nil, 1.0, false) + + req := httptest.NewRequest(http.MethodGet, "/api/me/recommendation-metrics", nil) + req = withUser(req, user) + rec := httptest.NewRecorder() + newMetricsRouter(h).ServeHTTP(rec, req) + if rec.Code != http.StatusOK { + t.Fatalf("status = %d, want 200", rec.Code) + } + + var resp recommendationMetricsResp + if err := json.Unmarshal(rec.Body.Bytes(), &resp); err != nil { + t.Fatalf("decode: %v", err) + } + if resp.WindowDays != recMetricsDefaultDays { + t.Errorf("window_days = %d, want %d", resp.WindowDays, recMetricsDefaultDays) + } + bySource := map[string]recommendationMetric{} + for _, m := range resp.Sources { + bySource[m.Source] = m + } + if _, present := bySource[""]; present { + t.Error("NULL-source (library) plays should be excluded") + } + fy, ok := bySource["for_you"] + if !ok { + t.Fatal("for_you metrics missing") + } + if fy.Plays != 3 || fy.Skips != 1 { + t.Errorf("for_you plays/skips = %d/%d, want 3/1", fy.Plays, fy.Skips) + } + if fy.SkipRate < 0.32 || fy.SkipRate > 0.34 { + t.Errorf("for_you skip_rate = %.3f, want ~0.333", fy.SkipRate) + } + if fy.AvgCompletion < 0.66 || fy.AvgCompletion > 0.67 { + t.Errorf("for_you avg_completion = %.4f, want ~0.6667", fy.AvgCompletion) + } + if d, ok := bySource["discover"]; !ok || d.Plays != 1 || d.Skips != 0 { + t.Errorf("discover metrics = %+v, want plays=1 skips=0", d) + } +} diff --git a/internal/db/dbq/recommendation_metrics.sql.go b/internal/db/dbq/recommendation_metrics.sql.go new file mode 100644 index 00000000..db1f6e77 --- /dev/null +++ b/internal/db/dbq/recommendation_metrics.sql.go @@ -0,0 +1,73 @@ +// Code generated by sqlc. DO NOT EDIT. +// versions: +// sqlc v1.31.1 +// source: recommendation_metrics.sql + +package dbq + +import ( + "context" + + "github.com/jackc/pgx/v5/pgtype" +) + +const recommendationSourceMetricsForUser = `-- name: RecommendationSourceMetricsForUser :many + +SELECT + pe.source, + count(*)::bigint AS plays, + count(*) FILTER (WHERE pe.was_skipped)::bigint AS skips, + COALESCE( + avg(pe.completion_ratio) FILTER (WHERE pe.completion_ratio IS NOT NULL), + 0)::float8 AS avg_completion +FROM play_events pe +WHERE pe.user_id = $1 + AND pe.source IS NOT NULL + AND pe.started_at > now() - ($2::float8 * INTERVAL '1 day') +GROUP BY pe.source +ORDER BY plays DESC +` + +type RecommendationSourceMetricsForUserParams struct { + UserID pgtype.UUID + Column2 float64 +} + +type RecommendationSourceMetricsForUserRow struct { + Source *string + Plays int64 + Skips int64 + AvgCompletion float64 +} + +// Recommendation observability (#796 phase 4). Per-source play outcomes so the +// operator can see whether each recommendation surface is landing and tune the +// taste weights. Source is stamped on play_events when a play is launched from +// a system-playlist surface ('for_you' | 'discover' | the discovery mixes); +// NULL for library / radio / user-playlist plays, which are excluded here. +// $1 user_id, $2 window_days. plays/skips are counts; avg_completion is the +// mean completion ratio over plays that recorded one (0 when none did). +func (q *Queries) RecommendationSourceMetricsForUser(ctx context.Context, arg RecommendationSourceMetricsForUserParams) ([]RecommendationSourceMetricsForUserRow, error) { + rows, err := q.db.Query(ctx, recommendationSourceMetricsForUser, arg.UserID, arg.Column2) + if err != nil { + return nil, err + } + defer rows.Close() + var items []RecommendationSourceMetricsForUserRow + for rows.Next() { + var i RecommendationSourceMetricsForUserRow + if err := rows.Scan( + &i.Source, + &i.Plays, + &i.Skips, + &i.AvgCompletion, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} diff --git a/internal/db/queries/recommendation_metrics.sql b/internal/db/queries/recommendation_metrics.sql new file mode 100644 index 00000000..c7d04159 --- /dev/null +++ b/internal/db/queries/recommendation_metrics.sql @@ -0,0 +1,22 @@ +-- Recommendation observability (#796 phase 4). Per-source play outcomes so the +-- operator can see whether each recommendation surface is landing and tune the +-- taste weights. Source is stamped on play_events when a play is launched from +-- a system-playlist surface ('for_you' | 'discover' | the discovery mixes); +-- NULL for library / radio / user-playlist plays, which are excluded here. + +-- name: RecommendationSourceMetricsForUser :many +-- $1 user_id, $2 window_days. plays/skips are counts; avg_completion is the +-- mean completion ratio over plays that recorded one (0 when none did). +SELECT + pe.source, + count(*)::bigint AS plays, + count(*) FILTER (WHERE pe.was_skipped)::bigint AS skips, + COALESCE( + avg(pe.completion_ratio) FILTER (WHERE pe.completion_ratio IS NOT NULL), + 0)::float8 AS avg_completion +FROM play_events pe +WHERE pe.user_id = $1 + AND pe.source IS NOT NULL + AND pe.started_at > now() - ($2::float8 * INTERVAL '1 day') +GROUP BY pe.source +ORDER BY plays DESC; diff --git a/web/src/lib/api/metrics.ts b/web/src/lib/api/metrics.ts new file mode 100644 index 00000000..be1fb87d --- /dev/null +++ b/web/src/lib/api/metrics.ts @@ -0,0 +1,46 @@ +import { createQuery } from '@tanstack/svelte-query'; +import { api } from './client'; + +// Mirrors internal/api/me_recommendation_metrics.go. +export type RecommendationMetric = { + source: string; + plays: number; + skips: number; + skip_rate: number; + avg_completion: number; +}; + +export type RecommendationMetrics = { + window_days: number; + sources: RecommendationMetric[]; +}; + +export function getRecommendationMetrics(): Promise { + return api.get('/api/me/recommendation-metrics'); +} + +export const REC_METRICS_QUERY_KEY = ['settings', 'recommendation-metrics'] as const; + +export function createRecommendationMetricsQuery() { + return createQuery({ + queryKey: REC_METRICS_QUERY_KEY, + queryFn: getRecommendationMetrics, + staleTime: 60_000 + }); +} + +// Friendly labels for the system-playlist source keys (play_events.source). +const SOURCE_LABELS: Record = { + for_you: 'For You', + discover: 'Discover', + deep_cuts: 'Deep cuts', + rediscover: 'Rediscover', + new_for_you: 'New for you', + on_this_day: 'On this day', + first_listens: 'First listens', + songs_like_artist: 'Songs like…' +}; + +export function sourceLabel(source: string): string { + return SOURCE_LABELS[source] ?? source; +} diff --git a/web/src/routes/settings/+page.svelte b/web/src/routes/settings/+page.svelte index 453b5245..eef29ee9 100644 --- a/web/src/routes/settings/+page.svelte +++ b/web/src/routes/settings/+page.svelte @@ -8,6 +8,11 @@ createEnabledMutation, type LBStatus } from '$lib/api/listenbrainz'; + import { + createRecommendationMetricsQuery, + sourceLabel, + type RecommendationMetrics + } from '$lib/api/metrics'; import { theme, setTheme, type ThemePreference } from '$lib/stores/theme.svelte'; import { player, setCrossfade } from '$lib/player/store.svelte'; import { @@ -24,6 +29,7 @@ const queryClient = useQueryClient(); const status = createLBStatusQuery() as CreateQueryResult; + const metrics = createRecommendationMetricsQuery() as CreateQueryResult; const tokenMutation = createTokenMutation(queryClient); const enabledMutation = createEnabledMutation(queryClient); @@ -267,6 +273,46 @@ {/if} + +
+

Recommendation metrics

+

+ How plays launched from each recommendation surface land, over the last + {$metrics.data?.window_days ?? 30} days. Lower skip rate and higher average + completion mean the surface is hitting. +

+ {#if $metrics.isPending} +

Loading…

+ {:else if $metrics.isError} +

Couldn't load metrics.

+ {:else if $metrics.data && $metrics.data.sources.length > 0} + + + + + + + + + + + {#each $metrics.data.sources as m (m.source)} + + + + + + + {/each} + +
SurfacePlaysSkip rateAvg completion
{sourceLabel(m.source)}{m.plays}{(m.skip_rate * 100).toFixed(0)}%{(m.avg_completion * 100).toFixed(0)}%
+ {:else} +

+ No recommendation plays yet. Play something from For You, Discover, or a mix. +

+ {/if} +
+

Profile

diff --git a/web/src/routes/settings/Appearance.test.ts b/web/src/routes/settings/Appearance.test.ts index 0de00816..d254e97b 100644 --- a/web/src/routes/settings/Appearance.test.ts +++ b/web/src/routes/settings/Appearance.test.ts @@ -25,6 +25,16 @@ vi.mock('$lib/api/listenbrainz', () => { }; }); +vi.mock('$lib/api/metrics', () => ({ + createRecommendationMetricsQuery: () => ({ + subscribe: (run: (v: unknown) => void) => { + run({ isPending: false, isError: false, data: { window_days: 30, sources: [] } }); + return () => {}; + } + }), + sourceLabel: (s: string) => s +})); + beforeEach(() => { globalThis.localStorage.clear(); document.documentElement.removeAttribute('data-theme'); diff --git a/web/src/routes/settings/settings.test.ts b/web/src/routes/settings/settings.test.ts index c3db35fd..ae913c07 100644 --- a/web/src/routes/settings/settings.test.ts +++ b/web/src/routes/settings/settings.test.ts @@ -20,6 +20,16 @@ vi.mock('$lib/api/me', () => ({ regenerateAPIToken: vi.fn() })); +vi.mock('$lib/api/metrics', () => ({ + createRecommendationMetricsQuery: () => ({ + subscribe: (run: (v: unknown) => void) => { + run({ isPending: false, isError: false, data: { window_days: 30, sources: [] } }); + return () => {}; + } + }), + sourceLabel: (s: string) => s +})); + import SettingsPage from './+page.svelte'; import { createLBStatusQuery,