60533073ad
The recommendation metrics table was observable but not actionable: raw source strings (album:<uuid> one-offs) drowned the stable surfaces, and manual plays were excluded so skip rates had no control group. - SQL: include NULL-source rows (the baseline) and carry completion_n so family merges can weight avg_completion correctly. - Handler buckets raw sources into stable families (radio:<uuid> → Radio, album:/artist: → direct plays, etc.) grouped by surface intent: go-to / discovery / direct — each band judged against its job, since discovery mixes are expected to skip hotter. Families under 20 plays are flagged low-confidence, not hidden. - Settings card renders the baseline row and per-surface deltas in percentage points vs baseline (worse-than-baseline deltas in danger color), intent hint copy per group, low-data rows dimmed. - Pure-unit test for the bucketing/merge; DB test updated to the new contract (baseline included, radio:<uuid> collapse). Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01TsF3cNoKrqCYsU78cXC8U6
177 lines
6.2 KiB
Go
177 lines
6.2 KiB
Go
package api
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"os"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/go-chi/chi/v5"
|
|
"github.com/jackc/pgx/v5/pgtype"
|
|
|
|
"git.fabledsword.com/bvandeusen/minstrel/internal/db/dbq"
|
|
)
|
|
|
|
func newMetricsRouter(h *handlers) chi.Router {
|
|
r := chi.NewRouter()
|
|
r.Get("/api/me/recommendation-metrics", h.handleGetRecommendationMetrics)
|
|
return r
|
|
}
|
|
|
|
// seedSourcedPlay inserts a play_event with an explicit source + completion +
|
|
// skip flag. A nil source inserts NULL (manual library play → baseline).
|
|
func seedSourcedPlay(
|
|
t *testing.T, h *handlers, userID, trackID, sessionID pgtype.UUID,
|
|
source *string, completion float64, skipped bool,
|
|
) {
|
|
t.Helper()
|
|
if _, err := h.pool.Exec(context.Background(),
|
|
`INSERT INTO play_events
|
|
(user_id, track_id, session_id, started_at, source, completion_ratio, was_skipped)
|
|
VALUES ($1, $2, $3, now(), $4, $5, $6)`,
|
|
userID, trackID, sessionID, source, completion, skipped); err != nil {
|
|
t.Fatalf("seed sourced play: %v", err)
|
|
}
|
|
}
|
|
|
|
func TestRecommendationMetrics_NoSession401(t *testing.T) {
|
|
h, _ := testHandlers(t)
|
|
req := httptest.NewRequest(http.MethodGet, "/api/me/recommendation-metrics", nil)
|
|
rec := httptest.NewRecorder()
|
|
newMetricsRouter(h).ServeHTTP(rec, req)
|
|
if rec.Code != http.StatusUnauthorized {
|
|
t.Errorf("status = %d, want 401", rec.Code)
|
|
}
|
|
}
|
|
|
|
// findSurface returns the named family from any group, or nil.
|
|
func findSurface(resp recommendationMetricsResp, key string) *surfaceMetric {
|
|
for _, g := range resp.Groups {
|
|
for i := range g.Surfaces {
|
|
if g.Surfaces[i].Key == key {
|
|
return &g.Surfaces[i]
|
|
}
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func TestBucketMetricsResponse_FamiliesGroupsBaseline(t *testing.T) {
|
|
src := func(s string) *string { return &s }
|
|
rows := []dbq.RecommendationSourceMetricsForUserRow{
|
|
{Source: src("for_you"), Plays: 3, Skips: 1, CompletionN: 3, AvgCompletion: 2.0 / 3},
|
|
// Two radio sessions collapse into one "radio" family; weighted
|
|
// completion = (0.2*1 + 0.8*1) / 2 = 0.5.
|
|
{Source: src("radio:aaaa"), Plays: 1, Skips: 1, CompletionN: 1, AvgCompletion: 0.2},
|
|
{Source: src("radio:bbbb"), Plays: 1, Skips: 0, CompletionN: 1, AvgCompletion: 0.8},
|
|
{Source: src("album:cccc"), Plays: 1, Skips: 0, CompletionN: 0, AvgCompletion: 0},
|
|
{Source: src("discover"), Plays: 1, Skips: 0, CompletionN: 1, AvgCompletion: 0.8},
|
|
// NULL source = manual plays → baseline, not a group row.
|
|
{Source: nil, Plays: 25, Skips: 5, CompletionN: 20, AvgCompletion: 0.9},
|
|
}
|
|
resp := bucketMetricsResponse(recMetricsDefaultDays, rows)
|
|
|
|
if resp.Baseline == nil {
|
|
t.Fatal("baseline missing")
|
|
}
|
|
if resp.Baseline.Plays != 25 || resp.Baseline.SkipRate != 0.2 {
|
|
t.Errorf("baseline = %+v, want plays=25 skip_rate=0.2", resp.Baseline)
|
|
}
|
|
if resp.Baseline.LowConfidence {
|
|
t.Error("baseline with 25 plays should not be low-confidence")
|
|
}
|
|
|
|
radio := findSurface(resp, "radio")
|
|
if radio == nil {
|
|
t.Fatal("radio family missing")
|
|
}
|
|
if radio.Plays != 2 || radio.Skips != 1 {
|
|
t.Errorf("radio plays/skips = %d/%d, want 2/1", radio.Plays, radio.Skips)
|
|
}
|
|
if radio.AvgCompletion < 0.49 || radio.AvgCompletion > 0.51 {
|
|
t.Errorf("radio avg_completion = %.3f, want 0.5 (play-weighted merge)", radio.AvgCompletion)
|
|
}
|
|
if !radio.LowConfidence {
|
|
t.Error("radio with 2 plays should be low-confidence")
|
|
}
|
|
|
|
if s := findSurface(resp, "direct_album"); s == nil || s.Plays != 1 {
|
|
t.Errorf("direct_album = %+v, want plays=1", s)
|
|
}
|
|
if s := findSurface(resp, ""); s != nil {
|
|
t.Error("NULL source must not appear as a surface family")
|
|
}
|
|
|
|
// Group ordering is intent-banded: go_to before discovery before direct.
|
|
wantOrder := []string{intentGoTo, intentDiscovery, intentDirect}
|
|
if len(resp.Groups) != len(wantOrder) {
|
|
t.Fatalf("groups = %d, want %d", len(resp.Groups), len(wantOrder))
|
|
}
|
|
for i, g := range resp.Groups {
|
|
if g.Intent != wantOrder[i] {
|
|
t.Errorf("group[%d].intent = %s, want %s", i, g.Intent, wantOrder[i])
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestRecommendationMetrics_BucketsWithBaseline(t *testing.T) {
|
|
if os.Getenv("MINSTREL_TEST_DATABASE_URL") == "" {
|
|
t.Skip("MINSTREL_TEST_DATABASE_URL not set")
|
|
}
|
|
h, pool := testHandlers(t)
|
|
user := seedUser(t, pool, "metrics", "pw", false)
|
|
artist := seedArtist(t, pool, "MetricArtist")
|
|
album := seedAlbum(t, pool, artist.ID, "MetricAlbum", 2020)
|
|
tk := seedTrack(t, pool, album.ID, artist.ID, "MetricTrack", 1, 200000)
|
|
session := seedPlaySession(t, pool, user.ID, time.Now())
|
|
|
|
forYou := "for_you"
|
|
radioA := "radio:11111111-1111-1111-1111-111111111111"
|
|
// for_you: 3 plays, 1 skipped; completions 1.0, 0.95, 0.05 → mean 0.6667.
|
|
seedSourcedPlay(t, h, user.ID, tk.ID, session, &forYou, 1.0, false)
|
|
seedSourcedPlay(t, h, user.ID, tk.ID, session, &forYou, 0.95, false)
|
|
seedSourcedPlay(t, h, user.ID, tk.ID, session, &forYou, 0.05, true)
|
|
// A radio session play collapses into the "radio" family.
|
|
seedSourcedPlay(t, h, user.ID, tk.ID, session, &radioA, 0.8, false)
|
|
// Manual play (NULL source) — the baseline row.
|
|
seedSourcedPlay(t, h, user.ID, tk.ID, session, nil, 1.0, false)
|
|
|
|
req := httptest.NewRequest(http.MethodGet, "/api/me/recommendation-metrics", nil)
|
|
req = withUser(req, user)
|
|
rec := httptest.NewRecorder()
|
|
newMetricsRouter(h).ServeHTTP(rec, req)
|
|
if rec.Code != http.StatusOK {
|
|
t.Fatalf("status = %d, want 200", rec.Code)
|
|
}
|
|
|
|
var resp recommendationMetricsResp
|
|
if err := json.Unmarshal(rec.Body.Bytes(), &resp); err != nil {
|
|
t.Fatalf("decode: %v", err)
|
|
}
|
|
if resp.WindowDays != recMetricsDefaultDays {
|
|
t.Errorf("window_days = %d, want %d", resp.WindowDays, recMetricsDefaultDays)
|
|
}
|
|
if resp.Baseline == nil || resp.Baseline.Plays != 1 {
|
|
t.Fatalf("baseline = %+v, want plays=1", resp.Baseline)
|
|
}
|
|
fy := findSurface(resp, "for_you")
|
|
if fy == nil {
|
|
t.Fatal("for_you metrics missing")
|
|
}
|
|
if fy.Plays != 3 || fy.Skips != 1 {
|
|
t.Errorf("for_you plays/skips = %d/%d, want 3/1", fy.Plays, fy.Skips)
|
|
}
|
|
if fy.SkipRate < 0.32 || fy.SkipRate > 0.34 {
|
|
t.Errorf("for_you skip_rate = %.3f, want ~0.333", fy.SkipRate)
|
|
}
|
|
if fy.AvgCompletion < 0.66 || fy.AvgCompletion > 0.67 {
|
|
t.Errorf("for_you avg_completion = %.4f, want ~0.6667", fy.AvgCompletion)
|
|
}
|
|
if radio := findSurface(resp, "radio"); radio == nil || radio.Plays != 1 {
|
|
t.Errorf("radio family = %+v, want plays=1 (collapsed from radio:<uuid>)", radio)
|
|
}
|
|
}
|