feat(tuning): weekly trend view — per-surface series + knob-turn markers
test-go / test (push) Successful in 33s
test-web / test (push) Failing after 38s
test-go / integration (push) Successful in 4m44s

The verify half of the tune→verify loop (#1251), on the same admin
Tuning page as the knobs:

- RecommendationWeeklyTrends: weekly per-source outcomes aggregated
  across all users (the knobs are global, so judging a turn needs
  global outcomes — rows carry rates only, no track/user identity),
  with a taste-hit count per bucket: plays whose track's artist has a
  positive weight in the player's current taste profile. That's the
  "cheap recompute" reading — retroactive over the whole window, at
  the cost of profile drift.
- GET /api/admin/recommendation-trends?weeks=N (default 12, cap 52):
  per-family weekly series (skip rate, sample-weighted completion,
  taste-hit rate) plus the tuning-audit markers inside the window.
- Web: sparkline table under the tuning cards — skip rate per week on
  a shared axis with dashed ticks at knob turns, latest-week columns,
  window taste-hit rate, low-volume rows dimmed as anecdote, and a
  plain-text list of the window's tuning changes.

Also fixes the revive unused-parameter lint on the tuning GET handler
that failed CI run 1903 on the previous commit.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01TsF3cNoKrqCYsU78cXC8U6
This commit is contained in:
2026-07-03 09:29:33 -04:00
parent 0d0a8f46b1
commit 9ad4343c76
9 changed files with 773 additions and 15 deletions
+202
View File
@@ -0,0 +1,202 @@
// Admin recommendation-trends endpoint (#1251): weekly per-surface
// outcome series with tuning-audit markers — the verify half of the
// tune→verify loop the tuning lab (#1250) opens. Aggregated across
// all users because the knobs are global; rows carry rates only.
package api
import (
"encoding/json"
"net/http"
"sort"
"strconv"
"time"
"git.fabledsword.com/bvandeusen/minstrel/internal/apierror"
"git.fabledsword.com/bvandeusen/minstrel/internal/db/dbq"
)
const (
trendsDefaultWeeks = 12
trendsMaxWeeks = 52
// trendsAuditFetchCap bounds the audit fetch; markers older than
// the window are dropped in Go. Far above any real knob-turn count
// inside a year.
trendsAuditFetchCap = 500
)
// trendPoint is one week of one surface family's outcomes.
type trendPoint struct {
WeekStart string `json:"week_start"` // ISO date (Monday)
Plays int64 `json:"plays"`
Skips int64 `json:"skips"`
SkipRate float64 `json:"skip_rate"`
AvgCompletion float64 `json:"avg_completion"`
// TasteHitRate is the share of plays whose track's artist carries a
// positive weight in the player's current taste profile — a drifted
// but retroactive read on whether the surface feeds taste-fitting
// tracks.
TasteHitRate float64 `json:"taste_hit_rate"`
// completionN carries the completion sample count through same-week
// merges so avg_completion stays sample-weighted; not serialized.
completionN int64
}
// trendSeries is one surface family's weekly series.
type trendSeries struct {
Key string `json:"key"`
Label string `json:"label"`
Intent string `json:"intent"` // go_to | discovery | direct; "" for the manual baseline
Plays int64 `json:"plays"` // window total, for sorting/volume-gating
Points []trendPoint `json:"points"`
}
// trendMarker is one tuning-audit event annotated on the timeline.
type trendMarker struct {
ChangedAt string `json:"changed_at"`
Scope string `json:"scope"`
Action string `json:"action"`
Changes json.RawMessage `json:"changes"`
}
type trendsResp struct {
Weeks int `json:"weeks"`
Series []trendSeries `json:"series"`
Markers []trendMarker `json:"markers"`
}
// handleGetRecommendationTrends implements
// GET /api/admin/recommendation-trends?weeks=N (default 12, cap 52).
func (h *handlers) handleGetRecommendationTrends(w http.ResponseWriter, r *http.Request) {
weeks := trendsDefaultWeeks
if v := r.URL.Query().Get("weeks"); v != "" {
n, err := strconv.Atoi(v)
if err != nil || n < 1 {
writeErr(w, apierror.BadRequest("bad_request", "invalid weeks"))
return
}
if n > trendsMaxWeeks {
n = trendsMaxWeeks
}
weeks = n
}
q := dbq.New(h.pool)
rows, err := q.RecommendationWeeklyTrends(r.Context(), int32(weeks))
if err != nil {
h.logger.Error("api: recommendation trends", "err", err)
writeErr(w, apierror.InternalMsg("lookup failed", err))
return
}
audits, err := q.ListTuningAudit(r.Context(), trendsAuditFetchCap)
if err != nil {
h.logger.Error("api: recommendation trends audit", "err", err)
writeErr(w, apierror.InternalMsg("lookup failed", err))
return
}
writeJSON(w, http.StatusOK, buildTrendsResponse(weeks, time.Now().UTC(), rows, audits))
}
// buildTrendsResponse folds weekly rows into per-family series and
// windows the audit markers. Split from the handler for pure-unit
// testability.
func buildTrendsResponse(
weeks int, now time.Time,
rows []dbq.RecommendationWeeklyTrendsRow,
audits []dbq.RecommendationTuningAudit,
) trendsResp {
type accum struct {
fam recFamily
plays int64
points []trendPoint
}
families := map[string]*accum{}
for _, row := range rows {
fam := recFamily{key: "manual", label: "Manual library plays"}
if row.Source != nil && *row.Source != "" {
fam = bucketRecSource(*row.Source)
}
acc, ok := families[fam.key]
if !ok {
acc = &accum{fam: fam}
families[fam.key] = acc
}
acc.plays += row.Plays
p := trendPoint{
WeekStart: row.WeekStart.Time.Format("2006-01-02"),
Plays: row.Plays,
Skips: row.Skips,
AvgCompletion: row.AvgCompletion,
completionN: row.CompletionN,
}
if row.Plays > 0 {
p.SkipRate = float64(row.Skips) / float64(row.Plays)
p.TasteHitRate = float64(row.TasteHits) / float64(row.Plays)
}
// Same family can arrive as several raw sources (radio:<uuid>);
// merge same-week points play-weighted.
if n := len(acc.points); n > 0 && acc.points[n-1].WeekStart == p.WeekStart {
acc.points[n-1] = mergeTrendPoints(acc.points[n-1], p)
} else {
acc.points = append(acc.points, p)
}
}
resp := trendsResp{Weeks: weeks, Series: []trendSeries{}, Markers: []trendMarker{}}
for _, acc := range families {
resp.Series = append(resp.Series, trendSeries{
Key: acc.fam.key,
Label: acc.fam.label,
Intent: acc.fam.intent,
Plays: acc.plays,
Points: acc.points,
})
}
sort.Slice(resp.Series, func(i, j int) bool {
if resp.Series[i].Plays != resp.Series[j].Plays {
return resp.Series[i].Plays > resp.Series[j].Plays
}
return resp.Series[i].Key < resp.Series[j].Key
})
cutoff := now.Add(-time.Duration(weeks) * 7 * 24 * time.Hour)
for _, a := range audits {
if a.ChangedAt.Time.Before(cutoff) {
continue
}
resp.Markers = append(resp.Markers, trendMarker{
ChangedAt: a.ChangedAt.Time.UTC().Format(time.RFC3339),
Scope: a.Scope,
Action: a.Action,
Changes: json.RawMessage(a.Changes),
})
}
// ListTuningAudit returns newest-first; the timeline reads better
// oldest-first.
sort.Slice(resp.Markers, func(i, j int) bool {
return resp.Markers[i].ChangedAt < resp.Markers[j].ChangedAt
})
return resp
}
// mergeTrendPoints combines two same-week points of one family:
// counts add, skip/taste rates re-derive from the merged counts, and
// avg_completion is weighted by each side's completion sample count.
func mergeTrendPoints(a, b trendPoint) trendPoint {
out := trendPoint{
WeekStart: a.WeekStart,
Plays: a.Plays + b.Plays,
Skips: a.Skips + b.Skips,
completionN: a.completionN + b.completionN,
}
if out.Plays > 0 {
out.SkipRate = float64(out.Skips) / float64(out.Plays)
out.TasteHitRate = (a.TasteHitRate*float64(a.Plays) + b.TasteHitRate*float64(b.Plays)) /
float64(out.Plays)
}
if out.completionN > 0 {
out.AvgCompletion = (a.AvgCompletion*float64(a.completionN) + b.AvgCompletion*float64(b.completionN)) /
float64(out.completionN)
}
return out
}
@@ -0,0 +1,152 @@
package api
import (
"context"
"encoding/json"
"net/http"
"net/http/httptest"
"testing"
"time"
"github.com/go-chi/chi/v5"
"github.com/jackc/pgx/v5/pgtype"
"git.fabledsword.com/bvandeusen/minstrel/internal/db/dbq"
)
func date(s string) pgtype.Date {
t, _ := time.Parse("2006-01-02", s)
return pgtype.Date{Time: t, Valid: true}
}
func TestBuildTrendsResponse_SeriesMergingAndMarkers(t *testing.T) {
src := func(s string) *string { return &s }
now := time.Date(2026, 7, 3, 12, 0, 0, 0, time.UTC)
rows := []dbq.RecommendationWeeklyTrendsRow{
// Two radio session sources in the same week collapse into one
// family point; completion weighting by sample count.
{WeekStart: date("2026-06-22"), Source: src("radio:aaaa"),
Plays: 3, Skips: 1, CompletionN: 3, AvgCompletion: 0.6, TasteHits: 3},
{WeekStart: date("2026-06-22"), Source: src("radio:bbbb"),
Plays: 1, Skips: 1, CompletionN: 1, AvgCompletion: 0.2, TasteHits: 0},
{WeekStart: date("2026-06-29"), Source: src("radio:aaaa"),
Plays: 2, Skips: 0, CompletionN: 2, AvgCompletion: 0.9, TasteHits: 1},
// NULL source = manual baseline family.
{WeekStart: date("2026-06-29"), Source: nil,
Plays: 5, Skips: 1, CompletionN: 5, AvgCompletion: 0.8, TasteHits: 4},
}
audits := []dbq.RecommendationTuningAudit{
{ID: 2, ChangedAt: pgtype.Timestamptz{Time: now.Add(-24 * time.Hour), Valid: true},
Scope: "radio", Action: "update", Changes: []byte(`[{"field":"taste_weight","old":1,"new":2}]`)},
// Older than the window → dropped.
{ID: 1, ChangedAt: pgtype.Timestamptz{Time: now.Add(-100 * 7 * 24 * time.Hour), Valid: true},
Scope: "taste", Action: "reset", Changes: []byte(`[]`)},
}
resp := buildTrendsResponse(12, now, rows, audits)
if len(resp.Series) != 2 {
t.Fatalf("series = %d, want 2 (radio + manual)", len(resp.Series))
}
radio := resp.Series[0] // 6 plays > manual's 5 → sorted first
if radio.Key != "radio" || radio.Plays != 6 {
t.Fatalf("series[0] = %s/%d, want radio/6", radio.Key, radio.Plays)
}
if len(radio.Points) != 2 {
t.Fatalf("radio points = %d, want 2 weeks", len(radio.Points))
}
wk1 := radio.Points[0]
if wk1.WeekStart != "2026-06-22" || wk1.Plays != 4 || wk1.Skips != 2 {
t.Errorf("week1 = %+v, want 2026-06-22 with 4 plays / 2 skips", wk1)
}
if wk1.SkipRate != 0.5 {
t.Errorf("week1 skip_rate = %v, want 0.5", wk1.SkipRate)
}
// Completion weighted by sample count: (0.6*3 + 0.2*1) / 4 = 0.5.
if wk1.AvgCompletion < 0.49 || wk1.AvgCompletion > 0.51 {
t.Errorf("week1 avg_completion = %v, want 0.5", wk1.AvgCompletion)
}
// Taste hits: 3 of 4 plays.
if wk1.TasteHitRate != 0.75 {
t.Errorf("week1 taste_hit_rate = %v, want 0.75", wk1.TasteHitRate)
}
if manual := resp.Series[1]; manual.Key != "manual" || manual.Intent != "" {
t.Errorf("series[1] = %+v, want the manual baseline family", manual)
}
if len(resp.Markers) != 1 {
t.Fatalf("markers = %d, want 1 (out-of-window marker dropped)", len(resp.Markers))
}
if resp.Markers[0].Scope != "radio" || resp.Markers[0].Action != "update" {
t.Errorf("marker = %+v, want radio/update", resp.Markers[0])
}
}
func newTrendsRouter(h *handlers) chi.Router {
r := chi.NewRouter()
r.Get("/api/admin/recommendation-trends", h.handleGetRecommendationTrends)
return r
}
func TestRecommendationTrends_EndToEnd(t *testing.T) {
h, pool := testHandlers(t)
user := seedUser(t, pool, "trends", "pw", true)
artist := seedArtist(t, pool, "TrendArtist")
album := seedAlbum(t, pool, artist.ID, "TrendAlbum", 2020)
tk := seedTrack(t, pool, album.ID, artist.ID, "TrendTrack", 1, 200000)
session := seedPlaySession(t, pool, user.ID, time.Now())
// Positive taste weight for the artist → plays count as taste hits.
if _, err := pool.Exec(context.Background(),
`INSERT INTO taste_profile_artists (user_id, artist_id, weight) VALUES ($1, $2, 1.5)`,
user.ID, artist.ID); err != nil {
t.Fatalf("seed taste profile: %v", err)
}
radio := "radio"
seedSourcedPlay(t, h, user.ID, tk.ID, session, &radio, nil, 0.9, false)
// A knob turn to mark the timeline.
if err := h.recSettings.UpdateProfile(context.Background(), "radio",
map[string]float64{"taste_weight": 2}); err != nil {
t.Fatalf("UpdateProfile: %v", err)
}
req := httptest.NewRequest(http.MethodGet, "/api/admin/recommendation-trends", nil)
rec := httptest.NewRecorder()
newTrendsRouter(h).ServeHTTP(rec, req)
if rec.Code != http.StatusOK {
t.Fatalf("status = %d, want 200 (%s)", rec.Code, rec.Body.String())
}
var resp trendsResp
if err := json.Unmarshal(rec.Body.Bytes(), &resp); err != nil {
t.Fatalf("decode: %v", err)
}
if resp.Weeks != trendsDefaultWeeks {
t.Errorf("weeks = %d, want %d", resp.Weeks, trendsDefaultWeeks)
}
var radioSeries *trendSeries
for i := range resp.Series {
if resp.Series[i].Key == "radio" {
radioSeries = &resp.Series[i]
}
}
if radioSeries == nil || len(radioSeries.Points) != 1 {
t.Fatalf("radio series = %+v, want one point", radioSeries)
}
if radioSeries.Points[0].TasteHitRate != 1.0 {
t.Errorf("taste_hit_rate = %v, want 1.0 (positive-weight artist)",
radioSeries.Points[0].TasteHitRate)
}
if len(resp.Markers) != 1 || resp.Markers[0].Scope != "radio" {
t.Errorf("markers = %+v, want the radio knob turn", resp.Markers)
}
}
func TestRecommendationTrends_InvalidWeeks(t *testing.T) {
h, _ := testHandlers(t)
req := httptest.NewRequest(http.MethodGet, "/api/admin/recommendation-trends?weeks=zero", nil)
rec := httptest.NewRecorder()
newTrendsRouter(h).ServeHTTP(rec, req)
if rec.Code != http.StatusBadRequest {
t.Errorf("status = %d, want 400", rec.Code)
}
}
+1 -1
View File
@@ -87,7 +87,7 @@ func (h *handlers) tuningSnapshot() tuningSnapshot {
}
// handleGetRecommendationTuning implements GET /api/admin/recommendation-tuning.
func (h *handlers) handleGetRecommendationTuning(w http.ResponseWriter, r *http.Request) {
func (h *handlers) handleGetRecommendationTuning(w http.ResponseWriter, _ *http.Request) {
writeJSON(w, http.StatusOK, h.tuningSnapshot())
}
+2
View File
@@ -209,6 +209,8 @@ func Mount(r chi.Router, pool *pgxpool.Pool, logger *slog.Logger, events *playev
admin.Get("/recommendation-tuning", h.handleGetRecommendationTuning)
admin.Patch("/recommendation-tuning/{scope}", h.handlePatchRecommendationTuning)
admin.Post("/recommendation-tuning/{scope}/reset", h.handleResetRecommendationTuning)
// Weekly outcome trends + knob-turn markers (#1251).
admin.Get("/recommendation-trends", h.handleGetRecommendationTrends)
})
authed.Get("/playlists", h.handleListPlaylists)
+80 -10
View File
@@ -12,7 +12,6 @@ import (
)
const recommendationSourceMetricsForUser = `-- name: RecommendationSourceMetricsForUser :many
SELECT
pe.source,
pe.pick_kind,
@@ -41,15 +40,6 @@ type RecommendationSourceMetricsForUserRow struct {
AvgCompletion float64
}
// Recommendation observability (#796 phase 4). Per-source play outcomes so the
// operator can see whether each recommendation surface is landing and tune the
// taste weights. Source is stamped on play_events when a play is launched from
// a recommendation surface; NULL means the user picked the track manually —
// those rows are INCLUDED here as the baseline control group the surfaces are
// judged against (milestone #127: delta-vs-baseline is what makes the numbers
// actionable). Raw source strings are bucketed into stable surface families in
// the Go handler; completion_n is carried so family merges can weight
// avg_completion correctly.
// $1 user_id, $2 window_days. plays/skips are counts; avg_completion is the
// mean completion ratio over the completion_n plays that recorded one.
// pick_kind splits For You plays into taste/fresh/unattributed (#1249);
@@ -80,3 +70,83 @@ func (q *Queries) RecommendationSourceMetricsForUser(ctx context.Context, arg Re
}
return items, nil
}
const recommendationWeeklyTrends = `-- name: RecommendationWeeklyTrends :many
SELECT
date_trunc('week', pe.started_at)::date AS week_start,
pe.source,
count(*)::bigint AS plays,
count(*) FILTER (WHERE pe.was_skipped)::bigint AS skips,
count(pe.completion_ratio)::bigint AS completion_n,
COALESCE(avg(pe.completion_ratio), 0)::float8 AS avg_completion,
count(*) FILTER (WHERE tpa.artist_id IS NOT NULL)::bigint AS taste_hits
FROM play_events pe
JOIN tracks t ON t.id = pe.track_id
LEFT JOIN taste_profile_artists tpa
ON tpa.user_id = pe.user_id
AND tpa.artist_id = t.artist_id
AND tpa.weight > 0
WHERE pe.started_at > now() - ($1::int * INTERVAL '1 week')
GROUP BY 1, 2
ORDER BY 1, 2
`
type RecommendationWeeklyTrendsRow struct {
WeekStart pgtype.Date
Source *string
Plays int64
Skips int64
CompletionN int64
AvgCompletion float64
TasteHits int64
}
// Recommendation observability (#796 phase 4). Per-source play outcomes so the
// operator can see whether each recommendation surface is landing and tune the
// taste weights. Source is stamped on play_events when a play is launched from
// a recommendation surface; NULL means the user picked the track manually —
// those rows are INCLUDED here as the baseline control group the surfaces are
// judged against (milestone #127: delta-vs-baseline is what makes the numbers
// actionable). Raw source strings are bucketed into stable surface families in
// the Go handler; completion_n is carried so family merges can weight
// avg_completion correctly.
// Weekly per-source outcome series for the tuning lab's trend view
// (#1251). Aggregated across ALL users: the tuning knobs are global,
// so judging a knob turn needs global outcomes — rows carry rates
// only, no track or user identity. NULL-source (manual) rows are
// included as the baseline family.
//
// taste_hits counts plays whose track's artist has a positive weight
// in that user's CURRENT taste profile — the "cheap recompute" option:
// retroactive over the whole window, at the cost of drift (the profile
// is today's, the play may be weeks old). Good enough to read whether
// a surface is feeding taste-fitting tracks.
// $1 window in weeks.
func (q *Queries) RecommendationWeeklyTrends(ctx context.Context, weeks int32) ([]RecommendationWeeklyTrendsRow, error) {
rows, err := q.db.Query(ctx, recommendationWeeklyTrends, weeks)
if err != nil {
return nil, err
}
defer rows.Close()
var items []RecommendationWeeklyTrendsRow
for rows.Next() {
var i RecommendationWeeklyTrendsRow
if err := rows.Scan(
&i.WeekStart,
&i.Source,
&i.Plays,
&i.Skips,
&i.CompletionN,
&i.AvgCompletion,
&i.TasteHits,
); err != nil {
return nil, err
}
items = append(items, i)
}
if err := rows.Err(); err != nil {
return nil, err
}
return items, nil
}
@@ -8,6 +8,37 @@
-- the Go handler; completion_n is carried so family merges can weight
-- avg_completion correctly.
-- name: RecommendationWeeklyTrends :many
-- Weekly per-source outcome series for the tuning lab's trend view
-- (#1251). Aggregated across ALL users: the tuning knobs are global,
-- so judging a knob turn needs global outcomes — rows carry rates
-- only, no track or user identity. NULL-source (manual) rows are
-- included as the baseline family.
--
-- taste_hits counts plays whose track's artist has a positive weight
-- in that user's CURRENT taste profile — the "cheap recompute" option:
-- retroactive over the whole window, at the cost of drift (the profile
-- is today's, the play may be weeks old). Good enough to read whether
-- a surface is feeding taste-fitting tracks.
-- $1 window in weeks.
SELECT
date_trunc('week', pe.started_at)::date AS week_start,
pe.source,
count(*)::bigint AS plays,
count(*) FILTER (WHERE pe.was_skipped)::bigint AS skips,
count(pe.completion_ratio)::bigint AS completion_n,
COALESCE(avg(pe.completion_ratio), 0)::float8 AS avg_completion,
count(*) FILTER (WHERE tpa.artist_id IS NOT NULL)::bigint AS taste_hits
FROM play_events pe
JOIN tracks t ON t.id = pe.track_id
LEFT JOIN taste_profile_artists tpa
ON tpa.user_id = pe.user_id
AND tpa.artist_id = t.artist_id
AND tpa.weight > 0
WHERE pe.started_at > now() - (sqlc.arg(weeks)::int * INTERVAL '1 week')
GROUP BY 1, 2
ORDER BY 1, 2;
-- name: RecommendationSourceMetricsForUser :many
-- $1 user_id, $2 window_days. plays/skips are counts; avg_completion is the
-- mean completion ratio over the completion_n plays that recorded one.