9ad4343c76
The verify half of the tune→verify loop (#1251), on the same admin Tuning page as the knobs: - RecommendationWeeklyTrends: weekly per-source outcomes aggregated across all users (the knobs are global, so judging a turn needs global outcomes — rows carry rates only, no track/user identity), with a taste-hit count per bucket: plays whose track's artist has a positive weight in the player's current taste profile. That's the "cheap recompute" reading — retroactive over the whole window, at the cost of profile drift. - GET /api/admin/recommendation-trends?weeks=N (default 12, cap 52): per-family weekly series (skip rate, sample-weighted completion, taste-hit rate) plus the tuning-audit markers inside the window. - Web: sparkline table under the tuning cards — skip rate per week on a shared axis with dashed ticks at knob turns, latest-week columns, window taste-hit rate, low-volume rows dimmed as anecdote, and a plain-text list of the window's tuning changes. Also fixes the revive unused-parameter lint on the tuning GET handler that failed CI run 1903 on the previous commit. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01TsF3cNoKrqCYsU78cXC8U6
435 lines
17 KiB
Svelte
435 lines
17 KiB
Svelte
<script lang="ts">
|
||
import { pageTitle } from '$lib/branding';
|
||
import {
|
||
getTuning,
|
||
patchTuning,
|
||
resetTuning,
|
||
getTrends,
|
||
type TuningScope,
|
||
type TuningSnapshot,
|
||
type WeightProfile,
|
||
type TasteTuning,
|
||
type TrendsResponse,
|
||
type TrendSeries,
|
||
type TrendMarker
|
||
} from '$lib/api/tuning';
|
||
import { errMessage } from '$lib/api/errors';
|
||
import { pushToast } from '$lib/stores/toast.svelte';
|
||
|
||
// The defaults-discovery lab (#1250): DB-backed scoring-weight
|
||
// profiles + taste-build knobs with live effect. This card exists to
|
||
// FIND good values — found-good values get baked into shipped
|
||
// defaults, so end users and other operators never need it.
|
||
|
||
const weightFields: { key: keyof WeightProfile; label: string; hint: string }[] = [
|
||
{ key: 'base_weight', label: 'Base weight', hint: 'Floor score every candidate starts from.' },
|
||
{ key: 'like_boost', label: 'Like boost', hint: 'Added when the track is liked.' },
|
||
{ key: 'recency_weight', label: 'Recency weight', hint: 'Rewards tracks not played recently (0–30d ramp).' },
|
||
{ key: 'skip_penalty', label: 'Skip penalty', hint: 'Subtracts skips/plays ratio.' },
|
||
{ key: 'jitter_magnitude', label: 'Jitter', hint: 'Random reshuffle magnitude for near-ties.' },
|
||
{ key: 'context_weight', label: 'Context weight', hint: 'Session-vector similarity contribution.' },
|
||
{ key: 'similarity_weight', label: 'Similarity weight', hint: 'Seed-similarity contribution.' },
|
||
{ key: 'taste_weight', label: 'Taste weight', hint: 'Learned taste-profile fit, in [-1, +1].' }
|
||
];
|
||
|
||
const tasteFields: { key: keyof TasteTuning; label: string; hint: string }[] = [
|
||
{ key: 'half_life_days', label: 'Half-life (days)', hint: "A play's influence halves every this-many days." },
|
||
{ key: 'engagement_hard_skip', label: 'Hard-skip point', hint: 'Completion at/below which a play reads −1.' },
|
||
{ key: 'engagement_neutral', label: 'Neutral point', hint: 'Completion at which a play reads 0.' },
|
||
{ key: 'engagement_full', label: 'Full point', hint: 'Completion at/above which a play reads +1.' }
|
||
];
|
||
|
||
const profileScopes: { scope: 'radio' | 'daily_mix'; label: string; blurb: string }[] = [
|
||
{ scope: 'radio', label: 'Radio', blurb: 'Seed-directed listening — the user picked a direction.' },
|
||
{ scope: 'daily_mix', label: 'Daily mixes', blurb: 'For You, Songs like…, and the discovery mixes.' }
|
||
];
|
||
|
||
let snapshot = $state<TuningSnapshot | null>(null);
|
||
let loadFailed = $state(false);
|
||
// Editable copies, string-typed for the inputs; parsed on save.
|
||
let form = $state<Record<string, Record<string, string>>>({});
|
||
let saving = $state<TuningScope | null>(null);
|
||
|
||
function fillForm(snap: TuningSnapshot) {
|
||
const f: Record<string, Record<string, string>> = { radio: {}, daily_mix: {}, taste: {} };
|
||
for (const p of ['radio', 'daily_mix'] as const) {
|
||
for (const { key } of weightFields) f[p][key] = String(snap.profiles[p][key]);
|
||
}
|
||
for (const { key } of tasteFields) f.taste[key] = String(snap.taste[key]);
|
||
form = f;
|
||
}
|
||
|
||
$effect(() => {
|
||
getTuning()
|
||
.then((snap) => {
|
||
fillForm(snap);
|
||
snapshot = snap;
|
||
})
|
||
.catch(() => {
|
||
loadFailed = true;
|
||
});
|
||
});
|
||
|
||
// A knob deviates when its CURRENT SAVED value differs from shipped;
|
||
// the dot marks where this install has drifted from defaults.
|
||
function deviates(scope: 'radio' | 'daily_mix' | 'taste', key: string): boolean {
|
||
if (!snapshot) return false;
|
||
if (scope === 'taste') {
|
||
return snapshot.taste[key as keyof TasteTuning] !== snapshot.shipped.taste[key as keyof TasteTuning];
|
||
}
|
||
return (
|
||
snapshot.profiles[scope][key as keyof WeightProfile] !==
|
||
snapshot.shipped.profiles[scope][key as keyof WeightProfile]
|
||
);
|
||
}
|
||
|
||
function currentValue(scope: TuningScope, key: string): number {
|
||
if (!snapshot) return 0;
|
||
if (scope === 'taste') return snapshot.taste[key as keyof TasteTuning];
|
||
return snapshot.profiles[scope][key as keyof WeightProfile];
|
||
}
|
||
|
||
async function save(scope: TuningScope) {
|
||
if (!snapshot) return;
|
||
const values: Record<string, number> = {};
|
||
for (const [key, raw] of Object.entries(form[scope] ?? {})) {
|
||
const v = Number(raw);
|
||
if (!Number.isFinite(v)) {
|
||
pushToast(`${key} is not a number.`, 'error');
|
||
return;
|
||
}
|
||
if (v !== currentValue(scope, key)) values[key] = v;
|
||
}
|
||
if (Object.keys(values).length === 0) {
|
||
pushToast('Nothing changed.');
|
||
return;
|
||
}
|
||
saving = scope;
|
||
try {
|
||
snapshot = await patchTuning(scope, values);
|
||
fillForm(snapshot);
|
||
pushToast('Saved — takes effect on the next scoring pass.');
|
||
} catch (e: unknown) {
|
||
pushToast(`Save failed: ${errMessage(e)}`, 'error');
|
||
} finally {
|
||
saving = null;
|
||
}
|
||
}
|
||
|
||
async function reset(scope: TuningScope) {
|
||
saving = scope;
|
||
try {
|
||
snapshot = await resetTuning(scope);
|
||
fillForm(snapshot);
|
||
pushToast('Reset to shipped defaults.');
|
||
} catch (e: unknown) {
|
||
pushToast(`Reset failed: ${errMessage(e)}`, 'error');
|
||
} finally {
|
||
saving = null;
|
||
}
|
||
}
|
||
|
||
// Trends (#1251): weekly skip-rate sparklines per surface with
|
||
// knob-turn markers — the verify half of the tune→verify loop.
|
||
|
||
let trends = $state<TrendsResponse | null>(null);
|
||
let trendsFailed = $state(false);
|
||
|
||
$effect(() => {
|
||
getTrends()
|
||
.then((t) => {
|
||
trends = t;
|
||
})
|
||
.catch(() => {
|
||
trendsFailed = true;
|
||
});
|
||
});
|
||
|
||
const SPARK_W = 220;
|
||
const SPARK_H = 36;
|
||
const TREND_LOW_VOLUME = 20;
|
||
|
||
// The week axis is the sorted union of every series' buckets, so all
|
||
// sparklines and markers share one x scale.
|
||
const weekAxis = $derived.by(() => {
|
||
if (!trends) return [] as string[];
|
||
const set = new Set<string>();
|
||
for (const s of trends.series) for (const p of s.points) set.add(p.week_start);
|
||
return [...set].sort();
|
||
});
|
||
|
||
function xFor(week: string): number {
|
||
const i = weekAxis.indexOf(week);
|
||
if (i < 0 || weekAxis.length < 2) return 0;
|
||
return (i / (weekAxis.length - 1)) * SPARK_W;
|
||
}
|
||
|
||
// Polyline of the series' weekly skip rate on a fixed [0,1] y-scale
|
||
// (higher = worse, drawn upward) so rows are visually comparable.
|
||
function sparkPoints(s: TrendSeries): string {
|
||
return s.points
|
||
.map((p) => `${xFor(p.week_start).toFixed(1)},${(SPARK_H - p.skip_rate * SPARK_H).toFixed(1)}`)
|
||
.join(' ');
|
||
}
|
||
|
||
// A marker lands on the latest axis week that starts at/before it.
|
||
function markerX(m: TrendMarker): number {
|
||
const day = m.changed_at.slice(0, 10);
|
||
let idx = -1;
|
||
for (let i = 0; i < weekAxis.length; i++) {
|
||
if (weekAxis[i] <= day) idx = i;
|
||
}
|
||
if (idx < 0 || weekAxis.length < 2) return 0;
|
||
return (idx / (weekAxis.length - 1)) * SPARK_W;
|
||
}
|
||
|
||
function windowTasteHitRate(s: TrendSeries): number {
|
||
let plays = 0;
|
||
let hits = 0;
|
||
for (const p of s.points) {
|
||
plays += p.plays;
|
||
hits += p.taste_hit_rate * p.plays;
|
||
}
|
||
return plays > 0 ? hits / plays : 0;
|
||
}
|
||
|
||
function latest(s: TrendSeries): { skip: number; completion: number } {
|
||
const last = s.points[s.points.length - 1];
|
||
return last ? { skip: last.skip_rate, completion: last.avg_completion } : { skip: 0, completion: 0 };
|
||
}
|
||
|
||
function pct(v: number): string {
|
||
return `${(v * 100).toFixed(0)}%`;
|
||
}
|
||
|
||
function markerSummary(m: TrendMarker): string {
|
||
const when = m.changed_at.slice(0, 10);
|
||
if (m.action === 'reset') return `${when} — ${m.scope} reset to defaults`;
|
||
const fields = (m.changes ?? []).map((c) => `${c.field} ${c.old}→${c.new}`).join(', ');
|
||
return `${when} — ${m.scope}: ${fields}`;
|
||
}
|
||
</script>
|
||
|
||
<svelte:head>
|
||
<title>{pageTitle('Tuning')}</title>
|
||
</svelte:head>
|
||
|
||
<div class="space-y-6 p-4">
|
||
<div>
|
||
<h1 class="text-xl font-semibold">Recommendation tuning</h1>
|
||
<p class="mt-1 max-w-3xl text-sm text-text-secondary">
|
||
The defaults-discovery lab. Changes apply live — radio on the next request, daily mixes on
|
||
the next rebuild — and every change is recorded so the metrics page can tie outcome shifts
|
||
to knob turns. Found-good values get baked into shipped defaults; a dot marks knobs that
|
||
currently deviate from them.
|
||
</p>
|
||
</div>
|
||
|
||
{#if loadFailed}
|
||
<p class="text-sm text-text-secondary">Couldn't load tuning settings.</p>
|
||
{:else if !snapshot}
|
||
<p class="text-sm text-text-secondary">Loading…</p>
|
||
{:else}
|
||
<section class="grid gap-4 lg:grid-cols-2">
|
||
{#each profileScopes as p (p.scope)}
|
||
<div class="space-y-3 rounded border border-border bg-surface p-4">
|
||
<div>
|
||
<h2 class="text-lg font-semibold">{p.label}</h2>
|
||
<p class="text-xs text-text-secondary">{p.blurb}</p>
|
||
</div>
|
||
<div class="space-y-2">
|
||
{#each weightFields as f (f.key)}
|
||
<div class="grid grid-cols-[1fr_7rem] items-center gap-2">
|
||
<label class="text-sm" for="{p.scope}-{f.key}" title={f.hint}>
|
||
{f.label}
|
||
{#if deviates(p.scope, f.key)}
|
||
<span
|
||
class="ml-1 inline-block h-1.5 w-1.5 rounded-full bg-accent align-middle"
|
||
title="Deviates from the shipped default ({snapshot.shipped.profiles[p.scope][f.key]})"
|
||
></span>
|
||
{/if}
|
||
</label>
|
||
<input
|
||
id="{p.scope}-{f.key}"
|
||
type="number"
|
||
step="0.1"
|
||
bind:value={form[p.scope][f.key]}
|
||
class="w-full rounded border border-border bg-background px-2 py-1 text-right text-sm tabular-nums outline-none focus:border-accent"
|
||
/>
|
||
</div>
|
||
{/each}
|
||
</div>
|
||
<div class="flex gap-2">
|
||
<button
|
||
type="button"
|
||
disabled={saving !== null}
|
||
onclick={() => save(p.scope)}
|
||
class="rounded bg-accent px-3 py-1.5 text-sm text-white disabled:opacity-50"
|
||
>
|
||
Save {p.label.toLowerCase()}
|
||
</button>
|
||
<button
|
||
type="button"
|
||
disabled={saving !== null}
|
||
onclick={() => reset(p.scope)}
|
||
class="rounded border border-border px-3 py-1.5 text-sm text-text-secondary hover:text-text-primary disabled:opacity-50"
|
||
>
|
||
Reset to defaults
|
||
</button>
|
||
</div>
|
||
</div>
|
||
{/each}
|
||
</section>
|
||
|
||
<section class="space-y-3 rounded border border-border bg-surface p-4 lg:max-w-xl">
|
||
<div>
|
||
<h2 class="text-lg font-semibold">Taste profile build</h2>
|
||
<p class="text-xs text-text-secondary">
|
||
How plays become the learned taste profile: the influence half-life and the
|
||
completion→engagement curve (must stay ordered: hard-skip < neutral < full).
|
||
</p>
|
||
</div>
|
||
<div class="space-y-2">
|
||
{#each tasteFields as f (f.key)}
|
||
<div class="grid grid-cols-[1fr_7rem] items-center gap-2">
|
||
<label class="text-sm" for="taste-{f.key}" title={f.hint}>
|
||
{f.label}
|
||
{#if deviates('taste', f.key)}
|
||
<span
|
||
class="ml-1 inline-block h-1.5 w-1.5 rounded-full bg-accent align-middle"
|
||
title="Deviates from the shipped default ({snapshot.shipped.taste[f.key]})"
|
||
></span>
|
||
{/if}
|
||
</label>
|
||
<input
|
||
id="taste-{f.key}"
|
||
type="number"
|
||
step={f.key === 'half_life_days' ? '1' : '0.05'}
|
||
bind:value={form.taste[f.key]}
|
||
class="w-full rounded border border-border bg-background px-2 py-1 text-right text-sm tabular-nums outline-none focus:border-accent"
|
||
/>
|
||
</div>
|
||
{/each}
|
||
</div>
|
||
<div class="flex gap-2">
|
||
<button
|
||
type="button"
|
||
disabled={saving !== null}
|
||
onclick={() => save('taste')}
|
||
class="rounded bg-accent px-3 py-1.5 text-sm text-white disabled:opacity-50"
|
||
>
|
||
Save taste
|
||
</button>
|
||
<button
|
||
type="button"
|
||
disabled={saving !== null}
|
||
onclick={() => reset('taste')}
|
||
class="rounded border border-border px-3 py-1.5 text-sm text-text-secondary hover:text-text-primary disabled:opacity-50"
|
||
>
|
||
Reset to defaults
|
||
</button>
|
||
</div>
|
||
</section>
|
||
{/if}
|
||
|
||
<!-- Weekly trends (#1251): the verify half. Sparklines share one
|
||
week axis; dashed ticks mark knob turns so cause→effect reads
|
||
off the chart. -->
|
||
<section class="space-y-3 rounded border border-border bg-surface p-4">
|
||
<div>
|
||
<h2 class="text-lg font-semibold">Weekly trends</h2>
|
||
<p class="text-xs text-text-secondary">
|
||
Skip rate per surface over the last {trends?.weeks ?? 12} weeks (lower is better; all
|
||
users aggregated, rates only). Dashed ticks mark tuning changes. Taste hit is the share
|
||
of plays whose artist fits the current taste profile.
|
||
</p>
|
||
</div>
|
||
{#if trendsFailed}
|
||
<p class="text-sm text-text-secondary">Couldn't load trends.</p>
|
||
{:else if !trends}
|
||
<p class="text-sm text-text-secondary">Loading…</p>
|
||
{:else if trends.series.length === 0}
|
||
<p class="text-sm text-text-secondary">
|
||
No plays recorded yet — trends appear once listening accumulates.
|
||
</p>
|
||
{:else}
|
||
<table class="w-full text-sm">
|
||
<thead>
|
||
<tr class="text-left text-text-secondary">
|
||
<th class="py-1 font-medium">Surface</th>
|
||
<th class="py-1 font-medium">Skip rate by week</th>
|
||
<th class="py-1 text-right font-medium">Plays</th>
|
||
<th class="py-1 text-right font-medium">Latest skip</th>
|
||
<th class="py-1 text-right font-medium">Latest completion</th>
|
||
<th class="py-1 text-right font-medium">Taste hit</th>
|
||
</tr>
|
||
</thead>
|
||
<tbody>
|
||
{#each trends.series as s (s.key)}
|
||
<tr
|
||
class="border-t border-border"
|
||
class:opacity-60={s.plays < TREND_LOW_VOLUME}
|
||
title={s.plays < TREND_LOW_VOLUME
|
||
? 'Fewer than 20 plays in the window — treat as anecdote, not signal.'
|
||
: undefined}
|
||
>
|
||
<td class="py-1.5 pr-2">{s.label}</td>
|
||
<td class="py-1.5">
|
||
<svg
|
||
width={SPARK_W}
|
||
height={SPARK_H}
|
||
viewBox="0 0 {SPARK_W} {SPARK_H}"
|
||
role="img"
|
||
aria-label="{s.label} weekly skip rate"
|
||
data-testid="sparkline-{s.key}"
|
||
>
|
||
<line x1="0" y1={SPARK_H - 0.5} x2={SPARK_W} y2={SPARK_H - 0.5}
|
||
stroke="currentColor" opacity="0.15" />
|
||
{#each trends.markers as m, i (i)}
|
||
<line
|
||
x1={markerX(m)} y1="0" x2={markerX(m)} y2={SPARK_H}
|
||
stroke="currentColor" opacity="0.35" stroke-dasharray="2,2"
|
||
>
|
||
<title>{markerSummary(m)}</title>
|
||
</line>
|
||
{/each}
|
||
{#if s.points.length > 1}
|
||
<polyline
|
||
class="text-accent"
|
||
points={sparkPoints(s)}
|
||
fill="none"
|
||
stroke="currentColor"
|
||
stroke-width="1.5"
|
||
/>
|
||
{:else if s.points.length === 1}
|
||
<circle
|
||
class="text-accent"
|
||
cx={xFor(s.points[0].week_start)}
|
||
cy={SPARK_H - s.points[0].skip_rate * SPARK_H}
|
||
r="2" fill="currentColor"
|
||
/>
|
||
{/if}
|
||
</svg>
|
||
</td>
|
||
<td class="py-1.5 text-right tabular-nums">{s.plays}</td>
|
||
<td class="py-1.5 text-right tabular-nums">{pct(latest(s).skip)}</td>
|
||
<td class="py-1.5 text-right tabular-nums">{pct(latest(s).completion)}</td>
|
||
<td class="py-1.5 text-right tabular-nums">{pct(windowTasteHitRate(s))}</td>
|
||
</tr>
|
||
{/each}
|
||
</tbody>
|
||
</table>
|
||
{#if trends.markers.length > 0}
|
||
<div class="space-y-1">
|
||
<h3 class="text-sm font-medium">Tuning changes in this window</h3>
|
||
<ul class="space-y-0.5 text-xs text-text-secondary">
|
||
{#each trends.markers as m, i (i)}
|
||
<li>{markerSummary(m)}</li>
|
||
{/each}
|
||
</ul>
|
||
</div>
|
||
{/if}
|
||
{/if}
|
||
</section>
|
||
</div>
|