feat(tuning): scoring weights → DB-backed admin tuning lab
The recommendation scoring knobs move out of YAML (radio profile) and out of the systemMixWeights hard-code (daily_mix profile) into DB-backed settings with live effect (#1250) — the defaults-discovery lab per decision #1247: the operator turns knobs to find good values, which then get baked back into shipped defaults; end users and other operators should never need the card. - Migration 0040: recommendation_weight_profiles (radio / daily_mix, 8 weight columns), taste_tuning singleton (engagement half-life + completion-curve points), recommendation_tuning_audit (one row per change with a {field, old, new} diff — the trend view's markers, #1251). - internal/recsettings: boot reconcile seeds shipped defaults without clobbering tuned rows (coverart SettingsService pattern), validates patches (bounds, curve ordering), writes audit rows, and pushes daily_mix weights + taste config into package playlists. No-op patches write no audit row. - playlists gains SetSystemMixWeights / SetTasteConfig swap points under a RWMutex — no signature threading through the producers; the scheduler's taste rebuild reads the pushed config. - Radio reads its weight profile from the service per request; the 8 weight fields leave config.RecommendationConfig (YAML keeps only RecentlyPlayedHours / RadioSize / RadioSizeMax). - Admin API: GET/PATCH/reset under /api/admin/recommendation-tuning, echoing current + shipped values. - Web: new admin Tuning tab — two weight profiles side by side, taste card, per-scope save (changed fields only) + reset, deviation dots against shipped defaults. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01TsF3cNoKrqCYsU78cXC8U6
This commit is contained in:
@@ -0,0 +1,49 @@
|
||||
import { api } from './client';
|
||||
|
||||
// Mirrors internal/api/admin_recommendation_tuning.go (#1250): the
|
||||
// recommendation tuning lab. Field names are the server's snake_case
|
||||
// keys, used verbatim in PATCH bodies.
|
||||
|
||||
export type WeightProfile = {
|
||||
base_weight: number;
|
||||
like_boost: number;
|
||||
recency_weight: number;
|
||||
skip_penalty: number;
|
||||
jitter_magnitude: number;
|
||||
context_weight: number;
|
||||
similarity_weight: number;
|
||||
taste_weight: number;
|
||||
};
|
||||
|
||||
export type TasteTuning = {
|
||||
half_life_days: number;
|
||||
engagement_hard_skip: number;
|
||||
engagement_neutral: number;
|
||||
engagement_full: number;
|
||||
};
|
||||
|
||||
export type TuningScope = 'radio' | 'daily_mix' | 'taste';
|
||||
|
||||
export type TuningSnapshot = {
|
||||
profiles: Record<'radio' | 'daily_mix', WeightProfile>;
|
||||
taste: TasteTuning;
|
||||
shipped: {
|
||||
profiles: Record<'radio' | 'daily_mix', WeightProfile>;
|
||||
taste: TasteTuning;
|
||||
};
|
||||
};
|
||||
|
||||
export function getTuning(): Promise<TuningSnapshot> {
|
||||
return api.get<TuningSnapshot>('/api/admin/recommendation-tuning');
|
||||
}
|
||||
|
||||
export function patchTuning(
|
||||
scope: TuningScope,
|
||||
values: Record<string, number>
|
||||
): Promise<TuningSnapshot> {
|
||||
return api.patch<TuningSnapshot>(`/api/admin/recommendation-tuning/${scope}`, { values });
|
||||
}
|
||||
|
||||
export function resetTuning(scope: TuningScope): Promise<TuningSnapshot> {
|
||||
return api.post<TuningSnapshot>(`/api/admin/recommendation-tuning/${scope}/reset`, {});
|
||||
}
|
||||
@@ -10,6 +10,7 @@
|
||||
{ href: '/admin/quarantine', label: 'Quarantine' },
|
||||
{ href: '/admin/playback-errors', label: 'Playback errors' },
|
||||
{ href: '/admin/diagnostics', label: 'Diagnostics' },
|
||||
{ href: '/admin/tuning', label: 'Tuning' },
|
||||
{ href: '/admin/users', label: 'Users' }
|
||||
];
|
||||
|
||||
|
||||
@@ -52,7 +52,7 @@ describe('AdminTabs', () => {
|
||||
);
|
||||
});
|
||||
|
||||
test('renders all seven tabs in order', () => {
|
||||
test('renders all eight tabs in order', () => {
|
||||
state.pageUrl = new URL('http://localhost/admin');
|
||||
render(AdminTabs);
|
||||
const links = screen.getAllByRole('link');
|
||||
@@ -63,6 +63,7 @@ describe('AdminTabs', () => {
|
||||
'Quarantine',
|
||||
'Playback errors',
|
||||
'Diagnostics',
|
||||
'Tuning',
|
||||
'Users'
|
||||
]);
|
||||
});
|
||||
|
||||
@@ -0,0 +1,250 @@
|
||||
<script lang="ts">
|
||||
import { pageTitle } from '$lib/branding';
|
||||
import {
|
||||
getTuning,
|
||||
patchTuning,
|
||||
resetTuning,
|
||||
type TuningScope,
|
||||
type TuningSnapshot,
|
||||
type WeightProfile,
|
||||
type TasteTuning
|
||||
} from '$lib/api/tuning';
|
||||
import { errMessage } from '$lib/api/errors';
|
||||
import { pushToast } from '$lib/stores/toast.svelte';
|
||||
|
||||
// The defaults-discovery lab (#1250): DB-backed scoring-weight
|
||||
// profiles + taste-build knobs with live effect. This card exists to
|
||||
// FIND good values — found-good values get baked into shipped
|
||||
// defaults, so end users and other operators never need it.
|
||||
|
||||
const weightFields: { key: keyof WeightProfile; label: string; hint: string }[] = [
|
||||
{ key: 'base_weight', label: 'Base weight', hint: 'Floor score every candidate starts from.' },
|
||||
{ key: 'like_boost', label: 'Like boost', hint: 'Added when the track is liked.' },
|
||||
{ key: 'recency_weight', label: 'Recency weight', hint: 'Rewards tracks not played recently (0–30d ramp).' },
|
||||
{ key: 'skip_penalty', label: 'Skip penalty', hint: 'Subtracts skips/plays ratio.' },
|
||||
{ key: 'jitter_magnitude', label: 'Jitter', hint: 'Random reshuffle magnitude for near-ties.' },
|
||||
{ key: 'context_weight', label: 'Context weight', hint: 'Session-vector similarity contribution.' },
|
||||
{ key: 'similarity_weight', label: 'Similarity weight', hint: 'Seed-similarity contribution.' },
|
||||
{ key: 'taste_weight', label: 'Taste weight', hint: 'Learned taste-profile fit, in [-1, +1].' }
|
||||
];
|
||||
|
||||
const tasteFields: { key: keyof TasteTuning; label: string; hint: string }[] = [
|
||||
{ key: 'half_life_days', label: 'Half-life (days)', hint: "A play's influence halves every this-many days." },
|
||||
{ key: 'engagement_hard_skip', label: 'Hard-skip point', hint: 'Completion at/below which a play reads −1.' },
|
||||
{ key: 'engagement_neutral', label: 'Neutral point', hint: 'Completion at which a play reads 0.' },
|
||||
{ key: 'engagement_full', label: 'Full point', hint: 'Completion at/above which a play reads +1.' }
|
||||
];
|
||||
|
||||
const profileScopes: { scope: 'radio' | 'daily_mix'; label: string; blurb: string }[] = [
|
||||
{ scope: 'radio', label: 'Radio', blurb: 'Seed-directed listening — the user picked a direction.' },
|
||||
{ scope: 'daily_mix', label: 'Daily mixes', blurb: 'For You, Songs like…, and the discovery mixes.' }
|
||||
];
|
||||
|
||||
let snapshot = $state<TuningSnapshot | null>(null);
|
||||
let loadFailed = $state(false);
|
||||
// Editable copies, string-typed for the inputs; parsed on save.
|
||||
let form = $state<Record<string, Record<string, string>>>({});
|
||||
let saving = $state<TuningScope | null>(null);
|
||||
|
||||
function fillForm(snap: TuningSnapshot) {
|
||||
const f: Record<string, Record<string, string>> = { radio: {}, daily_mix: {}, taste: {} };
|
||||
for (const p of ['radio', 'daily_mix'] as const) {
|
||||
for (const { key } of weightFields) f[p][key] = String(snap.profiles[p][key]);
|
||||
}
|
||||
for (const { key } of tasteFields) f.taste[key] = String(snap.taste[key]);
|
||||
form = f;
|
||||
}
|
||||
|
||||
$effect(() => {
|
||||
getTuning()
|
||||
.then((snap) => {
|
||||
fillForm(snap);
|
||||
snapshot = snap;
|
||||
})
|
||||
.catch(() => {
|
||||
loadFailed = true;
|
||||
});
|
||||
});
|
||||
|
||||
// A knob deviates when its CURRENT SAVED value differs from shipped;
|
||||
// the dot marks where this install has drifted from defaults.
|
||||
function deviates(scope: 'radio' | 'daily_mix' | 'taste', key: string): boolean {
|
||||
if (!snapshot) return false;
|
||||
if (scope === 'taste') {
|
||||
return snapshot.taste[key as keyof TasteTuning] !== snapshot.shipped.taste[key as keyof TasteTuning];
|
||||
}
|
||||
return (
|
||||
snapshot.profiles[scope][key as keyof WeightProfile] !==
|
||||
snapshot.shipped.profiles[scope][key as keyof WeightProfile]
|
||||
);
|
||||
}
|
||||
|
||||
function currentValue(scope: TuningScope, key: string): number {
|
||||
if (!snapshot) return 0;
|
||||
if (scope === 'taste') return snapshot.taste[key as keyof TasteTuning];
|
||||
return snapshot.profiles[scope][key as keyof WeightProfile];
|
||||
}
|
||||
|
||||
async function save(scope: TuningScope) {
|
||||
if (!snapshot) return;
|
||||
const values: Record<string, number> = {};
|
||||
for (const [key, raw] of Object.entries(form[scope] ?? {})) {
|
||||
const v = Number(raw);
|
||||
if (!Number.isFinite(v)) {
|
||||
pushToast(`${key} is not a number.`, 'error');
|
||||
return;
|
||||
}
|
||||
if (v !== currentValue(scope, key)) values[key] = v;
|
||||
}
|
||||
if (Object.keys(values).length === 0) {
|
||||
pushToast('Nothing changed.');
|
||||
return;
|
||||
}
|
||||
saving = scope;
|
||||
try {
|
||||
snapshot = await patchTuning(scope, values);
|
||||
fillForm(snapshot);
|
||||
pushToast('Saved — takes effect on the next scoring pass.');
|
||||
} catch (e: unknown) {
|
||||
pushToast(`Save failed: ${errMessage(e)}`, 'error');
|
||||
} finally {
|
||||
saving = null;
|
||||
}
|
||||
}
|
||||
|
||||
async function reset(scope: TuningScope) {
|
||||
saving = scope;
|
||||
try {
|
||||
snapshot = await resetTuning(scope);
|
||||
fillForm(snapshot);
|
||||
pushToast('Reset to shipped defaults.');
|
||||
} catch (e: unknown) {
|
||||
pushToast(`Reset failed: ${errMessage(e)}`, 'error');
|
||||
} finally {
|
||||
saving = null;
|
||||
}
|
||||
}
|
||||
</script>
|
||||
|
||||
<svelte:head>
|
||||
<title>{pageTitle('Tuning')}</title>
|
||||
</svelte:head>
|
||||
|
||||
<div class="space-y-6 p-4">
|
||||
<div>
|
||||
<h1 class="text-xl font-semibold">Recommendation tuning</h1>
|
||||
<p class="mt-1 max-w-3xl text-sm text-text-secondary">
|
||||
The defaults-discovery lab. Changes apply live — radio on the next request, daily mixes on
|
||||
the next rebuild — and every change is recorded so the metrics page can tie outcome shifts
|
||||
to knob turns. Found-good values get baked into shipped defaults; a dot marks knobs that
|
||||
currently deviate from them.
|
||||
</p>
|
||||
</div>
|
||||
|
||||
{#if loadFailed}
|
||||
<p class="text-sm text-text-secondary">Couldn't load tuning settings.</p>
|
||||
{:else if !snapshot}
|
||||
<p class="text-sm text-text-secondary">Loading…</p>
|
||||
{:else}
|
||||
<section class="grid gap-4 lg:grid-cols-2">
|
||||
{#each profileScopes as p (p.scope)}
|
||||
<div class="space-y-3 rounded border border-border bg-surface p-4">
|
||||
<div>
|
||||
<h2 class="text-lg font-semibold">{p.label}</h2>
|
||||
<p class="text-xs text-text-secondary">{p.blurb}</p>
|
||||
</div>
|
||||
<div class="space-y-2">
|
||||
{#each weightFields as f (f.key)}
|
||||
<div class="grid grid-cols-[1fr_7rem] items-center gap-2">
|
||||
<label class="text-sm" for="{p.scope}-{f.key}" title={f.hint}>
|
||||
{f.label}
|
||||
{#if deviates(p.scope, f.key)}
|
||||
<span
|
||||
class="ml-1 inline-block h-1.5 w-1.5 rounded-full bg-accent align-middle"
|
||||
title="Deviates from the shipped default ({snapshot.shipped.profiles[p.scope][f.key]})"
|
||||
></span>
|
||||
{/if}
|
||||
</label>
|
||||
<input
|
||||
id="{p.scope}-{f.key}"
|
||||
type="number"
|
||||
step="0.1"
|
||||
bind:value={form[p.scope][f.key]}
|
||||
class="w-full rounded border border-border bg-background px-2 py-1 text-right text-sm tabular-nums outline-none focus:border-accent"
|
||||
/>
|
||||
</div>
|
||||
{/each}
|
||||
</div>
|
||||
<div class="flex gap-2">
|
||||
<button
|
||||
type="button"
|
||||
disabled={saving !== null}
|
||||
onclick={() => save(p.scope)}
|
||||
class="rounded bg-accent px-3 py-1.5 text-sm text-white disabled:opacity-50"
|
||||
>
|
||||
Save {p.label.toLowerCase()}
|
||||
</button>
|
||||
<button
|
||||
type="button"
|
||||
disabled={saving !== null}
|
||||
onclick={() => reset(p.scope)}
|
||||
class="rounded border border-border px-3 py-1.5 text-sm text-text-secondary hover:text-text-primary disabled:opacity-50"
|
||||
>
|
||||
Reset to defaults
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
{/each}
|
||||
</section>
|
||||
|
||||
<section class="space-y-3 rounded border border-border bg-surface p-4 lg:max-w-xl">
|
||||
<div>
|
||||
<h2 class="text-lg font-semibold">Taste profile build</h2>
|
||||
<p class="text-xs text-text-secondary">
|
||||
How plays become the learned taste profile: the influence half-life and the
|
||||
completion→engagement curve (must stay ordered: hard-skip < neutral < full).
|
||||
</p>
|
||||
</div>
|
||||
<div class="space-y-2">
|
||||
{#each tasteFields as f (f.key)}
|
||||
<div class="grid grid-cols-[1fr_7rem] items-center gap-2">
|
||||
<label class="text-sm" for="taste-{f.key}" title={f.hint}>
|
||||
{f.label}
|
||||
{#if deviates('taste', f.key)}
|
||||
<span
|
||||
class="ml-1 inline-block h-1.5 w-1.5 rounded-full bg-accent align-middle"
|
||||
title="Deviates from the shipped default ({snapshot.shipped.taste[f.key]})"
|
||||
></span>
|
||||
{/if}
|
||||
</label>
|
||||
<input
|
||||
id="taste-{f.key}"
|
||||
type="number"
|
||||
step={f.key === 'half_life_days' ? '1' : '0.05'}
|
||||
bind:value={form.taste[f.key]}
|
||||
class="w-full rounded border border-border bg-background px-2 py-1 text-right text-sm tabular-nums outline-none focus:border-accent"
|
||||
/>
|
||||
</div>
|
||||
{/each}
|
||||
</div>
|
||||
<div class="flex gap-2">
|
||||
<button
|
||||
type="button"
|
||||
disabled={saving !== null}
|
||||
onclick={() => save('taste')}
|
||||
class="rounded bg-accent px-3 py-1.5 text-sm text-white disabled:opacity-50"
|
||||
>
|
||||
Save taste
|
||||
</button>
|
||||
<button
|
||||
type="button"
|
||||
disabled={saving !== null}
|
||||
onclick={() => reset('taste')}
|
||||
class="rounded border border-border px-3 py-1.5 text-sm text-text-secondary hover:text-text-primary disabled:opacity-50"
|
||||
>
|
||||
Reset to defaults
|
||||
</button>
|
||||
</div>
|
||||
</section>
|
||||
{/if}
|
||||
</div>
|
||||
@@ -0,0 +1,114 @@
|
||||
import { describe, expect, test, vi, beforeEach } from 'vitest';
|
||||
import { render, screen, fireEvent, waitFor } from '@testing-library/svelte';
|
||||
import type { TuningSnapshot } from '$lib/api/tuning';
|
||||
|
||||
vi.mock('$lib/api/tuning', async (orig) => {
|
||||
const actual = (await orig()) as Record<string, unknown>;
|
||||
return {
|
||||
...actual,
|
||||
getTuning: vi.fn(),
|
||||
patchTuning: vi.fn(),
|
||||
resetTuning: vi.fn()
|
||||
};
|
||||
});
|
||||
|
||||
const pushToast = vi.fn();
|
||||
vi.mock('$lib/stores/toast.svelte', () => ({ pushToast: (...a: unknown[]) => pushToast(...a) }));
|
||||
|
||||
import TuningPage from './+page.svelte';
|
||||
import { getTuning, patchTuning, resetTuning } from '$lib/api/tuning';
|
||||
|
||||
const weights = (over: Partial<Record<string, number>> = {}) => ({
|
||||
base_weight: 1,
|
||||
like_boost: 2,
|
||||
recency_weight: 1,
|
||||
skip_penalty: 2,
|
||||
jitter_magnitude: 0.1,
|
||||
context_weight: 0.5,
|
||||
similarity_weight: 1.5,
|
||||
taste_weight: 1.5,
|
||||
...over
|
||||
});
|
||||
|
||||
const taste = (over: Partial<Record<string, number>> = {}) => ({
|
||||
half_life_days: 75,
|
||||
engagement_hard_skip: 0.05,
|
||||
engagement_neutral: 0.3,
|
||||
engagement_full: 0.9,
|
||||
...over
|
||||
});
|
||||
|
||||
function snapshot(over: Partial<TuningSnapshot> = {}): TuningSnapshot {
|
||||
return {
|
||||
profiles: { radio: weights({ taste_weight: 1 }), daily_mix: weights() },
|
||||
taste: taste(),
|
||||
shipped: {
|
||||
profiles: { radio: weights({ taste_weight: 1 }), daily_mix: weights() },
|
||||
taste: taste()
|
||||
},
|
||||
...over
|
||||
} as TuningSnapshot;
|
||||
}
|
||||
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks();
|
||||
});
|
||||
|
||||
describe('Admin tuning page', () => {
|
||||
test('renders both profiles and the taste card with current values', async () => {
|
||||
(getTuning as ReturnType<typeof vi.fn>).mockResolvedValue(snapshot());
|
||||
render(TuningPage);
|
||||
await waitFor(() => expect(screen.getByText('Radio')).toBeInTheDocument());
|
||||
expect(screen.getByText('Daily mixes')).toBeInTheDocument();
|
||||
expect(screen.getByText('Taste profile build')).toBeInTheDocument();
|
||||
const radioTaste = screen.getByLabelText(/taste weight/i, {
|
||||
selector: '#radio-taste_weight'
|
||||
}) as HTMLInputElement;
|
||||
expect(radioTaste.value).toBe('1');
|
||||
const halfLife = document.getElementById('taste-half_life_days') as HTMLInputElement;
|
||||
expect(halfLife.value).toBe('75');
|
||||
});
|
||||
|
||||
test('save sends only the changed fields for the scope', async () => {
|
||||
(getTuning as ReturnType<typeof vi.fn>).mockResolvedValue(snapshot());
|
||||
(patchTuning as ReturnType<typeof vi.fn>).mockResolvedValue(snapshot());
|
||||
render(TuningPage);
|
||||
await waitFor(() => expect(screen.getByText('Radio')).toBeInTheDocument());
|
||||
const input = document.getElementById('radio-taste_weight') as HTMLInputElement;
|
||||
await fireEvent.input(input, { target: { value: '2.5' } });
|
||||
await fireEvent.click(screen.getByRole('button', { name: /save radio/i }));
|
||||
await waitFor(() =>
|
||||
expect(patchTuning).toHaveBeenCalledWith('radio', { taste_weight: 2.5 })
|
||||
);
|
||||
});
|
||||
|
||||
test('save with no changes calls nothing and says so', async () => {
|
||||
(getTuning as ReturnType<typeof vi.fn>).mockResolvedValue(snapshot());
|
||||
render(TuningPage);
|
||||
await waitFor(() => expect(screen.getByText('Radio')).toBeInTheDocument());
|
||||
await fireEvent.click(screen.getByRole('button', { name: /save radio/i }));
|
||||
expect(patchTuning).not.toHaveBeenCalled();
|
||||
await waitFor(() => expect(pushToast).toHaveBeenCalledWith('Nothing changed.'));
|
||||
});
|
||||
|
||||
test('reset calls resetTuning for the scope', async () => {
|
||||
(getTuning as ReturnType<typeof vi.fn>).mockResolvedValue(snapshot());
|
||||
(resetTuning as ReturnType<typeof vi.fn>).mockResolvedValue(snapshot());
|
||||
render(TuningPage);
|
||||
await waitFor(() => expect(screen.getByText('Taste profile build')).toBeInTheDocument());
|
||||
const resetButtons = screen.getAllByRole('button', { name: /reset to defaults/i });
|
||||
await fireEvent.click(resetButtons[resetButtons.length - 1]);
|
||||
await waitFor(() => expect(resetTuning).toHaveBeenCalledWith('taste'));
|
||||
});
|
||||
|
||||
test('marks knobs that deviate from shipped defaults', async () => {
|
||||
const snap = snapshot();
|
||||
snap.profiles.daily_mix = weights({ similarity_weight: 4 });
|
||||
(getTuning as ReturnType<typeof vi.fn>).mockResolvedValue(snap);
|
||||
render(TuningPage);
|
||||
await waitFor(() => expect(screen.getByText('Daily mixes')).toBeInTheDocument());
|
||||
expect(
|
||||
screen.getByTitle(/deviates from the shipped default \(1\.5\)/i)
|
||||
).toBeInTheDocument();
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user