feat(admin): the duplicates report — review proposed duplicate groups (M400 #3912)
test-web / test (push) Successful in 52s
test-go / test (push) Successful in 1m9s
test-go / integration (push) Successful in 3m31s
release / Build signed APK (releases and dev) (push) Successful in 4m32s
release / Build + push container image (push) Successful in 24s
release / Verify release artifacts (tag releases only) (push) Skipped

A new admin tab, Duplicates, beside Missing files: the proposals from
the duplicate sweep, with a Sweep now trigger and a Not duplicates
dismissal. Nothing on it merges or deletes; the merge is #3911.

Each group shows:
- whether it is identical audio or the same recording, with a match
  percentage from the weakest link between members
- every copy's format, size, duration, path, and the likes and plays
  it carries (every user's; this is admin-only, and it is what decides
  which copy to keep)
- the copy proposed to keep, and the rule that chose it

The survivor rule is library.ProposeSurvivor, a pure function the
merge will reuse: lossless over lossy, then the larger file, then the
copy in the library longest, then lowest id. Bitrate is not in it
because the scanner never fills tracks.bitrate, and for one recording
at one duration a larger file is the higher bitrate. m4a is not counted
as lossless: it may be AAC. The reason names the rule that separated
first place from second, not every rule the winner passed.

An empty report has three causes, and the page says which: still
fingerprinting, the sweep has never run, or it ran and found nothing.
The sweep's state and the backfill's progress come back with the groups
for that reason. Groups left with fewer than two members since the
sweep are not shown.

GET /api/admin/library/duplicates, POST .../sweep (202, or 409
sweep_in_progress), POST .../{id}/dismiss (404
duplicate_group_not_pending when already resolved).

Migration 0060 indexes play_events by track_id. Its only indexes led
with user_id, so each copy's play count, and the merge's repointing of
play history, would scan the whole table.

Web only, like Missing files: Android has no library-health admin
screens.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01SQ31KQpYbStyK5y58UmPLH
This commit is contained in:
2026-09-11 17:11:11 -04:00
co-authored by Claude Opus 5
parent 6379b6c31d
commit ff493a8c7d
18 changed files with 1179 additions and 1 deletions
+30
View File
@@ -0,0 +1,30 @@
import { describe, it, expect, vi, beforeEach } from 'vitest';
import { dismissDuplicateGroup, listDuplicates, runDuplicateSweep } from './admin';
vi.mock('./client', () => ({
api: { get: vi.fn(), post: vi.fn() }
}));
import { api } from './client';
describe('admin duplicates API', () => {
beforeEach(() => vi.clearAllMocks());
it('listDuplicates GETs the paged report', async () => {
(api.get as unknown as ReturnType<typeof vi.fn>).mockResolvedValueOnce({ groups: [] });
await listDuplicates(25, 25);
expect(api.get).toHaveBeenCalledWith('/api/admin/library/duplicates?limit=25&offset=25');
});
it('runDuplicateSweep POSTs the trigger', async () => {
(api.post as unknown as ReturnType<typeof vi.fn>).mockResolvedValueOnce({ started: true });
await runDuplicateSweep();
expect(api.post).toHaveBeenCalledWith('/api/admin/library/duplicates/sweep', {});
});
it('dismissDuplicateGroup POSTs to the group', async () => {
(api.post as unknown as ReturnType<typeof vi.fn>).mockResolvedValueOnce(undefined);
await dismissDuplicateGroup('g/1');
expect(api.post).toHaveBeenCalledWith('/api/admin/library/duplicates/g%2F1/dismiss', {});
});
});
+32
View File
@@ -4,6 +4,7 @@ import { qk } from './queries';
import type {
ActionResult,
AdminMissingResponse,
AdminDuplicatesResponse,
AdminPlaybackError,
AdminQuarantineRow,
LidarrConfig,
@@ -695,6 +696,37 @@ export async function updateNetworkSettings(hops: number): Promise<NetworkSettin
});
}
// Duplicates report (#3912) -------------------------------------------------
export async function listDuplicates(
offset: number = 0,
limit: number = 25
): Promise<AdminDuplicatesResponse> {
return api.get<AdminDuplicatesResponse>(
`/api/admin/library/duplicates?limit=${limit}&offset=${offset}`
);
}
// Takes a plain offset, like createMissingFilesQuery, so a $derived caller
// re-creates the query on paging. Polls while a sweep might be running: the
// page is where the operator waits for one to finish.
export function createDuplicatesQuery(offset: number = 0, limit: number = 25) {
return createQuery({
queryKey: qk.adminDuplicates(offset),
queryFn: () => listDuplicates(offset, limit),
staleTime: 30_000,
refetchInterval: 15_000
});
}
export async function runDuplicateSweep(): Promise<{ started: boolean }> {
return api.post<{ started: boolean }>('/api/admin/library/duplicates/sweep', {});
}
export async function dismissDuplicateGroup(id: string): Promise<void> {
await api.post(`/api/admin/library/duplicates/${encodeURIComponent(id)}/dismiss`, {});
}
// Missing files (#2527) -----------------------------------------------------
export async function listMissingFiles(
+2
View File
@@ -56,6 +56,8 @@ export const qk = {
['adminDiagnostics', f] as const,
adminMissingFiles: (offset?: number) =>
['adminMissingFiles', { offset: offset ?? 0 }] as const,
adminDuplicates: (offset?: number) =>
['adminDuplicates', { offset: offset ?? 0 }] as const,
adminDiagnosticDevices: (userId?: string) =>
['adminDiagnosticDevices', { userId: userId ?? 'all' }] as const,
smtpConfig: () => ['smtpConfig'] as const,
+50
View File
@@ -418,3 +418,53 @@ export type AdminMissingResponse = {
offset: number;
groups: AdminMissingGroup[];
};
// Duplicates report (#3912) -------------------------------------------------
// One copy in a proposed duplicate group. like_count and play_count cover every
// user: they are what the operator weighs when choosing which copy to keep.
export type AdminDuplicateMember = {
track_id: string;
title: string;
artist_name: string;
album_id: string;
album_title: string;
file_path: string;
file_format: string;
file_size: number;
duration_sec: number;
added_at: string;
like_count: number;
play_count: number;
};
// exact: identical encoded audio. acoustic: the same recording, differently
// encoded; worst_bit_error_rate is the weakest link between any two members.
export type AdminDuplicateGroup = {
id: string;
tier: 'exact' | 'acoustic';
worst_bit_error_rate: number | null;
detected_at: string;
survivor_track_id: string;
survivor_reason: string;
members: AdminDuplicateMember[];
};
export type AdminDuplicateSweep = {
state: 'never' | 'running' | 'finished';
started_at: string | null;
finished_at: string | null;
candidates: number | null;
groups_found: number | null;
oversize_clusters: number | null;
error_message: string | null;
};
export type AdminDuplicatesResponse = {
sweep: AdminDuplicateSweep;
fingerprints: { total: number; fingerprinted: number; rejected: number; pending: number };
total: number;
limit: number;
offset: number;
groups: AdminDuplicateGroup[];
};
+1
View File
@@ -9,6 +9,7 @@
{ href: '/admin/requests', label: 'Requests' },
{ href: '/admin/quarantine', label: 'Quarantine' },
{ href: '/admin/missing-files', label: 'Missing files' },
{ href: '/admin/duplicates', label: 'Duplicates' },
{ href: '/admin/playback-errors', label: 'Playback errors' },
{ href: '/admin/diagnostics', label: 'Diagnostics' },
{ href: '/admin/tuning', label: 'Tuning' },
+4 -1
View File
@@ -52,7 +52,7 @@ describe('AdminTabs', () => {
);
});
test('renders all nine tabs in order', () => {
test('renders all ten tabs in order', () => {
state.pageUrl = new URL('http://localhost/admin');
render(AdminTabs);
const links = screen.getAllByRole('link');
@@ -64,6 +64,9 @@ describe('AdminTabs', () => {
// Missing files sits with Quarantine and Playback errors: the three
// surfaces that show tracks needing an operator's attention.
'Missing files',
// Duplicates follows Missing files: both are library-health reports on
// what the library holds, rather than a queue of user reports.
'Duplicates',
'Playback errors',
'Diagnostics',
'Tuning',
+2
View File
@@ -42,6 +42,8 @@
"track_not_found": "That track no longer exists.",
"library_not_writable": "The music library isn't writable by the server.",
"file_delete_failed": "The file couldn't be deleted.",
"sweep_in_progress": "A duplicate sweep is already running.",
"duplicate_group_not_pending": "That group has already been resolved.",
"album_not_found": "That album no longer exists.",
"artist_not_found": "That artist no longer exists.",
"playlist_not_found": "That playlist no longer exists.",
@@ -0,0 +1,251 @@
<script lang="ts">
import { pageTitle } from '$lib/branding';
import { Copy } from 'lucide-svelte';
import {
createDuplicatesQuery,
runDuplicateSweep,
dismissDuplicateGroup
} from '$lib/api/admin';
import { errMessage } from '$lib/api/errors';
import { pushToast } from '$lib/stores/toast.svelte';
import { relativeTime } from '$lib/utils/relativeTime';
import type { AdminDuplicateGroup, AdminDuplicateMember } from '$lib/api/types';
// Tracks the duplicate sweep believes hold one recording (#3912). A group is a
// proposal: nothing here deletes or merges. Dismissing one says "these are not
// duplicates", and the sweep will not propose that set again.
const PAGE_SIZE = 25;
let offset = $state(0);
let sweeping = $state(false);
let dismissing = $state<string | null>(null);
const queryStore = $derived(createDuplicatesQuery(offset, PAGE_SIZE));
const query = $derived($queryStore);
const data = $derived(query.data);
const groups = $derived((data?.groups ?? []) as AdminDuplicateGroup[]);
const total = $derived(data?.total ?? 0);
const hasMore = $derived(offset + groups.length < total);
const sweep = $derived(data?.sweep);
const prints = $derived(data?.fingerprints);
async function onRunSweep() {
sweeping = true;
try {
await runDuplicateSweep();
pushToast('Duplicate sweep started.');
query.refetch();
} catch (e: unknown) {
pushToast(errMessage(e), 'error');
} finally {
sweeping = false;
}
}
async function onDismiss(group: AdminDuplicateGroup) {
dismissing = group.id;
try {
await dismissDuplicateGroup(group.id);
pushToast('Marked as not duplicates.');
query.refetch();
} catch (e: unknown) {
pushToast(errMessage(e), 'error');
} finally {
dismissing = null;
}
}
// "Identical audio" and "same recording" are different claims, and an
// operator deciding whether to merge needs to know which one they are
// looking at before anything else.
function tierLabel(g: AdminDuplicateGroup): string {
if (g.tier === 'exact') return 'Identical audio';
const match = Math.round((1 - (g.worst_bit_error_rate ?? 0)) * 100);
return `Same recording · ${match}% match`;
}
function sizeLabel(bytes: number): string {
return `${(bytes / 1_000_000).toFixed(1)} MB`;
}
function durationLabel(sec: number): string {
const m = Math.floor(sec / 60);
const s = sec % 60;
return `${m}:${s.toString().padStart(2, '0')}`;
}
function countLabel(n: number, one: string, many: string): string {
return n === 1 ? `1 ${one}` : `${n} ${many}`;
}
// What the copy carries is the fact that decides which to keep.
function historyLabel(m: AdminDuplicateMember): string {
if (m.like_count === 0 && m.play_count === 0) return 'no likes or plays';
return `${countLabel(m.like_count, 'like', 'likes')} · ${countLabel(m.play_count, 'play', 'plays')}`;
}
</script>
<svelte:head><title>{pageTitle('Admin · Duplicates')}</title></svelte:head>
<div class="space-y-6">
<header class="space-y-1">
<div class="flex items-center justify-between gap-4">
<div class="flex items-center gap-2">
<h2 class="font-display text-2xl font-medium text-text-primary">Duplicates</h2>
{#if total > 0}
<span
class="inline-flex items-center rounded-full bg-accent-tint px-2 py-0.5 text-xs text-accent"
data-testid="duplicates-count-pill"
>
{total}
</span>
{/if}
</div>
<button
type="button"
onclick={onRunSweep}
disabled={sweeping || sweep?.state === 'running'}
class="flex h-8 items-center gap-1 rounded-md bg-action-primary px-4 text-sm text-action-fg hover:opacity-90 disabled:opacity-50"
>
{sweeping ? 'Starting…' : 'Sweep now'}
</button>
</div>
<p class="text-text-secondary">
Tracks that hold the same recording more than once. Review each group; nothing is
merged or removed from here.
</p>
</header>
{#if sweep}
<p class="text-sm text-text-secondary" data-testid="sweep-status">
{#if sweep.state === 'never'}
The duplicate sweep hasn't run yet.
{:else if sweep.state === 'running'}
Sweeping now — started {relativeTime(sweep.started_at ?? '')}.
{:else}
Last swept {relativeTime(sweep.finished_at ?? '')}, comparing
{(sweep.candidates ?? 0).toLocaleString()} tracks.
{#if sweep.error_message}
<span class="text-oxblood">It stopped early: {sweep.error_message}</span>
{/if}
{/if}
{#if prints && prints.pending > 0}
{prints.pending.toLocaleString()} tracks are still waiting for a fingerprint and join
the comparison once they have one.
{/if}
</p>
{/if}
{#if query.isPending}
<p class="text-text-secondary">Loading duplicates…</p>
{:else if query.isError}
<p class="text-error">Couldn't load the duplicates report.</p>
{:else if groups.length === 0}
<!-- Three states all show zero groups and mean different things. Telling
them apart is what stops an empty page reading as a broken feature. -->
<div class="rounded-lg border border-border bg-surface p-6 text-center" data-testid="empty-state">
<Copy size={28} strokeWidth={1} class="mx-auto text-text-muted" />
{#if prints && prints.total > 0 && prints.fingerprinted === 0}
<p class="mt-3 text-text-primary">Nothing to compare yet.</p>
<p class="mt-1 text-sm text-text-secondary">
The library is still being fingerprinted — {prints.pending.toLocaleString()} tracks to go.
Duplicates appear here as the sweep finds them.
</p>
{:else if sweep?.state === 'never'}
<p class="mt-3 text-text-primary">The sweep hasn't run yet.</p>
<p class="mt-1 text-sm text-text-secondary">
It runs on its own whenever new fingerprints arrive, or now if you start it.
</p>
{:else if sweep?.state === 'running'}
<p class="mt-3 text-text-primary">Sweeping…</p>
<p class="mt-1 text-sm text-text-secondary">Anything it finds will appear here.</p>
{:else}
<p class="mt-3 text-text-primary">No duplicates found.</p>
<p class="mt-1 text-sm text-text-secondary">
A group appears when two tracks hold identical audio, or the same recording in a
different encoding. Groups you dismiss don't come back.
</p>
{/if}
</div>
{:else}
<ul class="space-y-4">
{#each groups as group (group.id)}
<li class="overflow-hidden rounded-lg border border-border bg-surface" data-testid="duplicate-group">
<div class="flex items-center justify-between gap-4 border-b border-border px-4 py-3">
<div class="min-w-0">
<h3 class="text-sm text-text-primary" data-testid="duplicate-tier">{tierLabel(group)}</h3>
<p class="text-xs text-text-secondary">Found {relativeTime(group.detected_at)}</p>
</div>
<button
type="button"
onclick={() => onDismiss(group)}
disabled={dismissing === group.id}
class="shrink-0 rounded-md border border-border px-3 py-1.5 text-sm text-text-secondary hover:bg-surface-hover hover:text-text-primary disabled:opacity-50"
>
Not duplicates
</button>
</div>
<ul class="divide-y divide-border">
{#each group.members as m (m.track_id)}
{@const keep = m.track_id === group.survivor_track_id}
<li class="flex items-start gap-3 px-4 py-3" data-testid="duplicate-member">
<div class="min-w-0 flex-1 space-y-0.5">
<div class="flex items-center gap-2">
<span class="truncate text-sm text-text-primary">{m.title}</span>
{#if keep}
<!-- The proposed copy to keep, with the rule that chose it.
A default the merge will let the operator override. -->
<span
class="shrink-0 rounded-full bg-accent-tint px-2 py-0.5 text-xs text-accent"
data-testid="survivor-badge"
title="Proposed to keep: {group.survivor_reason}"
>
Keep · {group.survivor_reason}
</span>
{/if}
</div>
<div class="truncate text-xs text-text-secondary">
{m.artist_name} · {m.album_title}
</div>
<div class="truncate font-mono text-xs text-text-muted" title={m.file_path}>
{m.file_path}
</div>
</div>
<div class="shrink-0 space-y-0.5 text-right text-xs text-text-muted">
<div>{m.file_format.toUpperCase()} · {sizeLabel(m.file_size)} · {durationLabel(m.duration_sec)}</div>
<div data-testid="member-history">{historyLabel(m)}</div>
</div>
</li>
{/each}
</ul>
</li>
{/each}
</ul>
{#if hasMore || offset > 0}
<nav class="flex items-center justify-between" aria-label="Duplicates pages">
<button
type="button"
class="rounded border border-border px-3 py-1.5 text-sm text-text-primary hover:bg-surface-hover disabled:cursor-not-allowed disabled:opacity-40"
disabled={offset === 0}
onclick={() => (offset = Math.max(0, offset - PAGE_SIZE))}
>
Previous
</button>
<span class="text-xs text-text-secondary">
{offset + 1}{offset + groups.length} of {total}
</span>
<button
type="button"
class="rounded border border-border px-3 py-1.5 text-sm text-text-primary hover:bg-surface-hover disabled:cursor-not-allowed disabled:opacity-40"
disabled={!hasMore}
onclick={() => (offset += PAGE_SIZE)}
>
Next
</button>
</nav>
{/if}
{/if}
</div>
@@ -0,0 +1,147 @@
import { afterEach, describe, expect, test, vi } from 'vitest';
import { fireEvent, render, screen } from '@testing-library/svelte';
import { mockQuery } from '../../../test-utils/query';
import type { AdminDuplicatesResponse } from '$lib/api/types';
vi.mock('$lib/api/admin', () => ({
createDuplicatesQuery: vi.fn(),
runDuplicateSweep: vi.fn().mockResolvedValue({ started: true }),
dismissDuplicateGroup: vi.fn().mockResolvedValue(undefined)
}));
import AdminDuplicatesPage from './+page.svelte';
import { createDuplicatesQuery, dismissDuplicateGroup, runDuplicateSweep } from '$lib/api/admin';
const HOUR = 3_600_000;
const ago = (ms: number) => new Date(Date.now() - ms).toISOString();
const finishedSweep = {
state: 'finished' as const,
started_at: ago(2 * HOUR),
finished_at: ago(HOUR),
candidates: 1200,
groups_found: 1,
oversize_clusters: 0,
error_message: null
};
const allFingerprinted = { total: 1200, fingerprinted: 1200, rejected: 0, pending: 0 };
function member(id: string, extra: Partial<AdminDuplicatesResponse['groups'][number]['members'][number]> = {}) {
return {
track_id: id,
title: 'WWW',
artist_name: 'Moe Shop',
album_id: 'al-1',
album_title: 'WWW',
file_path: `/music/Moe Shop/WWW (2020)/${id}.mp3`,
file_format: 'mp3',
file_size: 6_900_000,
duration_sec: 215,
added_at: ago(500 * HOUR),
like_count: 0,
play_count: 0,
...extra
};
}
function response(over: Partial<AdminDuplicatesResponse> = {}): AdminDuplicatesResponse {
return {
sweep: finishedSweep,
fingerprints: allFingerprinted,
total: 1,
limit: 25,
offset: 0,
groups: [
{
id: 'g-1',
tier: 'exact',
worst_bit_error_rate: null,
detected_at: ago(HOUR),
survivor_track_id: 'www-01',
survivor_reason: 'in the library longest',
members: [member('www-01', { like_count: 2, play_count: 14 }), member('www-02')]
}
],
...over
};
}
function renderWith(data: AdminDuplicatesResponse | undefined) {
vi.mocked(createDuplicatesQuery).mockReturnValue(
mockQuery({ data }) as ReturnType<typeof createDuplicatesQuery>
);
return render(AdminDuplicatesPage);
}
function text(el: HTMLElement): string {
return (el.textContent ?? '').replace(/\s+/g, ' ').trim();
}
afterEach(() => vi.clearAllMocks());
describe('admin duplicates', () => {
test('shows each group with its members, what they carry, and the copy to keep', () => {
renderWith(response());
expect(screen.getAllByTestId('duplicate-group')).toHaveLength(1);
expect(screen.getAllByTestId('duplicate-member')).toHaveLength(2);
expect(text(screen.getByTestId('duplicate-tier'))).toBe('Identical audio');
expect(text(screen.getByTestId('survivor-badge'))).toBe('Keep · in the library longest');
const history = screen.getAllByTestId('member-history').map(text);
expect(history).toEqual(['2 likes · 14 plays', 'no likes or plays']);
});
// The badge must sit on the proposed survivor, not merely appear somewhere.
test('the keep badge is on the survivor row', () => {
renderWith(response());
const rows = screen.getAllByTestId('duplicate-member');
expect(rows[0].querySelector('[data-testid="survivor-badge"]')).not.toBeNull();
expect(rows[1].querySelector('[data-testid="survivor-badge"]')).toBeNull();
});
test('an acoustic group reads as a match percentage', () => {
const r = response();
r.groups[0] = { ...r.groups[0], tier: 'acoustic', worst_bit_error_rate: 0.04 };
renderWith(r);
expect(text(screen.getByTestId('duplicate-tier'))).toBe('Same recording · 96% match');
});
// Three states all show zero groups. Each must say which it is.
test('empty while still fingerprinting says so', () => {
renderWith(
response({
groups: [],
total: 0,
fingerprints: { total: 1200, fingerprinted: 0, rejected: 0, pending: 1200 }
})
);
expect(text(screen.getByTestId('empty-state'))).toContain('still being fingerprinted');
});
test('empty before any sweep says the sweep has not run', () => {
renderWith(
response({
groups: [],
total: 0,
sweep: { ...finishedSweep, state: 'never', started_at: null, finished_at: null, candidates: null }
})
);
expect(text(screen.getByTestId('empty-state'))).toContain("hasn't run yet");
});
test('empty after a sweep says no duplicates were found', () => {
renderWith(response({ groups: [], total: 0 }));
expect(text(screen.getByTestId('empty-state'))).toContain('No duplicates found');
});
test('Not duplicates dismisses that group', async () => {
renderWith(response());
await fireEvent.click(screen.getByRole('button', { name: 'Not duplicates' }));
expect(dismissDuplicateGroup).toHaveBeenCalledWith('g-1');
});
test('Sweep now starts a sweep', async () => {
renderWith(response());
await fireEvent.click(screen.getByRole('button', { name: 'Sweep now' }));
expect(runDuplicateSweep).toHaveBeenCalledTimes(1);
});
});