feat(library): backfill fingerprints for the existing library — M400 #3908
test-web / test (push) Failing after 50s
test-go / test (push) Successful in 1m7s
test-go / integration (push) Successful in 3m27s
release / Build + push container image (push) Canceled after 0s
release / Verify release artifacts (tag releases only) (push) Canceled after 0s
release / Build signed APK (releases and dev) (push) Canceled after 4m23s

The scan fingerprints only bytes it has not seen, so everything imported
before fingerprinting existed, and any row derived by an older
fingerprintVersion, needs a pass of its own.

That pass is a background worker, not a stage in RunScan. RunScan runs
at boot and then every 12h, and an in-flight scan older than an hour is
reaped and a second started beside it. A stage would have to stop inside
the hour: a few hundred decodes a run, so about a month for a 50k-track
library. It would also hold the run in flight and answer manual
rescans with 409 while it worked.

FingerprintBackfillWorker runs once at start, then hourly. Nothing a
pass does (error or panic) can stop the next tick. A pass walks tracks
with no fingerprint or a stale version, skipping missing tracks,
keyset-paged on id. The cursor is what lets a pass end: an inconclusive
attempt writes no row, so a file that keeps timing out would otherwise
be re-listed and retried forever. Two decodes at a time, deliberately:
they compete with transcoding for CPU and with streaming for the mount.

storeFingerprint is now one package function shared by the scan and
the worker, and reports whether the attempt was fingerprinted,
rejected, inconclusive or failed to store.

Progress is a live gauge on the Admin scan card, served by
GET /api/admin/library/fingerprints: fingerprinted / rejected / pending
of total, with missing tracks excluded so it can reach the end.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01SQ31KQpYbStyK5y58UmPLH
This commit is contained in:
2026-09-11 14:56:18 -04:00
co-authored by Claude Opus 5
parent 71d4335584
commit b8855b480f
13 changed files with 633 additions and 16 deletions
@@ -0,0 +1,32 @@
import { describe, it, expect, vi, beforeEach } from 'vitest';
import { getFingerprintCoverage, type FingerprintCoverage } from './admin';
vi.mock('./client', () => ({
api: { get: vi.fn(), post: vi.fn() }
}));
import { api } from './client';
describe('admin fingerprint coverage API', () => {
beforeEach(() => vi.clearAllMocks());
it('getFingerprintCoverage GETs the correct path', async () => {
const sample: FingerprintCoverage = {
total: 18026,
fingerprinted: 9400,
rejected: 12,
pending: 8614
};
(api.get as unknown as ReturnType<typeof vi.fn>).mockResolvedValueOnce(sample);
const got = await getFingerprintCoverage();
expect(api.get).toHaveBeenCalledWith('/api/admin/library/fingerprints');
expect(got).toEqual(sample);
});
it('buckets sum to the total', async () => {
const sample: FingerprintCoverage = { total: 10, fingerprinted: 6, rejected: 1, pending: 3 };
(api.get as unknown as ReturnType<typeof vi.fn>).mockResolvedValueOnce(sample);
const got = await getFingerprintCoverage();
expect(got.fingerprinted + got.rejected + got.pending).toBe(got.total);
});
});
+24
View File
@@ -313,6 +313,30 @@ export function createCoverageQuery() {
});
}
// Fingerprint backfill (#3908) --------------------------------------------
export type FingerprintCoverage = {
total: number;
fingerprinted: number;
rejected: number;
pending: number;
};
export async function getFingerprintCoverage(): Promise<FingerprintCoverage> {
return api.get<FingerprintCoverage>('/api/admin/library/fingerprints');
}
// Polled far less often than the cover gauge: the backfill decodes files two at
// a time, so the count moves by a few tracks a minute and a 3s poll is noise.
export function createFingerprintCoverageQuery() {
return createQuery({
queryKey: qk.fingerprintCoverage(),
queryFn: getFingerprintCoverage,
staleTime: 30_000,
refetchInterval: 30_000
});
}
// Cover-art providers ------------------------------------------------------
export type CoverProviderCapability = 'album_cover' | 'artist_thumb' | 'artist_fanart';
+1
View File
@@ -47,6 +47,7 @@ export const qk = {
['adminPlaybackErrors', { resolved: resolved ?? false }] as const,
scanStatus: () => ['scanStatus'] as const,
coverage: () => ['coverage'] as const,
fingerprintCoverage: () => ['fingerprintCoverage'] as const,
coverProviders: () => ['coverProviders'] as const,
tagProviders: () => ['tagProviders'] as const,
adminUsers: () => ['adminUsers'] as const,
+28
View File
@@ -10,6 +10,7 @@
createAdminQuarantineQuery,
createScanStatusQuery,
createCoverageQuery,
createFingerprintCoverageQuery,
approveRequest,
rejectRequest,
resolveQuarantine,
@@ -177,6 +178,11 @@
const coverageQ = $derived($coverageStore);
const coverage = $derived(coverageQ.data);
// ---- Fingerprint backfill gauge (#3908) ----
const fingerprintStore = $derived(createFingerprintCoverageQuery());
const fingerprintQ = $derived($fingerprintStore);
const fingerprints = $derived(fingerprintQ.data);
let triggering = $state(false);
let triggerResult = $state<string | null>(null);
@@ -428,6 +434,28 @@
{#if triggerResult}
<p class="mt-2 text-sm">{triggerResult}</p>
{/if}
<!-- Fingerprint backfill (#3908). A worker of its own rather than a scan
stage, so its progress is read live here, not from the run above. -->
{#if fingerprints && fingerprints.total > 0}
<div class="mt-3 flex flex-wrap items-center gap-3 text-sm">
<span class="text-xs font-medium uppercase tracking-wide text-text-muted">Fingerprints</span>
<span>{fingerprints.fingerprinted.toLocaleString()} of {fingerprints.total.toLocaleString()} tracks</span>
{#if fingerprints.pending > 0}
<span class="text-text-muted">·</span>
<span>{fingerprints.pending.toLocaleString()} pending</span>
{/if}
{#if fingerprints.rejected > 0}
<span class="text-text-muted">·</span>
<span
class="cursor-help"
title="The fingerprint tools could not read these files. Each is tried again when its file changes."
>
{fingerprints.rejected.toLocaleString()} unreadable
</span>
{/if}
</div>
{/if}
</section>
<!-- Cover art bulk refetch -->