feat(library): backfill fingerprints for the existing library — M400 #3908
test-web / test (push) Failing after 50s
test-go / test (push) Successful in 1m7s
test-go / integration (push) Successful in 3m27s
release / Build + push container image (push) Canceled after 0s
release / Verify release artifacts (tag releases only) (push) Canceled after 0s
release / Build signed APK (releases and dev) (push) Canceled after 4m23s
test-web / test (push) Failing after 50s
test-go / test (push) Successful in 1m7s
test-go / integration (push) Successful in 3m27s
release / Build + push container image (push) Canceled after 0s
release / Verify release artifacts (tag releases only) (push) Canceled after 0s
release / Build signed APK (releases and dev) (push) Canceled after 4m23s
The scan fingerprints only bytes it has not seen, so everything imported before fingerprinting existed, and any row derived by an older fingerprintVersion, needs a pass of its own. That pass is a background worker, not a stage in RunScan. RunScan runs at boot and then every 12h, and an in-flight scan older than an hour is reaped and a second started beside it. A stage would have to stop inside the hour: a few hundred decodes a run, so about a month for a 50k-track library. It would also hold the run in flight and answer manual rescans with 409 while it worked. FingerprintBackfillWorker runs once at start, then hourly. Nothing a pass does (error or panic) can stop the next tick. A pass walks tracks with no fingerprint or a stale version, skipping missing tracks, keyset-paged on id. The cursor is what lets a pass end: an inconclusive attempt writes no row, so a file that keeps timing out would otherwise be re-listed and retried forever. Two decodes at a time, deliberately: they compete with transcoding for CPU and with streaming for the mount. storeFingerprint is now one package function shared by the scan and the worker, and reports whether the attempt was fingerprinted, rejected, inconclusive or failed to store. Progress is a live gauge on the Admin scan card, served by GET /api/admin/library/fingerprints: fingerprinted / rejected / pending of total, with missing tracks excluded so it can reach the end. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01SQ31KQpYbStyK5y58UmPLH
This commit is contained in:
@@ -0,0 +1,32 @@
|
||||
import { describe, it, expect, vi, beforeEach } from 'vitest';
|
||||
import { getFingerprintCoverage, type FingerprintCoverage } from './admin';
|
||||
|
||||
vi.mock('./client', () => ({
|
||||
api: { get: vi.fn(), post: vi.fn() }
|
||||
}));
|
||||
|
||||
import { api } from './client';
|
||||
|
||||
describe('admin fingerprint coverage API', () => {
|
||||
beforeEach(() => vi.clearAllMocks());
|
||||
|
||||
it('getFingerprintCoverage GETs the correct path', async () => {
|
||||
const sample: FingerprintCoverage = {
|
||||
total: 18026,
|
||||
fingerprinted: 9400,
|
||||
rejected: 12,
|
||||
pending: 8614
|
||||
};
|
||||
(api.get as unknown as ReturnType<typeof vi.fn>).mockResolvedValueOnce(sample);
|
||||
const got = await getFingerprintCoverage();
|
||||
expect(api.get).toHaveBeenCalledWith('/api/admin/library/fingerprints');
|
||||
expect(got).toEqual(sample);
|
||||
});
|
||||
|
||||
it('buckets sum to the total', async () => {
|
||||
const sample: FingerprintCoverage = { total: 10, fingerprinted: 6, rejected: 1, pending: 3 };
|
||||
(api.get as unknown as ReturnType<typeof vi.fn>).mockResolvedValueOnce(sample);
|
||||
const got = await getFingerprintCoverage();
|
||||
expect(got.fingerprinted + got.rejected + got.pending).toBe(got.total);
|
||||
});
|
||||
});
|
||||
@@ -313,6 +313,30 @@ export function createCoverageQuery() {
|
||||
});
|
||||
}
|
||||
|
||||
// Fingerprint backfill (#3908) --------------------------------------------
|
||||
|
||||
export type FingerprintCoverage = {
|
||||
total: number;
|
||||
fingerprinted: number;
|
||||
rejected: number;
|
||||
pending: number;
|
||||
};
|
||||
|
||||
export async function getFingerprintCoverage(): Promise<FingerprintCoverage> {
|
||||
return api.get<FingerprintCoverage>('/api/admin/library/fingerprints');
|
||||
}
|
||||
|
||||
// Polled far less often than the cover gauge: the backfill decodes files two at
|
||||
// a time, so the count moves by a few tracks a minute and a 3s poll is noise.
|
||||
export function createFingerprintCoverageQuery() {
|
||||
return createQuery({
|
||||
queryKey: qk.fingerprintCoverage(),
|
||||
queryFn: getFingerprintCoverage,
|
||||
staleTime: 30_000,
|
||||
refetchInterval: 30_000
|
||||
});
|
||||
}
|
||||
|
||||
// Cover-art providers ------------------------------------------------------
|
||||
|
||||
export type CoverProviderCapability = 'album_cover' | 'artist_thumb' | 'artist_fanart';
|
||||
|
||||
@@ -47,6 +47,7 @@ export const qk = {
|
||||
['adminPlaybackErrors', { resolved: resolved ?? false }] as const,
|
||||
scanStatus: () => ['scanStatus'] as const,
|
||||
coverage: () => ['coverage'] as const,
|
||||
fingerprintCoverage: () => ['fingerprintCoverage'] as const,
|
||||
coverProviders: () => ['coverProviders'] as const,
|
||||
tagProviders: () => ['tagProviders'] as const,
|
||||
adminUsers: () => ['adminUsers'] as const,
|
||||
|
||||
@@ -10,6 +10,7 @@
|
||||
createAdminQuarantineQuery,
|
||||
createScanStatusQuery,
|
||||
createCoverageQuery,
|
||||
createFingerprintCoverageQuery,
|
||||
approveRequest,
|
||||
rejectRequest,
|
||||
resolveQuarantine,
|
||||
@@ -177,6 +178,11 @@
|
||||
const coverageQ = $derived($coverageStore);
|
||||
const coverage = $derived(coverageQ.data);
|
||||
|
||||
// ---- Fingerprint backfill gauge (#3908) ----
|
||||
const fingerprintStore = $derived(createFingerprintCoverageQuery());
|
||||
const fingerprintQ = $derived($fingerprintStore);
|
||||
const fingerprints = $derived(fingerprintQ.data);
|
||||
|
||||
let triggering = $state(false);
|
||||
let triggerResult = $state<string | null>(null);
|
||||
|
||||
@@ -428,6 +434,28 @@
|
||||
{#if triggerResult}
|
||||
<p class="mt-2 text-sm">{triggerResult}</p>
|
||||
{/if}
|
||||
|
||||
<!-- Fingerprint backfill (#3908). A worker of its own rather than a scan
|
||||
stage, so its progress is read live here, not from the run above. -->
|
||||
{#if fingerprints && fingerprints.total > 0}
|
||||
<div class="mt-3 flex flex-wrap items-center gap-3 text-sm">
|
||||
<span class="text-xs font-medium uppercase tracking-wide text-text-muted">Fingerprints</span>
|
||||
<span>{fingerprints.fingerprinted.toLocaleString()} of {fingerprints.total.toLocaleString()} tracks</span>
|
||||
{#if fingerprints.pending > 0}
|
||||
<span class="text-text-muted">·</span>
|
||||
<span>{fingerprints.pending.toLocaleString()} pending</span>
|
||||
{/if}
|
||||
{#if fingerprints.rejected > 0}
|
||||
<span class="text-text-muted">·</span>
|
||||
<span
|
||||
class="cursor-help"
|
||||
title="The fingerprint tools could not read these files. Each is tried again when its file changes."
|
||||
>
|
||||
{fingerprints.rejected.toLocaleString()} unreadable
|
||||
</span>
|
||||
{/if}
|
||||
</div>
|
||||
{/if}
|
||||
</section>
|
||||
|
||||
<!-- Cover art bulk refetch -->
|
||||
|
||||
Reference in New Issue
Block a user