test-web / test (push) Failing after 50s
test-go / test (push) Successful in 1m7s
test-go / integration (push) Successful in 3m27s
release / Build + push container image (push) Canceled after 0s
release / Verify release artifacts (tag releases only) (push) Canceled after 0s
release / Build signed APK (releases and dev) (push) Canceled after 4m23s
The scan fingerprints only bytes it has not seen, so everything imported before fingerprinting existed, and any row derived by an older fingerprintVersion, needs a pass of its own. That pass is a background worker, not a stage in RunScan. RunScan runs at boot and then every 12h, and an in-flight scan older than an hour is reaped and a second started beside it. A stage would have to stop inside the hour: a few hundred decodes a run, so about a month for a 50k-track library. It would also hold the run in flight and answer manual rescans with 409 while it worked. FingerprintBackfillWorker runs once at start, then hourly. Nothing a pass does (error or panic) can stop the next tick. A pass walks tracks with no fingerprint or a stale version, skipping missing tracks, keyset-paged on id. The cursor is what lets a pass end: an inconclusive attempt writes no row, so a file that keeps timing out would otherwise be re-listed and retried forever. Two decodes at a time, deliberately: they compete with transcoding for CPU and with streaming for the mount. storeFingerprint is now one package function shared by the scan and the worker, and reports whether the attempt was fingerprinted, rejected, inconclusive or failed to store. Progress is a live gauge on the Admin scan card, served by GET /api/admin/library/fingerprints: fingerprinted / rejected / pending of total, with missing tracks excluded so it can reach the end. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01SQ31KQpYbStyK5y58UmPLH
149 lines
4.7 KiB
Go
149 lines
4.7 KiB
Go
// Code generated by sqlc. DO NOT EDIT.
|
|
// versions:
|
|
// sqlc v1.31.1
|
|
// source: fingerprints.sql
|
|
|
|
package dbq
|
|
|
|
import (
|
|
"context"
|
|
|
|
"github.com/jackc/pgx/v5/pgtype"
|
|
)
|
|
|
|
const deleteTrackFingerprint = `-- name: DeleteTrackFingerprint :exec
|
|
DELETE FROM track_fingerprints WHERE track_id = $1
|
|
`
|
|
|
|
// A file changed but could not be fingerprinted, for a reason unrelated to the
|
|
// file. The stored row describes the OLD bytes, so it goes and the backfill
|
|
// re-derives it — nothing may keep trusting a stale identity.
|
|
func (q *Queries) DeleteTrackFingerprint(ctx context.Context, trackID pgtype.UUID) error {
|
|
_, err := q.db.Exec(ctx, deleteTrackFingerprint, trackID)
|
|
return err
|
|
}
|
|
|
|
const getFingerprintCoverage = `-- name: GetFingerprintCoverage :one
|
|
SELECT count(*)::bigint AS total,
|
|
count(*) FILTER (
|
|
WHERE f.fingerprint_version >= $1
|
|
AND f.audio_stream_sha256 IS NOT NULL AND f.chromaprint IS NOT NULL
|
|
)::bigint AS fingerprinted,
|
|
count(*) FILTER (
|
|
WHERE f.fingerprint_version >= $1
|
|
AND (f.audio_stream_sha256 IS NULL OR f.chromaprint IS NULL)
|
|
)::bigint AS rejected,
|
|
count(*) FILTER (
|
|
WHERE f.track_id IS NULL OR f.fingerprint_version < $1
|
|
)::bigint AS pending
|
|
FROM tracks t
|
|
LEFT JOIN track_fingerprints f ON f.track_id = t.id
|
|
WHERE t.missing_since IS NULL
|
|
`
|
|
|
|
type GetFingerprintCoverageRow struct {
|
|
Total int64
|
|
Fingerprinted int64
|
|
Rejected int64
|
|
Pending int64
|
|
}
|
|
|
|
// The admin gauge for the backfill. fingerprinted + rejected + pending = total.
|
|
// rejected is a row at the current version with a NULL half: a tool ran and
|
|
// refused the file, which is settled rather than waiting. Missing tracks are
|
|
// excluded, or the gauge could never reach the end.
|
|
func (q *Queries) GetFingerprintCoverage(ctx context.Context, currentVersion int16) (GetFingerprintCoverageRow, error) {
|
|
row := q.db.QueryRow(ctx, getFingerprintCoverage, currentVersion)
|
|
var i GetFingerprintCoverageRow
|
|
err := row.Scan(
|
|
&i.Total,
|
|
&i.Fingerprinted,
|
|
&i.Rejected,
|
|
&i.Pending,
|
|
)
|
|
return i, err
|
|
}
|
|
|
|
const listTracksNeedingFingerprint = `-- name: ListTracksNeedingFingerprint :many
|
|
SELECT t.id, t.file_path
|
|
FROM tracks t
|
|
LEFT JOIN track_fingerprints f ON f.track_id = t.id
|
|
WHERE t.missing_since IS NULL
|
|
AND (f.track_id IS NULL OR f.fingerprint_version < $1)
|
|
AND t.id > $2
|
|
ORDER BY t.id
|
|
LIMIT $3
|
|
`
|
|
|
|
type ListTracksNeedingFingerprintParams struct {
|
|
CurrentVersion int16
|
|
AfterID pgtype.UUID
|
|
BatchLimit int32
|
|
}
|
|
|
|
type ListTracksNeedingFingerprintRow struct {
|
|
ID pgtype.UUID
|
|
FilePath string
|
|
}
|
|
|
|
// The backfill's work queue (#3908): tracks with no fingerprint, or one derived
|
|
// by an older method. Keyset-paged on id so a pass visits each track at most
|
|
// once. That cursor is load-bearing: an inconclusive attempt writes no row, so
|
|
// without it a file that keeps timing out would be listed again straight away
|
|
// and retried in a tight loop. Missing tracks are skipped — there is no file to
|
|
// read.
|
|
func (q *Queries) ListTracksNeedingFingerprint(ctx context.Context, arg ListTracksNeedingFingerprintParams) ([]ListTracksNeedingFingerprintRow, error) {
|
|
rows, err := q.db.Query(ctx, listTracksNeedingFingerprint, arg.CurrentVersion, arg.AfterID, arg.BatchLimit)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
defer rows.Close()
|
|
var items []ListTracksNeedingFingerprintRow
|
|
for rows.Next() {
|
|
var i ListTracksNeedingFingerprintRow
|
|
if err := rows.Scan(&i.ID, &i.FilePath); err != nil {
|
|
return nil, err
|
|
}
|
|
items = append(items, i)
|
|
}
|
|
if err := rows.Err(); err != nil {
|
|
return nil, err
|
|
}
|
|
return items, nil
|
|
}
|
|
|
|
const upsertTrackFingerprint = `-- name: UpsertTrackFingerprint :exec
|
|
INSERT INTO track_fingerprints (
|
|
track_id, audio_stream_sha256, chromaprint, fingerprint_version
|
|
) VALUES (
|
|
$1, $2, $3,
|
|
$4
|
|
)
|
|
ON CONFLICT (track_id) DO UPDATE SET
|
|
audio_stream_sha256 = EXCLUDED.audio_stream_sha256,
|
|
chromaprint = EXCLUDED.chromaprint,
|
|
fingerprint_version = EXCLUDED.fingerprint_version,
|
|
computed_at = now()
|
|
`
|
|
|
|
type UpsertTrackFingerprintParams struct {
|
|
TrackID pgtype.UUID
|
|
AudioStreamSha256 []byte
|
|
Chromaprint []int32
|
|
FingerprintVersion int16
|
|
}
|
|
|
|
// Written whenever a track's fingerprint is derived: by the scan when a file is
|
|
// new or its bytes changed, and by the backfill (#3908) for rows derived by an
|
|
// older method. Replaces the row wholesale — a fingerprint of the old bytes has
|
|
// no standing once the file has changed.
|
|
func (q *Queries) UpsertTrackFingerprint(ctx context.Context, arg UpsertTrackFingerprintParams) error {
|
|
_, err := q.db.Exec(ctx, upsertTrackFingerprint,
|
|
arg.TrackID,
|
|
arg.AudioStreamSha256,
|
|
arg.Chromaprint,
|
|
arg.FingerprintVersion,
|
|
)
|
|
return err
|
|
}
|