M400: acoustic duplicate detection, history-preserving merge, and fingerprinting settings #134

Merged
bvandeusen merged 8 commits from dev into main 2026-09-11 20:56:56 -04:00
6 changed files with 143 additions and 136 deletions
Showing only changes of commit c8bf9dc929 - Show all commits
+16 -19
View File
@@ -148,39 +148,36 @@ func (q *Queries) DeleteTrack(ctx context.Context, id pgtype.UUID) (DeleteTrackR
return i, err
}
const findMissingTrackByFingerprint = `-- name: FindMissingTrackByFingerprint :many
SELECT id, file_path FROM tracks
WHERE missing_since IS NOT NULL
AND file_size = $1
AND duration_ms = $2
const findMissingTrackByAudioHash = `-- name: FindMissingTrackByAudioHash :many
SELECT t.id, t.file_path
FROM tracks t
JOIN track_fingerprints f ON f.track_id = t.id
WHERE t.missing_since IS NOT NULL
AND f.audio_stream_sha256 = $1
LIMIT 2
`
type FindMissingTrackByFingerprintParams struct {
FileSize int64
DurationMs int32
}
type FindMissingTrackByFingerprintRow struct {
type FindMissingTrackByAudioHashRow struct {
ID pgtype.UUID
FilePath string
}
// Move detection fallback for files with no MBID (#2528). Exact byte size AND
// exact decoded duration is a strong pair: a plain move or rename preserves
// both, while a re-encode changes at least one — and a re-encode genuinely is a
// different file, so failing to match there is correct rather than a gap.
// Move detection fallback for files with no MBID (#2528, #3914). The audio stream
// hash identifies the encoded audio itself, so it survives a rename, a move and a
// retag — anything short of a re-encode. It replaced (file_size, duration_ms),
// which could pair two unrelated files that happened to share a byte count and a
// duration, and missed a file retagged in place, whose size changes.
//
// Same missing-only constraint and same LIMIT 2 rationale as the MBID variant.
func (q *Queries) FindMissingTrackByFingerprint(ctx context.Context, arg FindMissingTrackByFingerprintParams) ([]FindMissingTrackByFingerprintRow, error) {
rows, err := q.db.Query(ctx, findMissingTrackByFingerprint, arg.FileSize, arg.DurationMs)
func (q *Queries) FindMissingTrackByAudioHash(ctx context.Context, audioStreamSha256 []byte) ([]FindMissingTrackByAudioHashRow, error) {
rows, err := q.db.Query(ctx, findMissingTrackByAudioHash, audioStreamSha256)
if err != nil {
return nil, err
}
defer rows.Close()
var items []FindMissingTrackByFingerprintRow
var items []FindMissingTrackByAudioHashRow
for rows.Next() {
var i FindMissingTrackByFingerprintRow
var i FindMissingTrackByAudioHashRow
if err := rows.Scan(&i.ID, &i.FilePath); err != nil {
return nil, err
}
+11 -9
View File
@@ -155,17 +155,19 @@ SELECT id, file_path FROM tracks
AND mbid = sqlc.arg(mbid)::text
LIMIT 2;
-- name: FindMissingTrackByFingerprint :many
-- Move detection fallback for files with no MBID (#2528). Exact byte size AND
-- exact decoded duration is a strong pair: a plain move or rename preserves
-- both, while a re-encode changes at least one — and a re-encode genuinely is a
-- different file, so failing to match there is correct rather than a gap.
-- name: FindMissingTrackByAudioHash :many
-- Move detection fallback for files with no MBID (#2528, #3914). The audio stream
-- hash identifies the encoded audio itself, so it survives a rename, a move and a
-- retag — anything short of a re-encode. It replaced (file_size, duration_ms),
-- which could pair two unrelated files that happened to share a byte count and a
-- duration, and missed a file retagged in place, whose size changes.
--
-- Same missing-only constraint and same LIMIT 2 rationale as the MBID variant.
SELECT id, file_path FROM tracks
WHERE missing_since IS NOT NULL
AND file_size = sqlc.arg(file_size)
AND duration_ms = sqlc.arg(duration_ms)
SELECT t.id, t.file_path
FROM tracks t
JOIN track_fingerprints f ON f.track_id = t.id
WHERE t.missing_since IS NOT NULL
AND f.audio_stream_sha256 = sqlc.arg(audio_stream_sha256)
LIMIT 2;
-- name: AdoptTrackPath :execrows
+13 -15
View File
@@ -41,7 +41,7 @@ import (
// match/ambiguity logic can be tested against a fake.
type trackAdopter interface {
FindMissingTrackByMbid(ctx context.Context, mbid string) ([]dbq.FindMissingTrackByMbidRow, error)
FindMissingTrackByFingerprint(ctx context.Context, arg dbq.FindMissingTrackByFingerprintParams) ([]dbq.FindMissingTrackByFingerprintRow, error)
FindMissingTrackByAudioHash(ctx context.Context, audioStreamSha256 []byte) ([]dbq.FindMissingTrackByAudioHashRow, error)
AdoptTrackPath(ctx context.Context, arg dbq.AdoptTrackPathParams) (int64, error)
}
@@ -53,10 +53,10 @@ type trackAdopter interface {
// pre-#2528 behaviour.
func (s *Scanner) adoptMovedTrack(
ctx context.Context, q trackAdopter, newPath string,
fileSize int64, durationMs int32, recordingMBID string,
audioHash []byte, recordingMBID string,
) bool {
// MBID first. It identifies the recording rather than the bytes, so it
// survives a re-encode that the fingerprint cannot.
// survives a re-encode that the audio hash cannot.
if recordingMBID != "" {
rows, err := q.FindMissingTrackByMbid(ctx, recordingMBID)
if err != nil {
@@ -67,19 +67,17 @@ func (s *Scanner) adoptMovedTrack(
}
}
// Fingerprint fallback for untagged files. Both components must be real:
// duration_ms is 0 when ffprobe failed, and matching 0 against 0 would pair
// up unrelated broken files.
if fileSize > 0 && durationMs > 0 {
rows, err := q.FindMissingTrackByFingerprint(ctx, dbq.FindMissingTrackByFingerprintParams{
FileSize: fileSize,
DurationMs: durationMs,
})
// Audio-hash fallback for untagged files (#3914): the encoded audio itself,
// which a rename, a move or a retag leaves unchanged. Absent when the hash
// could not be taken, and then nothing is matched — an empty hash must never
// be looked up, or every unhashable file would pair with every other.
if len(audioHash) > 0 {
rows, err := q.FindMissingTrackByAudioHash(ctx, audioHash)
if err != nil {
s.logger.Warn("library scan: move lookup by fingerprint failed",
s.logger.Warn("library scan: move lookup by audio hash failed",
"path", newPath, "err", err)
} else if c, ok := s.uniqueMatch(rowsFromFingerprint(rows), newPath, "fingerprint"); ok {
return s.adopt(ctx, q, c, newPath, "fingerprint")
} else if c, ok := s.uniqueMatch(rowsFromAudioHash(rows), newPath, "audio_hash"); ok {
return s.adopt(ctx, q, c, newPath, "audio_hash")
}
}
@@ -100,7 +98,7 @@ func rowsFromMbid(rows []dbq.FindMissingTrackByMbidRow) []candidate {
return out
}
func rowsFromFingerprint(rows []dbq.FindMissingTrackByFingerprintRow) []candidate {
func rowsFromAudioHash(rows []dbq.FindMissingTrackByAudioHashRow) []candidate {
out := make([]candidate, 0, len(rows))
for _, r := range rows {
out = append(out, candidate{id: r.ID, filePath: r.FilePath})
+87 -77
View File
@@ -1,6 +1,7 @@
package library
import (
"bytes"
"context"
"errors"
"testing"
@@ -10,17 +11,20 @@ import (
"git.fabledsword.com/bvandeusen/minstrel/internal/db/dbq"
)
// fakeAdopter answers the audio-hash lookup only for the hash it holds, the way
// the real query does. A fake that returned its rows for any hash could not tell
// adoption by identity apart from adoption by coincidence.
type fakeAdopter struct {
byMbid []dbq.FindMissingTrackByMbidRow
byFingerprint []dbq.FindMissingTrackByFingerprintRow
hash []byte
byHash []dbq.FindMissingTrackByAudioHashRow
mbidErr error
fingerprintErr error
hashErr error
adoptErr error
adoptRows int64
mbidQueried []string
fingerprintQueried []dbq.FindMissingTrackByFingerprintParams
hashQueried [][]byte
adopted []dbq.AdoptTrackPathParams
}
@@ -29,11 +33,17 @@ func (f *fakeAdopter) FindMissingTrackByMbid(_ context.Context, mbid string) ([]
return f.byMbid, f.mbidErr
}
func (f *fakeAdopter) FindMissingTrackByFingerprint(
_ context.Context, arg dbq.FindMissingTrackByFingerprintParams,
) ([]dbq.FindMissingTrackByFingerprintRow, error) {
f.fingerprintQueried = append(f.fingerprintQueried, arg)
return f.byFingerprint, f.fingerprintErr
func (f *fakeAdopter) FindMissingTrackByAudioHash(
_ context.Context, audioStreamSha256 []byte,
) ([]dbq.FindMissingTrackByAudioHashRow, error) {
f.hashQueried = append(f.hashQueried, audioStreamSha256)
if f.hashErr != nil {
return nil, f.hashErr
}
if !bytes.Equal(audioStreamSha256, f.hash) {
return nil, nil
}
return f.byHash, nil
}
func (f *fakeAdopter) AdoptTrackPath(_ context.Context, arg dbq.AdoptTrackPathParams) (int64, error) {
@@ -51,10 +61,12 @@ func mbidRow(n byte, path string) dbq.FindMissingTrackByMbidRow {
return dbq.FindMissingTrackByMbidRow{ID: testUUID(n), FilePath: path}
}
func fpRow(n byte, path string) dbq.FindMissingTrackByFingerprintRow {
return dbq.FindMissingTrackByFingerprintRow{ID: testUUID(n), FilePath: path}
func hashRow(n byte, path string) dbq.FindMissingTrackByAudioHashRow {
return dbq.FindMissingTrackByAudioHashRow{ID: testUUID(n), FilePath: path}
}
func audioHash(b byte) []byte { return bytes.Repeat([]byte{b}, 32) }
const (
oldPath = "/music/Linkin Park/Minutes to Midnight/02 - Bleed It Out.mp3"
newPath = "/music/Linkin Park/Minutes to Midnight/04 - Bleed It Out.mp3"
@@ -64,47 +76,53 @@ func TestAdoptMovedTrack_MatchesByMbid(t *testing.T) {
s := testScanner(t)
q := &fakeAdopter{byMbid: []dbq.FindMissingTrackByMbidRow{mbidRow(7, oldPath)}, adoptRows: 1}
if !s.adoptMovedTrack(context.Background(), q, newPath, 5_000_000, 200_000, "rec-mbid") {
if !s.adoptMovedTrack(context.Background(), q, newPath, audioHash(1), "rec-mbid") {
t.Fatal("expected the moved track to be adopted")
}
if len(q.adopted) != 1 {
t.Fatalf("adopted %d rows, want 1", len(q.adopted))
}
if q.adopted[0].ID != testUUID(7) {
t.Errorf("adopted the wrong row: %v", q.adopted[0].ID)
}
if q.adopted[0].FilePath != newPath {
t.Errorf("adopted FilePath = %q, want %q", q.adopted[0].FilePath, newPath)
if len(q.adopted) != 1 || q.adopted[0].ID != testUUID(7) || q.adopted[0].FilePath != newPath {
t.Fatalf("adopted = %+v, want row 7 at %q", q.adopted, newPath)
}
// MBID matched, so the weaker signal should not have been consulted.
if len(q.fingerprintQueried) != 0 {
t.Errorf("queried the fingerprint despite an MBID match")
if len(q.hashQueried) != 0 {
t.Errorf("queried the audio hash despite an MBID match")
}
}
func TestAdoptMovedTrack_FallsBackToFingerprint(t *testing.T) {
func TestAdoptMovedTrack_FallsBackToAudioHash(t *testing.T) {
s := testScanner(t)
q := &fakeAdopter{byFingerprint: []dbq.FindMissingTrackByFingerprintRow{fpRow(3, oldPath)}, adoptRows: 1}
q := &fakeAdopter{hash: audioHash(3), byHash: []dbq.FindMissingTrackByAudioHashRow{hashRow(3, oldPath)}, adoptRows: 1}
// No MBID: an untagged file, which is exactly what the fallback is for.
if !s.adoptMovedTrack(context.Background(), q, newPath, 4_200_000, 187_000, "") {
t.Fatal("expected adoption via fingerprint")
if !s.adoptMovedTrack(context.Background(), q, newPath, audioHash(3), "") {
t.Fatal("expected adoption via the audio hash")
}
if len(q.mbidQueried) != 0 {
t.Errorf("queried by MBID with no MBID available")
}
if len(q.fingerprintQueried) != 1 {
t.Fatalf("fingerprint queried %d times, want 1", len(q.fingerprintQueried))
}
got := q.fingerprintQueried[0]
if got.FileSize != 4_200_000 || got.DurationMs != 187_000 {
t.Errorf("fingerprint = %+v, want size 4200000 duration 187000", got)
if len(q.hashQueried) != 1 || !bytes.Equal(q.hashQueried[0], audioHash(3)) {
t.Fatalf("hash queried = %x, want exactly the file's hash", q.hashQueried)
}
if len(q.adopted) != 1 || q.adopted[0].ID != testUUID(3) {
t.Errorf("adopted = %+v, want row 3", q.adopted)
}
}
// The case the old (file_size, duration_ms) pair got wrong: an unrelated file
// that happens to share a size and a duration with a missing track. Size and
// duration are no longer inputs at all; only the audio itself can match, and a
// different recording has a different hash.
func TestAdoptMovedTrack_DifferentAudioIsNotAdopted(t *testing.T) {
s := testScanner(t)
q := &fakeAdopter{hash: audioHash(0xAA), byHash: []dbq.FindMissingTrackByAudioHashRow{hashRow(1, oldPath)}, adoptRows: 1}
if s.adoptMovedTrack(context.Background(), q, newPath, audioHash(0xBB), "") {
t.Fatal("adopted a missing track whose audio differs")
}
if len(q.adopted) != 0 {
t.Errorf("adopted = %+v, want none", q.adopted)
}
}
// Two missing rows carrying the same recording MBID means real duplicates.
// Adopting one arbitrarily would attach this file's future history to a coin
// flip, so it must insert fresh instead.
@@ -115,7 +133,7 @@ func TestAdoptMovedTrack_RefusesAmbiguousMbidMatch(t *testing.T) {
mbidRow(2, "/music/b.mp3"),
}, adoptRows: 1}
if s.adoptMovedTrack(context.Background(), q, newPath, 0, 0, "rec-mbid") {
if s.adoptMovedTrack(context.Background(), q, newPath, nil, "rec-mbid") {
t.Fatal("expected refusal on an ambiguous MBID match")
}
if len(q.adopted) != 0 {
@@ -123,66 +141,57 @@ func TestAdoptMovedTrack_RefusesAmbiguousMbidMatch(t *testing.T) {
}
}
// An ambiguous MBID may still be resolvable by the fingerprint, which is a
// An ambiguous MBID may still be resolvable by the audio hash, which is a
// narrower signal — so falling through is allowed to succeed.
func TestAdoptMovedTrack_AmbiguousMbidFallsThroughToFingerprint(t *testing.T) {
func TestAdoptMovedTrack_AmbiguousMbidFallsThroughToAudioHash(t *testing.T) {
s := testScanner(t)
q := &fakeAdopter{
byMbid: []dbq.FindMissingTrackByMbidRow{
mbidRow(1, "/music/a.mp3"),
mbidRow(2, "/music/b.mp3"),
},
byFingerprint: []dbq.FindMissingTrackByFingerprintRow{fpRow(2, "/music/b.mp3")},
hash: audioHash(2),
byHash: []dbq.FindMissingTrackByAudioHashRow{hashRow(2, "/music/b.mp3")},
adoptRows: 1,
}
if !s.adoptMovedTrack(context.Background(), q, newPath, 1000, 2000, "rec-mbid") {
t.Fatal("expected the fingerprint to disambiguate")
if !s.adoptMovedTrack(context.Background(), q, newPath, audioHash(2), "rec-mbid") {
t.Fatal("expected the audio hash to disambiguate")
}
if len(q.adopted) != 1 || q.adopted[0].ID != testUUID(2) {
t.Errorf("adopted = %+v, want row 2", q.adopted)
}
}
func TestAdoptMovedTrack_RefusesAmbiguousFingerprintMatch(t *testing.T) {
// Two missing tracks with identical audio are duplicates of each other; a new
// file matching both cannot be assigned to either.
func TestAdoptMovedTrack_RefusesAmbiguousAudioHashMatch(t *testing.T) {
s := testScanner(t)
q := &fakeAdopter{byFingerprint: []dbq.FindMissingTrackByFingerprintRow{
fpRow(1, "/music/a.mp3"),
fpRow(2, "/music/b.mp3"),
q := &fakeAdopter{hash: audioHash(5), byHash: []dbq.FindMissingTrackByAudioHashRow{
hashRow(1, "/music/a.mp3"),
hashRow(2, "/music/b.mp3"),
}, adoptRows: 1}
if s.adoptMovedTrack(context.Background(), q, newPath, 1000, 2000, "") {
t.Fatal("expected refusal on an ambiguous fingerprint match")
if s.adoptMovedTrack(context.Background(), q, newPath, audioHash(5), "") {
t.Fatal("expected refusal on an ambiguous audio-hash match")
}
if len(q.adopted) != 0 {
t.Errorf("adopted despite ambiguity: %+v", q.adopted)
}
}
// duration_ms is 0 when ffprobe failed. Matching 0 against 0 would pair up
// unrelated broken files, so the fingerprint must not be attempted.
func TestAdoptMovedTrack_SkipsFingerprintWithoutRealValues(t *testing.T) {
tests := []struct {
name string
size int64
duration int32
}{
{"no duration", 1000, 0},
{"no size", 0, 2000},
{"neither", 0, 0},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
// No hash means it could not be taken. An empty hash must never be looked up:
// every unhashable file would pair with every other.
func TestAdoptMovedTrack_SkipsAudioHashWhenAbsent(t *testing.T) {
for name, hash := range map[string][]byte{"nil": nil, "empty": {}} {
t.Run(name, func(t *testing.T) {
s := testScanner(t)
q := &fakeAdopter{
byFingerprint: []dbq.FindMissingTrackByFingerprintRow{fpRow(1, oldPath)},
adoptRows: 1,
q := &fakeAdopter{hash: hash, byHash: []dbq.FindMissingTrackByAudioHashRow{hashRow(1, oldPath)}, adoptRows: 1}
if s.adoptMovedTrack(context.Background(), q, newPath, hash, "") {
t.Error("adopted without an audio hash")
}
if s.adoptMovedTrack(context.Background(), q, newPath, tc.size, tc.duration, "") {
t.Error("adopted on an unusable fingerprint")
}
if len(q.fingerprintQueried) != 0 {
t.Error("queried the fingerprint with unusable values")
if len(q.hashQueried) != 0 {
t.Error("looked up an absent audio hash")
}
})
}
@@ -192,7 +201,7 @@ func TestAdoptMovedTrack_NoCandidates(t *testing.T) {
s := testScanner(t)
q := &fakeAdopter{adoptRows: 1}
if s.adoptMovedTrack(context.Background(), q, newPath, 1000, 2000, "rec-mbid") {
if s.adoptMovedTrack(context.Background(), q, newPath, audioHash(9), "rec-mbid") {
t.Fatal("expected no adoption when nothing matches")
}
if len(q.adopted) != 0 {
@@ -209,7 +218,7 @@ func TestAdoptMovedTrack_LostRaceReportsNotAdopted(t *testing.T) {
adoptRows: 0,
}
if s.adoptMovedTrack(context.Background(), q, newPath, 1000, 2000, "rec-mbid") {
if s.adoptMovedTrack(context.Background(), q, newPath, audioHash(5), "rec-mbid") {
t.Fatal("expected not-adopted when the update matched no rows")
}
}
@@ -223,7 +232,7 @@ func TestAdoptMovedTrack_ToleratesQueryErrors(t *testing.T) {
q *fakeAdopter
}{
{"mbid lookup fails", &fakeAdopter{mbidErr: sentinel}},
{"fingerprint lookup fails", &fakeAdopter{fingerprintErr: sentinel}},
{"audio hash lookup fails", &fakeAdopter{hashErr: sentinel}},
{"adopt fails", &fakeAdopter{
byMbid: []dbq.FindMissingTrackByMbidRow{mbidRow(1, oldPath)},
adoptErr: sentinel,
@@ -232,24 +241,25 @@ func TestAdoptMovedTrack_ToleratesQueryErrors(t *testing.T) {
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
s := testScanner(t)
if s.adoptMovedTrack(context.Background(), tc.q, newPath, 1000, 2000, "rec-mbid") {
if s.adoptMovedTrack(context.Background(), tc.q, newPath, audioHash(1), "rec-mbid") {
t.Error("reported adoption despite a query error")
}
})
}
}
// A failed MBID lookup must not stop the fingerprint from being tried.
func TestAdoptMovedTrack_MbidErrorStillTriesFingerprint(t *testing.T) {
// A failed MBID lookup must not stop the audio hash from being tried.
func TestAdoptMovedTrack_MbidErrorStillTriesAudioHash(t *testing.T) {
s := testScanner(t)
q := &fakeAdopter{
mbidErr: errors.New("db hiccup"),
byFingerprint: []dbq.FindMissingTrackByFingerprintRow{fpRow(9, oldPath)},
hash: audioHash(9),
byHash: []dbq.FindMissingTrackByAudioHashRow{hashRow(9, oldPath)},
adoptRows: 1,
}
if !s.adoptMovedTrack(context.Background(), q, newPath, 1000, 2000, "rec-mbid") {
t.Fatal("expected the fingerprint to be tried after an MBID lookup error")
if !s.adoptMovedTrack(context.Background(), q, newPath, audioHash(9), "rec-mbid") {
t.Fatal("expected the audio hash to be tried after an MBID lookup error")
}
if len(q.adopted) != 1 || q.adopted[0].ID != testUUID(9) {
t.Errorf("adopted = %+v, want row 9", q.adopted)
@@ -280,9 +290,9 @@ func TestRowConverters(t *testing.T) {
if len(got) != 2 || got[0].id != testUUID(1) || got[1].filePath != "/b" {
t.Errorf("rowsFromMbid = %+v", got)
}
got = rowsFromFingerprint([]dbq.FindMissingTrackByFingerprintRow{fpRow(3, "/c")})
got = rowsFromAudioHash([]dbq.FindMissingTrackByAudioHashRow{hashRow(3, "/c")})
if len(got) != 1 || got[0].id != testUUID(3) || got[0].filePath != "/c" {
t.Errorf("rowsFromFingerprint = %+v", got)
t.Errorf("rowsFromAudioHash = %+v", got)
}
}
+4 -4
View File
@@ -325,11 +325,11 @@ func (s *Scanner) scanFile(
// file_path and updates THAT row: same track id, likes and play history
// intact. Without this, renumbering an album forks every track on it.
//
// Runs here rather than earlier because the fingerprint needs the probed
// duration, and only for genuinely unknown paths — a known path is already
// the row we're going to update.
// Runs after fingerprinting because, for a file with no MBID, adoption matches
// on its audio hash (#3914), and only for genuinely unknown paths — a known
// path is already the row we're going to update.
if !knownTrack {
if s.adoptMovedTrack(ctx, q, path, info.Size(), durationMs, recordingMBID) {
if s.adoptMovedTrack(ctx, q, path, fp.streamSHA256, recordingMBID) {
// Count it as an update: the row existed, and reporting it as Added
// would overstate library growth on every reorganisation.
knownTrack = true
+3 -3
View File
@@ -211,9 +211,9 @@ func writeTestMP3(t *testing.T, path string, frames map[string]string) {
// playlist memberships travel with it — rather than forking into a marked ghost
// plus a fresh zero-history row.
//
// Uses the MBID path. The synthetic MP3s here carry no real audio, so ffprobe
// yields duration 0 and the size+duration fingerprint is deliberately unusable —
// which is why the recording MBID is the signal under test.
// Uses the MBID path. The synthetic MP3s here carry no real audio, so their audio
// stream hash cannot be relied on — which is why the recording MBID is the signal
// under test. The audio-hash path is covered by moved_test.go.
//
// Eight tracks with one rename keeps the marked fraction at 12.5%, under
// missingMarkMaxFraction. That is load-bearing: if the rename exceeded the cap,