minstrel/internal/library/acoustid_lookup_test.go

package library

import (
	"context"
	"errors"
	"fmt"
	"io"
	"log/slog"
	"path/filepath"
	"strings"
	"sync"
	"testing"
	"time"

	"github.com/jackc/pgx/v5/pgtype"
	"github.com/jackc/pgx/v5/pgxpool"

	"git.fabledsword.com/bvandeusen/minstrel/internal/acoustid"
	"git.fabledsword.com/bvandeusen/minstrel/internal/db/dbq"
	"git.fabledsword.com/bvandeusen/minstrel/internal/dbtest"
)

func cand(id, title string, sec int, score float64) acoustid.Candidate {
	return acoustid.Candidate{Recording: acoustid.Recording{ID: id, Title: title, DurationSec: sec}, Score: score}
}

// D5, as a table over what decides it: how many candidates clear the
// threshold, and whether title and length leave exactly one of them.
func TestChooseRecording(t *testing.T) {
	const (
		minScore = 0.85
		title    = "Don't Stop"
		fileMs   = 200_400
	)
	cases := []struct {
		name  string
		cands []acoustid.Candidate
		state string
		rec   string
	}{
		{"no candidates", nil, lookupNoMatch, ""},
		{"all below the threshold", []acoustid.Candidate{cand("a", title, 200, 0.84)}, lookupNoMatch, ""},
		{"one above the threshold", []acoustid.Candidate{cand("a", "Other title", 999, 0.9), cand("b", title, 200, 0.4)}, lookupMatched, "a"},
		{"exactly at the threshold counts", []acoustid.Candidate{cand("a", title, 200, 0.85)}, lookupMatched, "a"},
		{"several, one with the title and length",
			[]acoustid.Candidate{cand("album", "Dont stop", 201, 0.95), cand("edit", "Don't Stop (Radio Edit)", 181, 0.95)},
			lookupMatched, "album"},
		{"several, title alike but only one length fits",
			[]acoustid.Candidate{cand("live", title, 260, 0.9), cand("studio", title, 198, 0.9)},
			lookupMatched, "studio"},
		{"several with the same title and length",
			[]acoustid.Candidate{cand("r1", title, 200, 0.95), cand("r2", title, 201, 0.95)},
			lookupAmbiguous, ""},
		{"several, none with the title",
			[]acoustid.Candidate{cand("x", "Something", 200, 0.95), cand("y", "Else", 200, 0.95)},
			lookupAmbiguous, ""},
		// An unknown length rules nothing out, so it cannot break a tie on
		// its own: both stay and the lookup is ambiguous.
		{"unknown lengths do not disambiguate",
			[]acoustid.Candidate{cand("r1", title, 0, 0.95), cand("r2", title, 0, 0.95)},
			lookupAmbiguous, ""},
	}
	for _, c := range cases {
		d := chooseRecording(c.cands, minScore, title, fileMs)
		if d.state != c.state || d.recording != c.rec {
			t.Errorf("%s: got %s %q, want %s %q", c.name, d.state, d.recording, c.state, c.rec)
		}
		if d.candidates != len(c.cands) {
			t.Errorf("%s: candidates = %d, want %d", c.name, d.candidates, len(c.cands))
		}
	}
	if d := chooseRecording([]acoustid.Candidate{cand("a", title, 200, 0.97)}, minScore, title, fileMs); d.bestScore != 0.97 {
		t.Errorf("best score = %v, want 0.97", d.bestScore)
	}
}

func TestNormalizeTitle(t *testing.T) {
	if normalizeTitle("Don't Stop!") != normalizeTitle("dont   stop") {
		t.Error("punctuation and case should not separate titles")
	}
	if normalizeTitle("Song (Radio Edit)") == normalizeTitle("Song") {
		t.Error("an edit's title must not match the original's")
	}
	if normalizeTitle("Ænima") != "ænima" {
		t.Errorf("non-ASCII letters must survive: %q", normalizeTitle("Ænima"))
	}
}

// fakeLookup answers by fingerprint. The worker passes each file's path
// through as its fingerprint (see testWorker), so a test keys answers by file.
type fakeLookup struct {
	mu      sync.Mutex
	answers map[string][]acoustid.Candidate
	err     error
	calls   int
}

func (f *fakeLookup) Lookup(_ context.Context, _ string, fingerprint string, _ int) ([]acoustid.Candidate, error) {
	f.mu.Lock()
	defer f.mu.Unlock()
	f.calls++
	if f.err != nil {
		return nil, f.err
	}
	return f.answers[fingerprint], nil
}

func testWorker(t *testing.T, pool *pgxpool.Pool, client recordingLookup) *AcoustIDLookupWorker {
	t.Helper()
	ctx := context.Background()
	settings, err := NewAcoustIDSettingsService(ctx, pool)
	if err != nil {
		t.Fatal(err)
	}
	key := "test-key"
	if _, err := settings.Set(ctx, AcoustIDSettingsUpdate{Enabled: true, MinScore: 0.85, APIKey: &key}); err != nil {
		t.Fatal(err)
	}
	w := NewAcoustIDLookupWorker(pool, slog.New(slog.NewTextHandler(io.Discard, nil)), settings, client)
	w.fingerprint = func(_ context.Context, path string) (lookupFingerprint, error) {
		return lookupFingerprint{fingerprint: path, durationSec: 200}, nil
	}
	return w
}

func ptr(s string) *string { return &s }

// mbidOf reads a track's MBID and its source.
func mbidOf(t *testing.T, pool *pgxpool.Pool, id pgtype.UUID) (string, string) {
	t.Helper()
	var mbid, source *string
	if err := pool.QueryRow(context.Background(),
		`SELECT mbid, mbid_source FROM tracks WHERE id = $1`, id).Scan(&mbid, &source); err != nil {
		t.Fatal(err)
	}
	deref := func(p *string) string {
		if p == nil {
			return ""
		}
		return *p
	}
	return deref(mbid), deref(source)
}

func addTrack(t *testing.T, q *dbq.Queries, album dbq.Album, artist dbq.Artist, title, path string, mbid *string) dbq.Track {
	t.Helper()
	tr, err := q.UpsertTrack(context.Background(), dbq.UpsertTrackParams{
		Title: title, AlbumID: album.ID, ArtistID: artist.ID, DurationMs: 200_000,
		FilePath: path, FileSize: 100, FileFormat: "mp3", Mbid: mbid,
	})
	if err != nil {
		t.Fatalf("track %s: %v", title, err)
	}
	return tr
}

// D4: the file's tag outranks a lookup, in both directions. Falsified by
// restoring `mbid = EXCLUDED.mbid` in UpsertTrack: the first re-read below
// then erases the looked-up id.
func TestUpsertTrack_KeepsALookedUpMbidUntilATagCarriesOne_Integration(t *testing.T) {
	pool := newPool(t)
	ctx := context.Background()
	q := dbq.New(pool)
	dir := t.TempDir()
	tr, album, artist := seedTrack(t, pool, filepath.Join(dir, "a.mp3"))
	reread := func(mbid *string) {
		t.Helper()
		if _, err := q.UpsertTrack(ctx, dbq.UpsertTrackParams{
			Title: tr.Title, AlbumID: album.ID, ArtistID: artist.ID, DurationMs: 1000,
			FilePath: tr.FilePath, FileSize: 100, FileFormat: "mp3", Mbid: mbid,
		}); err != nil {
			t.Fatal(err)
		}
	}

	if _, err := q.SetTrackMbidFromAcoustID(ctx, dbq.SetTrackMbidFromAcoustIDParams{ID: tr.ID, Mbid: ptr("rec-looked-up")}); err != nil {
		t.Fatal(err)
	}
	reread(nil)
	if m, s := mbidOf(t, pool, tr.ID); m != "rec-looked-up" || s != "acoustid" {
		t.Errorf("after an untagged re-read: %q from %q, want the looked-up id kept", m, s)
	}

	reread(ptr("rec-from-tag"))
	if m, s := mbidOf(t, pool, tr.ID); m != "rec-from-tag" || s != "tag" {
		t.Errorf("after a tagged re-read: %q from %q, want the tag's id", m, s)
	}

	// A tag id is the file's to take away: removing the tag clears it, and a
	// lookup cannot write over one.
	if n, err := q.SetTrackMbidFromAcoustID(ctx, dbq.SetTrackMbidFromAcoustIDParams{ID: tr.ID, Mbid: ptr("rec-other")}); err != nil || n != 0 {
		t.Errorf("a lookup wrote over a tag id: %d rows, %v", n, err)
	}
	reread(nil)
	if m, s := mbidOf(t, pool, tr.ID); m != "" || s != "" {
		t.Errorf("after the tag was removed: %q from %q, want none", m, s)
	}
}

func TestAcoustIDLookup_FillsOnlyWhatItCanSettle_Integration(t *testing.T) {
	pool := newPool(t)
	ctx := context.Background()
	q := dbq.New(pool)
	dir := t.TempDir()
	_, album, artist := seedTrack(t, pool, filepath.Join(dir, "seed.mp3"))
	path := func(n string) string { return filepath.Join(dir, n+".mp3") }

	matched := addTrack(t, q, album, artist, "Matched", path("matched"), nil)
	ambiguous := addTrack(t, q, album, artist, "Twice", path("ambiguous"), nil)
	none := addTrack(t, q, album, artist, "Unknown", path("none"), nil)
	tagged := addTrack(t, q, album, artist, "Tagged", path("tagged"), ptr("rec-tag"))

	fake := &fakeLookup{answers: map[string][]acoustid.Candidate{
		path("matched"):   {cand("rec-matched", "Matched", 200, 0.97)},
		path("ambiguous"): {cand("rec-1", "Twice", 200, 0.95), cand("rec-2", "Twice", 200, 0.95)},
		path("tagged"):    {cand("rec-wrong", "Tagged", 200, 0.99)},
	}}
	w := testWorker(t, pool, fake)
	res, err := w.pass(ctx)
	if err != nil {
		t.Fatal(err)
	}
	// seed.mp3 has no answer, so it settles as no_match beside "Unknown".
	if res.Matched != 1 || res.Ambiguous != 1 || res.NoMatch != 2 {
		t.Errorf("pass = %+v, want 1 matched, 1 ambiguous, 2 no match", res)
	}
	if m, s := mbidOf(t, pool, matched.ID); m != "rec-matched" || s != "acoustid" {
		t.Errorf("matched track: %q from %q", m, s)
	}
	for _, tr := range []dbq.Track{ambiguous, none} {
		if m, _ := mbidOf(t, pool, tr.ID); m != "" {
			t.Errorf("%s got MBID %q from an unsettled lookup", tr.Title, m)
		}
	}
	// The tagged track is never looked up, so the fake's wrong answer for it
	// is never seen.
	if m, s := mbidOf(t, pool, tagged.ID); m != "rec-tag" || s != "tag" {
		t.Errorf("tagged track: %q from %q, want its tag untouched", m, s)
	}

	// Settled tracks are not looked up again.
	before := fake.calls
	if res, err := w.pass(ctx); err != nil || res.Processed != 0 || fake.calls != before {
		t.Errorf("second pass = %+v (err %v, %d new calls), want nothing to do", res, err, fake.calls-before)
	}

	// The matched file changes and its new bytes no longer match: the scan
	// drops the lookup, and the re-lookup takes the id back.
	if err := q.DeleteAcoustIDLookup(ctx, matched.ID); err != nil {
		t.Fatal(err)
	}
	fake.mu.Lock()
	fake.answers[path("matched")] = nil
	fake.mu.Unlock()
	if _, err := w.pass(ctx); err != nil {
		t.Fatal(err)
	}
	if m, s := mbidOf(t, pool, matched.ID); m != "" || s != "" {
		t.Errorf("after a non-matching re-lookup: %q from %q, want the looked-up id taken back", m, s)
	}

	cov, err := AcoustIDCoverage(ctx, pool)
	if err != nil {
		t.Fatal(err)
	}
	if cov.Total != 5 || cov.FromTag != 1 || cov.FromAcoustid != 0 || cov.Ambiguous != 1 || cov.NoMatch != 3 || cov.Pending != 0 {
		t.Errorf("coverage = %+v", cov)
	}
}

func TestAcoustIDLookup_StopsWhenTheServiceCannotAnswer_Integration(t *testing.T) {
	pool := newPool(t)
	ctx := context.Background()
	q := dbq.New(pool)
	dir := t.TempDir()
	_, album, artist := seedTrack(t, pool, filepath.Join(dir, "seed.mp3"))
	for i := range 5 {
		addTrack(t, q, album, artist, fmt.Sprint("T", i), filepath.Join(dir, fmt.Sprint(i, ".mp3")), nil)
	}
	for _, c := range []struct {
		err     error
		problem string
	}{
		{fmt.Errorf("%w: invalid API key", acoustid.ErrInvalidKey), "refused the API key"},
		{fmt.Errorf("%w: status 503", acoustid.ErrUnavailable), "could not be reached"},
	} {
		fake := &fakeLookup{err: c.err}
		w := testWorker(t, pool, fake)
		w.runOnce(ctx)
		// Stopped at the first answer, not one failed call per track. Two may
		// already be in flight at concurrency 2.
		if fake.calls > acoustIDLookupConcurrency {
			t.Errorf("%v: %d lookups made, want the pass to stop at the first", c.err, fake.calls)
		}
		if st := w.Status(); st.Running || !strings.Contains(st.Problem, c.problem) {
			t.Errorf("%v: status = %+v, want a problem mentioning %q", c.err, st, c.problem)
		}
		var n int
		if err := pool.QueryRow(ctx, `SELECT count(*) FROM track_acoustid_lookups`).Scan(&n); err != nil || n != 0 {
			t.Errorf("%v: %d lookups stored (err %v); a service failure says nothing about a track", c.err, n, err)
		}
	}
}

// The point of the milestone (#3921): once a lookup fills an MBID, the
// similarity worker picks the track as a seed and can map ListenBrainz's
// answers back to it.
func TestAcoustIDLookup_FilledTrackReachesSimilarity_Integration(t *testing.T) {
	pool := newPool(t)
	ctx := context.Background()
	q := dbq.New(pool)
	dir := t.TempDir()
	tr, _, _ := seedTrack(t, pool, filepath.Join(dir, "played.mp3"))

	u, err := q.CreateUser(ctx, dbq.CreateUserParams{
		Username: dbtest.TestUserPrefix + "acoustid", PasswordHash: "x", ApiTokenHash: "acoustid-token",
	})
	if err != nil {
		t.Fatal(err)
	}
	now := pgtype.Timestamptz{Time: time.Now(), Valid: true}
	session, err := q.InsertPlaySession(ctx, dbq.InsertPlaySessionParams{UserID: u.ID, StartedAt: now})
	if err != nil {
		t.Fatal(err)
	}
	if _, err := q.InsertPlayEvent(ctx, dbq.InsertPlayEventParams{
		UserID: u.ID, TrackID: tr.ID, SessionID: session.ID, StartedAt: now,
	}); err != nil {
		t.Fatal(err)
	}
	seeds := func() int {
		t.Helper()
		rows, err := q.ListPlayedTracksNeedingSimilarity(ctx, 100)
		if err != nil {
			t.Fatal(err)
		}
		return len(rows)
	}
	if n := seeds(); n != 0 {
		t.Fatalf("an untagged track is already a similarity seed (%d)", n)
	}

	fake := &fakeLookup{answers: map[string][]acoustid.Candidate{
		tr.FilePath: {cand("rec-played", tr.Title, 1, 0.99)},
	}}
	if _, err := testWorker(t, pool, fake).pass(ctx); err != nil {
		t.Fatal(err)
	}
	if n := seeds(); n != 1 {
		t.Errorf("after the lookup, %d similarity seeds, want the played track", n)
	}
	got, err := q.GetTracksByMBIDs(ctx, []string{"rec-played"})
	if err != nil || len(got) != 1 || got[0].ID != tr.ID {
		t.Errorf("GetTracksByMBIDs = %+v (err %v), want the played track", got, err)
	}
}

func TestAcoustIDLookup_IdleUntilReady(t *testing.T) {
	var nilSvc *AcoustIDSettingsService
	if nilSvc.Get().Ready() {
		t.Error("the defaults are ready; the lookup must ship off")
	}
	for _, s := range []AcoustIDSettings{{Enabled: true}, {APIKey: "k"}} {
		if s.Ready() {
			t.Errorf("%+v reads as ready", s)
		}
	}
	if _, err := nilSvc.Set(context.Background(), AcoustIDSettingsUpdate{MinScore: 0.2}); !errors.Is(err, ErrAcoustIDSettingOutOfRange) {
		t.Errorf("min_score 0.2: err = %v, want out of range", err)
	}
}