feat(library): AcoustID lookup worker fills the MBIDs tags leave empty (M401 #3920 #3921)
release / web (push) Successful in 1m44s
release / go (push) Successful in 2m1s
release / govulncheck (push) Successful in 17s
release / integration (push) Successful in 5m22s
release / android (push) Successful in 5m48s
release / Build signed APK (releases and dev) (push) Successful in 5m53s
release / Attach APK to the Release (tag releases only) (push) Skipped
release / Build + push container image (push) Successful in 2m6s
release / Verify release artifacts (tag releases only) (push) Skipped

Migration 0069 adds tracks.mbid_source (tag | acoustid), a lookup state per
track (matched | ambiguous | no_match | failed) and the acoustid_settings
row (off, no key, min score 0.85).

The file's tag outranks a lookup (D4). UpsertTrack keeps a looked-up id
through a re-read that finds no tag id and replaces it as soon as one
appears. SetTrackMbidFromAcoustID refuses to write over a tag id.

The worker fingerprints each untagged track with fpcalc's compressed
print, looks it up and writes an id only when D5 settles it: one
recording at or above the threshold, or one left after matching title and
length. Ambiguous and no-match results write nothing. A key AcoustID
refuses, or the service being unreachable, stops the pass and is reported
in the worker's status. It never counts as a verdict on a track.

A changed file drops its lookup in the scan. The re-lookup takes back an
id that no longer matches.

Admin API: GET /api/admin/library/acoustid (settings, status, coverage by
source), PUT …/acoustid-settings (write-only key), POST …/acoustid/run,
GET …/acoustid/unsettled.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
2026-10-06 23:26:03 -04:00
co-authored by Claude Opus 5.5
parent f13da62797
commit 3c575b137c
24 changed files with 1743 additions and 29 deletions
+16
View File
@@ -14,6 +14,7 @@ import (
"github.com/jackc/pgx/v5/pgxpool"
"git.fabledsword.com/bvandeusen/minstrel/internal/acoustid"
"git.fabledsword.com/bvandeusen/minstrel/internal/config"
"git.fabledsword.com/bvandeusen/minstrel/internal/coplay"
"git.fabledsword.com/bvandeusen/minstrel/internal/coverart"
@@ -139,6 +140,15 @@ func run() error {
logger.Warn("loudness settings: using defaults", "err", loudErr)
}
// AcoustID lookup settings (M401): shared by the lookup worker and the
// admin API. A load failure leaves the lookup off.
acoustIDSettings, acoustIDErr := library.NewAcoustIDSettingsService(ctx, pool)
if acoustIDErr != nil {
logger.Warn("acoustid settings: using defaults (lookup off)", "err", acoustIDErr)
}
acoustIDLookup := library.NewAcoustIDLookupWorker(pool, logger.With("component", "acoustid_lookup"),
acoustIDSettings, acoustid.New(acoustid.DefaultBaseURL))
contact := cfg.Library.ContactEmail
if contact == "" {
contact = "https://git.fabledsword.com/bvandeusen/minstrel"
@@ -240,6 +250,11 @@ func run() error {
// measurement. See internal/library/loudness_backfill.go.
go library.NewLoudnessBackfillWorker(pool, logger.With("component", "loudness_backfill"), loudSettings).Run(ctx)
// AcoustID lookup (M401): fills the recording MBIDs untagged tracks lack,
// so the similarity arms can see them. Idle until the operator adds a key
// and switches it on. See internal/library/acoustid_lookup.go.
go acoustIDLookup.Run(ctx)
// Duplicate sweep (M400 #3910): proposes groups of tracks holding one
// recording, from the fingerprints above. Sweeps only when fingerprints have
// changed since the last sweep.
@@ -390,6 +405,7 @@ func run() error {
srv.TagSettings = tagSettings
srv.FingerprintSettings = fpSettings
srv.LoudnessSettings = loudSettings
srv.AcoustIDLookup = acoustIDLookup
// The sweeper above holds this same instance, so a save from the admin
// card changes what it does on its next tick (#3936).
srv.ReacqSettings = reacqSettings
+171
View File
@@ -0,0 +1,171 @@
package api
import (
"encoding/json"
"errors"
"net/http"
"time"
"git.fabledsword.com/bvandeusen/minstrel/internal/apierror"
"git.fabledsword.com/bvandeusen/minstrel/internal/db/dbq"
"git.fabledsword.com/bvandeusen/minstrel/internal/library"
)
// AcoustID recording-id lookup, admin side (M401 #3922).
// acoustIDUnsettledLimit caps the admin list of tracks a lookup could not
// settle. It is a list to read, not to page through.
const acoustIDUnsettledLimit = 200
// acoustIDSettingsResp never carries the key, only whether one is stored.
type acoustIDSettingsResp struct {
Enabled bool `json:"enabled"`
MinScore float32 `json:"min_score"`
APIKeySet bool `json:"api_key_set"`
}
func acoustIDSettingsRespOf(s library.AcoustIDSettings) acoustIDSettingsResp {
return acoustIDSettingsResp{Enabled: s.Enabled, MinScore: s.MinScore, APIKeySet: s.APIKey != ""}
}
type acoustIDStatusResp struct {
Running bool `json:"running"`
LastPassAt *time.Time `json:"last_pass_at"`
// Problem is why the last pass stopped short, in words for the card.
Problem string `json:"problem"`
}
// acoustIDCoverageResp: from_tag + from_acoustid + the tracks with no MBID =
// total. ambiguous, no_match and failed are lookup states of tracks with no
// MBID; pending is what the worker still has to look at.
type acoustIDCoverageResp struct {
Total int64 `json:"total"`
FromTag int64 `json:"from_tag"`
FromAcoustID int64 `json:"from_acoustid"`
Ambiguous int64 `json:"ambiguous"`
NoMatch int64 `json:"no_match"`
Failed int64 `json:"failed"`
Pending int64 `json:"pending"`
}
type acoustIDResp struct {
Settings acoustIDSettingsResp `json:"settings"`
// Ready is whether the worker will call AcoustID at all: switched on
// and holding a key.
Ready bool `json:"ready"`
Status acoustIDStatusResp `json:"status"`
Coverage acoustIDCoverageResp `json:"coverage"`
}
// handleGetAcoustID implements GET /api/admin/library/acoustid: the settings,
// what the worker is doing, and how much of the library has an MBID by source.
func (h *handlers) handleGetAcoustID(w http.ResponseWriter, r *http.Request) {
row, err := library.AcoustIDCoverage(r.Context(), h.pool)
if err != nil {
writeErrWithLog(w, h.logger, "admin: get acoustid coverage", apierror.InternalMsg("lookup failed", err))
return
}
cfg := h.acoustIDLookup.Settings().Get()
st := h.acoustIDLookup.Status()
resp := acoustIDResp{
Settings: acoustIDSettingsRespOf(cfg),
Ready: cfg.Ready(),
Status: acoustIDStatusResp{Running: st.Running, Problem: st.Problem},
Coverage: acoustIDCoverageResp{
Total: row.Total,
FromTag: row.FromTag,
FromAcoustID: row.FromAcoustid,
Ambiguous: row.Ambiguous,
NoMatch: row.NoMatch,
Failed: row.Failed,
Pending: row.Pending,
},
}
if !st.LastPassAt.IsZero() {
at := st.LastPassAt.UTC()
resp.Status.LastPassAt = &at
}
writeJSON(w, http.StatusOK, resp)
}
// acoustIDSettingsReq is the PUT body. api_key left out keeps the stored key;
// an empty string clears it.
type acoustIDSettingsReq struct {
Enabled bool `json:"enabled"`
MinScore float32 `json:"min_score"`
APIKey *string `json:"api_key"`
}
// handleUpdateAcoustIDSettings implements PUT /api/admin/library/acoustid-settings.
func (h *handlers) handleUpdateAcoustIDSettings(w http.ResponseWriter, r *http.Request) {
var req acoustIDSettingsReq
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
writeErr(w, apierror.BadRequest("invalid_body", "malformed JSON"))
return
}
saved, err := h.acoustIDLookup.Settings().Set(r.Context(), library.AcoustIDSettingsUpdate{
Enabled: req.Enabled, MinScore: req.MinScore, APIKey: req.APIKey,
})
if err != nil {
if errors.Is(err, library.ErrAcoustIDSettingOutOfRange) {
writeErr(w, apierror.BadRequest("invalid_setting", err.Error()))
return
}
writeErrWithLog(w, h.logger, "admin acoustid settings: update failed", apierror.Internal(err))
return
}
writeJSON(w, http.StatusOK, acoustIDSettingsRespOf(saved))
}
// handleRunAcoustIDLookup implements POST /api/admin/library/acoustid/run:
// start a pass now rather than at the next tick. 409 when the lookup is off
// or has no key, so the card can say so rather than appear to do nothing.
func (h *handlers) handleRunAcoustIDLookup(w http.ResponseWriter, _ *http.Request) {
if !h.acoustIDLookup.Settings().Get().Ready() {
writeErr(w, apierror.Conflict("not_ready",
"Switch the AcoustID lookup on and add an API key first."))
return
}
h.acoustIDLookup.Kick()
w.WriteHeader(http.StatusAccepted)
}
type acoustIDUnsettledResp struct {
TrackID string `json:"track_id"`
Title string `json:"title"`
Artist string `json:"artist"`
Album string `json:"album"`
FilePath string `json:"file_path"`
State string `json:"state"`
BestScore *float32 `json:"best_score"`
Candidates int32 `json:"candidates"`
Detail *string `json:"detail"`
LookedUpAt time.Time `json:"looked_up_at"`
}
// handleListUnsettledAcoustID implements GET /api/admin/library/acoustid/unsettled:
// the tracks a lookup could not settle (ambiguous or failed), so they can be
// found and tagged by hand.
func (h *handlers) handleListUnsettledAcoustID(w http.ResponseWriter, r *http.Request) {
rows, err := dbq.New(h.pool).ListUnsettledAcoustIDLookups(r.Context(), acoustIDUnsettledLimit)
if err != nil {
writeErrWithLog(w, h.logger, "admin: list unsettled acoustid lookups", apierror.InternalMsg("lookup failed", err))
return
}
out := make([]acoustIDUnsettledResp, 0, len(rows))
for _, row := range rows {
out = append(out, acoustIDUnsettledResp{
TrackID: uuidToString(row.ID),
Title: row.Title,
Artist: row.ArtistName,
Album: row.AlbumTitle,
FilePath: row.FilePath,
State: row.State,
BestScore: row.BestScore,
Candidates: row.Candidates,
Detail: row.Detail,
LookedUpAt: row.LookedUpAt.Time.UTC(),
})
}
writeJSON(w, http.StatusOK, out)
}
+9 -1
View File
@@ -35,7 +35,7 @@ import (
// Mount attaches /api/* handlers to r. Public endpoints (login) are outside
// RequireUser; everything else is gated by the middleware. The events writer
// is shared with the Subsonic mount so /rest/scrobble feeds the same store.
func Mount(r chi.Router, pool *pgxpool.Pool, logger *slog.Logger, events *playevents.Writer, recCfg config.RecommendationConfig, recSettings *recsettings.Service, lidarrCfg *lidarrconfig.Service, lidarrReqs *lidarrrequests.Service, lidarrQuar *lidarrquarantine.Service, tracksSvc *tracks.Service, playlistsSvc *playlists.Service, coverEnricher *coverart.Enricher, coverSettings *coverart.SettingsService, tagSettings *tags.SettingsService, scanner *library.Scanner, scanCfg library.RunScanConfig, dataDir string, sender mailer.Sender, bus *eventbus.Bus, playlistScheduler *playlists.Scheduler, streamSecret []byte, netSettings *netsettings.Service, reacqSettings *reacquisition.SettingsService, fpSettings *library.FingerprintSettingsService, loudSettings *library.LoudnessSettingsService) {
func Mount(r chi.Router, pool *pgxpool.Pool, logger *slog.Logger, events *playevents.Writer, recCfg config.RecommendationConfig, recSettings *recsettings.Service, lidarrCfg *lidarrconfig.Service, lidarrReqs *lidarrrequests.Service, lidarrQuar *lidarrquarantine.Service, tracksSvc *tracks.Service, playlistsSvc *playlists.Service, coverEnricher *coverart.Enricher, coverSettings *coverart.SettingsService, tagSettings *tags.SettingsService, scanner *library.Scanner, scanCfg library.RunScanConfig, dataDir string, sender mailer.Sender, bus *eventbus.Bus, playlistScheduler *playlists.Scheduler, streamSecret []byte, netSettings *netsettings.Service, reacqSettings *reacquisition.SettingsService, fpSettings *library.FingerprintSettingsService, loudSettings *library.LoudnessSettingsService, acoustIDLookup *library.AcoustIDLookupWorker) {
rng := rand.New(rand.NewSource(rand.Int63()))
setupToken, err := auth.NewSetupToken()
if err != nil {
@@ -67,6 +67,7 @@ func Mount(r chi.Router, pool *pgxpool.Pool, logger *slog.Logger, events *playev
reacqSettings: reacqSettings,
fingerprintSettings: fpSettings,
loudnessSettings: loudSettings,
acoustIDLookup: acoustIDLookup,
leveled: newLeveledRenderer(dataDir, loudSettings, logger),
librarySize: recommendation.NewLibrarySize(nil),
loginGuard: auth.NewLoginGuard(),
@@ -251,6 +252,10 @@ func Mount(r chi.Router, pool *pgxpool.Pool, logger *slog.Logger, events *playev
admin.Get("/library/loudness", h.handleGetLoudnessCoverage)
admin.Get("/library/loudness-settings", h.handleGetLoudnessSettings)
admin.Put("/library/loudness-settings", h.handleUpdateLoudnessSettings)
admin.Get("/library/acoustid", h.handleGetAcoustID)
admin.Put("/library/acoustid-settings", h.handleUpdateAcoustIDSettings)
admin.Post("/library/acoustid/run", h.handleRunAcoustIDLookup)
admin.Get("/library/acoustid/unsettled", h.handleListUnsettledAcoustID)
// Duplicates report (#3912): proposals from the duplicate sweep, a
// trigger to sweep now, dismissal, and the merge (#3911), which deletes
// the removed copies' files after moving their history onto the kept one.
@@ -348,6 +353,9 @@ type handlers struct {
// loudnessSettings is the loudness analysis policy (M464 #4995), the same
// instance the loudness backfill reads. Nil serves the defaults.
loudnessSettings *library.LoudnessSettingsService
// acoustIDLookup is the AcoustID lookup worker (M401): its settings, its
// status and "look up now". Nil serves the defaults (off).
acoustIDLookup *library.AcoustIDLookupWorker
// leveled renders the leveled streams handed to Sonos/UPnP speakers
// (M464 #5001). Nil when its cache directory cannot be made: a level
// request then gets the plain stream.
+1 -1
View File
@@ -465,7 +465,7 @@ func TestRoutesRegisteredInMount(t *testing.T) {
r := chi.NewRouter()
w := playevents.NewWriter(h.pool, slog.New(slog.NewTextHandler(io.Discard, nil)),
30*time.Minute, 0.5, 30000)
Mount(r, h.pool, h.logger, w, config.RecommendationConfig{RadioSize: 50, RadioSizeMax: 200, RecentlyPlayedHours: 1}, h.recSettings, h.lidarrCfg, h.lidarrRequests, h.lidarrQuarantine, h.tracks, h.playlists, h.coverart, h.coverSettings, h.tagSettings, h.scanner, h.scanCfg, h.dataDir, nil, eventbus.New(), nil, nil, h.netSettings, nil, nil, nil)
Mount(r, h.pool, h.logger, w, config.RecommendationConfig{RadioSize: 50, RadioSizeMax: 200, RecentlyPlayedHours: 1}, h.recSettings, h.lidarrCfg, h.lidarrRequests, h.lidarrQuarantine, h.tracks, h.playlists, h.coverart, h.coverSettings, h.tagSettings, h.scanner, h.scanCfg, h.dataDir, nil, eventbus.New(), nil, nil, h.netSettings, nil, nil, nil, nil)
paths := []string{
"/api/artists",
+309
View File
@@ -0,0 +1,309 @@
// Code generated by sqlc. DO NOT EDIT.
// versions:
// sqlc v1.31.1
// source: acoustid.sql
package dbq
import (
"context"
"github.com/jackc/pgx/v5/pgtype"
)
const clearTrackAcoustIDMbid = `-- name: ClearTrackAcoustIDMbid :exec
UPDATE tracks
SET mbid = NULL, mbid_source = NULL
WHERE id = $1
AND mbid_source = 'acoustid'
`
// A re-lookup of changed bytes that no longer matches takes back the id an
// earlier lookup wrote. A tag id is never touched.
func (q *Queries) ClearTrackAcoustIDMbid(ctx context.Context, id pgtype.UUID) error {
_, err := q.db.Exec(ctx, clearTrackAcoustIDMbid, id)
return err
}
const deleteAcoustIDLookup = `-- name: DeleteAcoustIDLookup :exec
DELETE FROM track_acoustid_lookups WHERE track_id = $1
`
// The scan drops the lookup of bytes that are gone, so the worker checks the
// new ones.
func (q *Queries) DeleteAcoustIDLookup(ctx context.Context, trackID pgtype.UUID) error {
_, err := q.db.Exec(ctx, deleteAcoustIDLookup, trackID)
return err
}
const getAcoustIDSettings = `-- name: GetAcoustIDSettings :one
SELECT id, enabled, api_key, min_score, updated_at FROM acoustid_settings WHERE id = true
`
func (q *Queries) GetAcoustIDSettings(ctx context.Context) (AcoustidSetting, error) {
row := q.db.QueryRow(ctx, getAcoustIDSettings)
var i AcoustidSetting
err := row.Scan(
&i.ID,
&i.Enabled,
&i.ApiKey,
&i.MinScore,
&i.UpdatedAt,
)
return i, err
}
const getMbidCoverage = `-- name: GetMbidCoverage :one
SELECT count(*)::bigint AS total,
count(*) FILTER (WHERE t.mbid_source = 'tag')::bigint AS from_tag,
count(*) FILTER (WHERE t.mbid_source = 'acoustid')::bigint AS from_acoustid,
count(*) FILTER (WHERE l.state = 'ambiguous')::bigint AS ambiguous,
count(*) FILTER (WHERE l.state = 'no_match')::bigint AS no_match,
count(*) FILTER (WHERE l.state = 'failed')::bigint AS failed,
count(*) FILTER (
WHERE l.track_id IS NULL AND (t.mbid IS NULL OR t.mbid_source = 'acoustid')
)::bigint AS pending
FROM tracks t
LEFT JOIN track_acoustid_lookups l ON l.track_id = t.id
WHERE t.missing_since IS NULL
`
type GetMbidCoverageRow struct {
Total int64
FromTag int64
FromAcoustid int64
Ambiguous int64
NoMatch int64
Failed int64
Pending int64
}
// The admin gauge: present tracks by where their MBID came from, and the
// lookup states of those a lookup has seen. pending is what the worker's
// queue holds.
func (q *Queries) GetMbidCoverage(ctx context.Context) (GetMbidCoverageRow, error) {
row := q.db.QueryRow(ctx, getMbidCoverage)
var i GetMbidCoverageRow
err := row.Scan(
&i.Total,
&i.FromTag,
&i.FromAcoustid,
&i.Ambiguous,
&i.NoMatch,
&i.Failed,
&i.Pending,
)
return i, err
}
const listTracksNeedingAcoustIDLookup = `-- name: ListTracksNeedingAcoustIDLookup :many
SELECT t.id, t.file_path, t.title, t.duration_ms
FROM tracks t
WHERE (t.mbid IS NULL OR t.mbid_source = 'acoustid')
AND t.missing_since IS NULL
AND t.id > $1::uuid
AND NOT EXISTS (SELECT 1 FROM track_acoustid_lookups l WHERE l.track_id = t.id)
ORDER BY t.id
LIMIT $2
`
type ListTracksNeedingAcoustIDLookupParams struct {
AfterID pgtype.UUID
BatchLimit int32
}
type ListTracksNeedingAcoustIDLookupRow struct {
ID pgtype.UUID
FilePath string
Title string
DurationMs int32
}
// AcoustID recording-id lookup (M401). Migration 0069 says what each
// lookup state means.
// The lookup worker's queue: present tracks with no tag MBID and no lookup
// since their file last changed. A looked-up id is included once its row is
// gone (the file changed), so the new bytes are checked rather than trusted.
// Keyset-paged on id, as the loudness queue is, so a pass ends even when
// every attempt is inconclusive and writes nothing.
func (q *Queries) ListTracksNeedingAcoustIDLookup(ctx context.Context, arg ListTracksNeedingAcoustIDLookupParams) ([]ListTracksNeedingAcoustIDLookupRow, error) {
rows, err := q.db.Query(ctx, listTracksNeedingAcoustIDLookup, arg.AfterID, arg.BatchLimit)
if err != nil {
return nil, err
}
defer rows.Close()
var items []ListTracksNeedingAcoustIDLookupRow
for rows.Next() {
var i ListTracksNeedingAcoustIDLookupRow
if err := rows.Scan(
&i.ID,
&i.FilePath,
&i.Title,
&i.DurationMs,
); err != nil {
return nil, err
}
items = append(items, i)
}
if err := rows.Err(); err != nil {
return nil, err
}
return items, nil
}
const listUnsettledAcoustIDLookups = `-- name: ListUnsettledAcoustIDLookups :many
SELECT t.id, t.title, t.file_path, ar.name AS artist_name, al.title AS album_title,
l.state, l.best_score, l.candidates, l.detail, l.looked_up_at
FROM track_acoustid_lookups l
JOIN tracks t ON t.id = l.track_id
JOIN artists ar ON ar.id = t.artist_id
JOIN albums al ON al.id = t.album_id
WHERE l.state IN ('ambiguous', 'failed')
AND t.missing_since IS NULL
ORDER BY ar.name, al.title, t.disc_number NULLS FIRST, t.track_number NULLS FIRST, t.title
LIMIT $1
`
type ListUnsettledAcoustIDLookupsRow struct {
ID pgtype.UUID
Title string
FilePath string
ArtistName string
AlbumTitle string
State string
BestScore *float32
Candidates int32
Detail *string
LookedUpAt pgtype.Timestamptz
}
// The admin list of tracks a lookup could not settle (ambiguous or failed),
// so they are findable rather than silently empty.
func (q *Queries) ListUnsettledAcoustIDLookups(ctx context.Context, rowLimit int32) ([]ListUnsettledAcoustIDLookupsRow, error) {
rows, err := q.db.Query(ctx, listUnsettledAcoustIDLookups, rowLimit)
if err != nil {
return nil, err
}
defer rows.Close()
var items []ListUnsettledAcoustIDLookupsRow
for rows.Next() {
var i ListUnsettledAcoustIDLookupsRow
if err := rows.Scan(
&i.ID,
&i.Title,
&i.FilePath,
&i.ArtistName,
&i.AlbumTitle,
&i.State,
&i.BestScore,
&i.Candidates,
&i.Detail,
&i.LookedUpAt,
); err != nil {
return nil, err
}
items = append(items, i)
}
if err := rows.Err(); err != nil {
return nil, err
}
return items, nil
}
const recordAcoustIDLookup = `-- name: RecordAcoustIDLookup :exec
INSERT INTO track_acoustid_lookups (track_id, state, best_score, candidates, recording_mbid, detail)
VALUES ($1, $2, $3, $4,
$5, $6)
ON CONFLICT (track_id) DO UPDATE SET
state = EXCLUDED.state,
best_score = EXCLUDED.best_score,
candidates = EXCLUDED.candidates,
recording_mbid = EXCLUDED.recording_mbid,
detail = EXCLUDED.detail,
looked_up_at = now()
`
type RecordAcoustIDLookupParams struct {
TrackID pgtype.UUID
State string
BestScore *float32
Candidates int32
RecordingMbid *string
Detail *string
}
func (q *Queries) RecordAcoustIDLookup(ctx context.Context, arg RecordAcoustIDLookupParams) error {
_, err := q.db.Exec(ctx, recordAcoustIDLookup,
arg.TrackID,
arg.State,
arg.BestScore,
arg.Candidates,
arg.RecordingMbid,
arg.Detail,
)
return err
}
const setTrackMbidFromAcoustID = `-- name: SetTrackMbidFromAcoustID :execrows
UPDATE tracks
SET mbid = $1, mbid_source = 'acoustid'
WHERE id = $2
AND (mbid IS NULL OR mbid_source = 'acoustid')
`
type SetTrackMbidFromAcoustIDParams struct {
Mbid *string
ID pgtype.UUID
}
// Never over a tag id (D4): the guard is here, not only in the worker, so a
// tag the scan read between the worker's list and this write still wins.
// updated_at is left alone: it gates the scan's mtime skip, and moving it
// past a file's mtime would hide a change made just before the lookup.
func (q *Queries) SetTrackMbidFromAcoustID(ctx context.Context, arg SetTrackMbidFromAcoustIDParams) (int64, error) {
result, err := q.db.Exec(ctx, setTrackMbidFromAcoustID, arg.Mbid, arg.ID)
if err != nil {
return 0, err
}
return result.RowsAffected(), nil
}
const updateAcoustIDSettings = `-- name: UpdateAcoustIDSettings :one
UPDATE acoustid_settings
SET enabled = $1,
min_score = $2,
api_key = CASE WHEN $3::boolean
THEN NULLIF($4::text, '')
ELSE api_key END,
updated_at = now()
WHERE id = true
RETURNING id, enabled, api_key, min_score, updated_at
`
type UpdateAcoustIDSettingsParams struct {
Enabled bool
MinScore float32
SetApiKey bool
ApiKey string
}
// The key is write-only from the admin card: set_api_key false keeps the
// stored one, and an empty api_key with set_api_key true clears it.
func (q *Queries) UpdateAcoustIDSettings(ctx context.Context, arg UpdateAcoustIDSettingsParams) (AcoustidSetting, error) {
row := q.db.QueryRow(ctx, updateAcoustIDSettings,
arg.Enabled,
arg.MinScore,
arg.SetApiKey,
arg.ApiKey,
)
var i AcoustidSetting
err := row.Scan(
&i.ID,
&i.Enabled,
&i.ApiKey,
&i.MinScore,
&i.UpdatedAt,
)
return i, err
}
+2 -1
View File
@@ -261,7 +261,7 @@ func (q *Queries) InsertSkipEvent(ctx context.Context, arg InsertSkipEventParams
}
const listRecentSessionTracks = `-- name: ListRecentSessionTracks :many
SELECT t.id, t.title, t.album_id, t.artist_id, t.track_number, t.disc_number, t.duration_ms, t.file_path, t.file_size, t.file_format, t.bitrate, t.mbid, t.genre, t.added_at, t.updated_at, t.tag_source, t.tag_sources_version, t.tag_read_version, t.missing_since FROM tracks t
SELECT t.id, t.title, t.album_id, t.artist_id, t.track_number, t.disc_number, t.duration_ms, t.file_path, t.file_size, t.file_format, t.bitrate, t.mbid, t.genre, t.added_at, t.updated_at, t.tag_source, t.tag_sources_version, t.tag_read_version, t.missing_since, t.mbid_source FROM tracks t
JOIN play_events pe ON pe.track_id = t.id
WHERE pe.session_id = $1
AND pe.started_at < $2
@@ -307,6 +307,7 @@ func (q *Queries) ListRecentSessionTracks(ctx context.Context, arg ListRecentSes
&i.TagSourcesVersion,
&i.TagReadVersion,
&i.MissingSince,
&i.MbidSource,
); err != nil {
return nil, err
}
+2 -1
View File
@@ -14,7 +14,7 @@ import (
const listUserHistory = `-- name: ListUserHistory :many
SELECT pe.id AS event_id,
pe.started_at,
t.id, t.title, t.album_id, t.artist_id, t.track_number, t.disc_number, t.duration_ms, t.file_path, t.file_size, t.file_format, t.bitrate, t.mbid, t.genre, t.added_at, t.updated_at, t.tag_source, t.tag_sources_version, t.tag_read_version, t.missing_since,
t.id, t.title, t.album_id, t.artist_id, t.track_number, t.disc_number, t.duration_ms, t.file_path, t.file_size, t.file_format, t.bitrate, t.mbid, t.genre, t.added_at, t.updated_at, t.tag_source, t.tag_sources_version, t.tag_read_version, t.missing_since, t.mbid_source,
albums.title AS album_title,
artists.name AS artist_name
FROM play_events pe
@@ -81,6 +81,7 @@ func (q *Queries) ListUserHistory(ctx context.Context, arg ListUserHistoryParams
&i.Track.TagSourcesVersion,
&i.Track.TagReadVersion,
&i.Track.MissingSince,
&i.Track.MbidSource,
&i.AlbumTitle,
&i.ArtistName,
); err != nil {
+2 -1
View File
@@ -259,7 +259,7 @@ func (q *Queries) ListLikedTrackIDs(ctx context.Context, userID pgtype.UUID) ([]
}
const listLikedTrackRows = `-- name: ListLikedTrackRows :many
SELECT t.id, t.title, t.album_id, t.artist_id, t.track_number, t.disc_number, t.duration_ms, t.file_path, t.file_size, t.file_format, t.bitrate, t.mbid, t.genre, t.added_at, t.updated_at, t.tag_source, t.tag_sources_version, t.tag_read_version, t.missing_since FROM tracks t
SELECT t.id, t.title, t.album_id, t.artist_id, t.track_number, t.disc_number, t.duration_ms, t.file_path, t.file_size, t.file_format, t.bitrate, t.mbid, t.genre, t.added_at, t.updated_at, t.tag_source, t.tag_sources_version, t.tag_read_version, t.missing_since, t.mbid_source FROM tracks t
JOIN general_likes l ON l.track_id = t.id
WHERE l.user_id = $1
ORDER BY l.liked_at DESC
@@ -301,6 +301,7 @@ func (q *Queries) ListLikedTrackRows(ctx context.Context, arg ListLikedTrackRows
&i.TagSourcesVersion,
&i.TagReadVersion,
&i.MissingSince,
&i.MbidSource,
); err != nil {
return nil, err
}
+1 -1
View File
@@ -196,7 +196,7 @@ func (q *Queries) MergeCopyTrackTags(ctx context.Context, arg MergeCopyTrackTags
const mergeInheritTrackMbid = `-- name: MergeInheritTrackMbid :exec
UPDATE tracks AS survivor
SET mbid = loser.mbid
SET mbid = loser.mbid, mbid_source = loser.mbid_source
FROM tracks AS loser
WHERE survivor.id = $1::uuid
AND loser.id = $2::uuid
+19
View File
@@ -187,6 +187,14 @@ func (ns NullLidarrRequestStatus) Value() (driver.Value, error) {
return string(ns.LidarrRequestStatus), nil
}
type AcoustidSetting struct {
ID bool
Enabled bool
ApiKey *string
MinScore float32
UpdatedAt pgtype.Timestamptz
}
type Album struct {
ID pgtype.UUID
Title string
@@ -720,6 +728,17 @@ type Track struct {
TagSourcesVersion int32
TagReadVersion int16
MissingSince pgtype.Timestamptz
MbidSource *string
}
type TrackAcoustidLookup struct {
TrackID pgtype.UUID
State string
BestScore *float32
Candidates int32
RecordingMbid *string
Detail *string
LookedUpAt pgtype.Timestamptz
}
type TrackFingerprint struct {
+8 -4
View File
@@ -208,7 +208,7 @@ WITH plays AS (
WHERE user_id = $2 AND was_skipped = false
GROUP BY track_id
)
SELECT t.id, t.title, t.album_id, t.artist_id, t.track_number, t.disc_number, t.duration_ms, t.file_path, t.file_size, t.file_format, t.bitrate, t.mbid, t.genre, t.added_at, t.updated_at, t.tag_source, t.tag_sources_version, t.tag_read_version, t.missing_since,
SELECT t.id, t.title, t.album_id, t.artist_id, t.track_number, t.disc_number, t.duration_ms, t.file_path, t.file_size, t.file_format, t.bitrate, t.mbid, t.genre, t.added_at, t.updated_at, t.tag_source, t.tag_sources_version, t.tag_read_version, t.missing_since, t.mbid_source,
albums.title AS album_title,
artists.name AS artist_name
FROM plays p
@@ -270,6 +270,7 @@ func (q *Queries) ListMostPlayedTracksForArtist(ctx context.Context, arg ListMos
&i.Track.TagSourcesVersion,
&i.Track.TagReadVersion,
&i.Track.MissingSince,
&i.Track.MbidSource,
&i.AlbumTitle,
&i.ArtistName,
); err != nil {
@@ -290,7 +291,7 @@ WITH plays AS (
WHERE user_id = $1 AND was_skipped = false
GROUP BY track_id
)
SELECT t.id, t.title, t.album_id, t.artist_id, t.track_number, t.disc_number, t.duration_ms, t.file_path, t.file_size, t.file_format, t.bitrate, t.mbid, t.genre, t.added_at, t.updated_at, t.tag_source, t.tag_sources_version, t.tag_read_version, t.missing_since,
SELECT t.id, t.title, t.album_id, t.artist_id, t.track_number, t.disc_number, t.duration_ms, t.file_path, t.file_size, t.file_format, t.bitrate, t.mbid, t.genre, t.added_at, t.updated_at, t.tag_source, t.tag_sources_version, t.tag_read_version, t.missing_since, t.mbid_source,
albums.title AS album_title,
artists.name AS artist_name
FROM plays p
@@ -354,6 +355,7 @@ func (q *Queries) ListMostPlayedTracksForUser(ctx context.Context, arg ListMostP
&i.Track.TagSourcesVersion,
&i.Track.TagReadVersion,
&i.Track.MissingSince,
&i.Track.MbidSource,
&i.AlbumTitle,
&i.ArtistName,
); err != nil {
@@ -691,7 +693,7 @@ func (q *Queries) ListRediscoverArtistsForUser(ctx context.Context, arg ListRedi
const loadRadioCandidates = `-- name: LoadRadioCandidates :many
SELECT
t.id, t.title, t.album_id, t.artist_id, t.track_number, t.disc_number, t.duration_ms, t.file_path, t.file_size, t.file_format, t.bitrate, t.mbid, t.genre, t.added_at, t.updated_at, t.tag_source, t.tag_sources_version, t.tag_read_version, t.missing_since,
t.id, t.title, t.album_id, t.artist_id, t.track_number, t.disc_number, t.duration_ms, t.file_path, t.file_size, t.file_format, t.bitrate, t.mbid, t.genre, t.added_at, t.updated_at, t.tag_source, t.tag_sources_version, t.tag_read_version, t.missing_since, t.mbid_source,
(l.user_id IS NOT NULL)::bool AS is_liked,
pe.last_played_at::timestamptz AS last_played_at,
pe.play_count,
@@ -772,6 +774,7 @@ func (q *Queries) LoadRadioCandidates(ctx context.Context, arg LoadRadioCandidat
&i.Track.TagSourcesVersion,
&i.Track.TagReadVersion,
&i.Track.MissingSince,
&i.Track.MbidSource,
&i.IsLiked,
&i.LastPlayedAt,
&i.PlayCount,
@@ -904,7 +907,7 @@ random_fill AS (
LIMIT $9
)
SELECT
t.id, t.title, t.album_id, t.artist_id, t.track_number, t.disc_number, t.duration_ms, t.file_path, t.file_size, t.file_format, t.bitrate, t.mbid, t.genre, t.added_at, t.updated_at, t.tag_source, t.tag_sources_version, t.tag_read_version, t.missing_since,
t.id, t.title, t.album_id, t.artist_id, t.track_number, t.disc_number, t.duration_ms, t.file_path, t.file_size, t.file_format, t.bitrate, t.mbid, t.genre, t.added_at, t.updated_at, t.tag_source, t.tag_sources_version, t.tag_read_version, t.missing_since, t.mbid_source,
(l.user_id IS NOT NULL)::bool AS is_liked,
pe.last_played_at::timestamptz AS last_played_at,
pe.play_count,
@@ -1031,6 +1034,7 @@ func (q *Queries) LoadRadioCandidatesV2(ctx context.Context, arg LoadRadioCandid
&i.Track.TagSourcesVersion,
&i.Track.TagReadVersion,
&i.Track.MissingSince,
&i.Track.MbidSource,
&i.IsLiked,
&i.LastPlayedAt,
&i.PlayCount,
+27 -12
View File
@@ -233,7 +233,7 @@ func (q *Queries) FindMissingTrackByMbid(ctx context.Context, mbid string) ([]Fi
}
const getTrackByID = `-- name: GetTrackByID :one
SELECT id, title, album_id, artist_id, track_number, disc_number, duration_ms, file_path, file_size, file_format, bitrate, mbid, genre, added_at, updated_at, tag_source, tag_sources_version, tag_read_version, missing_since FROM tracks WHERE id = $1
SELECT id, title, album_id, artist_id, track_number, disc_number, duration_ms, file_path, file_size, file_format, bitrate, mbid, genre, added_at, updated_at, tag_source, tag_sources_version, tag_read_version, missing_since, mbid_source FROM tracks WHERE id = $1
`
func (q *Queries) GetTrackByID(ctx context.Context, id pgtype.UUID) (Track, error) {
@@ -259,12 +259,13 @@ func (q *Queries) GetTrackByID(ctx context.Context, id pgtype.UUID) (Track, erro
&i.TagSourcesVersion,
&i.TagReadVersion,
&i.MissingSince,
&i.MbidSource,
)
return i, err
}
const getTrackByPath = `-- name: GetTrackByPath :one
SELECT id, title, album_id, artist_id, track_number, disc_number, duration_ms, file_path, file_size, file_format, bitrate, mbid, genre, added_at, updated_at, tag_source, tag_sources_version, tag_read_version, missing_since FROM tracks WHERE file_path = $1
SELECT id, title, album_id, artist_id, track_number, disc_number, duration_ms, file_path, file_size, file_format, bitrate, mbid, genre, added_at, updated_at, tag_source, tag_sources_version, tag_read_version, missing_since, mbid_source FROM tracks WHERE file_path = $1
`
func (q *Queries) GetTrackByPath(ctx context.Context, filePath string) (Track, error) {
@@ -290,12 +291,13 @@ func (q *Queries) GetTrackByPath(ctx context.Context, filePath string) (Track, e
&i.TagSourcesVersion,
&i.TagReadVersion,
&i.MissingSince,
&i.MbidSource,
)
return i, err
}
const getTracksByIDs = `-- name: GetTracksByIDs :many
SELECT id, title, album_id, artist_id, track_number, disc_number, duration_ms, file_path, file_size, file_format, bitrate, mbid, genre, added_at, updated_at, tag_source, tag_sources_version, tag_read_version, missing_since FROM tracks WHERE id = ANY($1::uuid[])
SELECT id, title, album_id, artist_id, track_number, disc_number, duration_ms, file_path, file_size, file_format, bitrate, mbid, genre, added_at, updated_at, tag_source, tag_sources_version, tag_read_version, missing_since, mbid_source FROM tracks WHERE id = ANY($1::uuid[])
`
// Batched lookup used by /api/library/sync to hydrate upsert payloads
@@ -329,6 +331,7 @@ func (q *Queries) GetTracksByIDs(ctx context.Context, dollar_1 []pgtype.UUID) ([
&i.TagSourcesVersion,
&i.TagReadVersion,
&i.MissingSince,
&i.MbidSource,
); err != nil {
return nil, err
}
@@ -341,7 +344,7 @@ func (q *Queries) GetTracksByIDs(ctx context.Context, dollar_1 []pgtype.UUID) ([
}
const listArtistTracksForUser = `-- name: ListArtistTracksForUser :many
SELECT t.id, t.title, t.album_id, t.artist_id, t.track_number, t.disc_number, t.duration_ms, t.file_path, t.file_size, t.file_format, t.bitrate, t.mbid, t.genre, t.added_at, t.updated_at, t.tag_source, t.tag_sources_version, t.tag_read_version, t.missing_since,
SELECT t.id, t.title, t.album_id, t.artist_id, t.track_number, t.disc_number, t.duration_ms, t.file_path, t.file_size, t.file_format, t.bitrate, t.mbid, t.genre, t.added_at, t.updated_at, t.tag_source, t.tag_sources_version, t.tag_read_version, t.missing_since, t.mbid_source,
albums.title AS album_title,
artists.name AS artist_name
FROM tracks t
@@ -401,6 +404,7 @@ func (q *Queries) ListArtistTracksForUser(ctx context.Context, arg ListArtistTra
&i.Track.TagSourcesVersion,
&i.Track.TagReadVersion,
&i.Track.MissingSince,
&i.Track.MbidSource,
&i.AlbumTitle,
&i.ArtistName,
); err != nil {
@@ -501,7 +505,7 @@ func (q *Queries) ListMissingTracks(ctx context.Context, arg ListMissingTracksPa
}
const listRandomTracksForUser = `-- name: ListRandomTracksForUser :many
SELECT t.id, t.title, t.album_id, t.artist_id, t.track_number, t.disc_number, t.duration_ms, t.file_path, t.file_size, t.file_format, t.bitrate, t.mbid, t.genre, t.added_at, t.updated_at, t.tag_source, t.tag_sources_version, t.tag_read_version, t.missing_since,
SELECT t.id, t.title, t.album_id, t.artist_id, t.track_number, t.disc_number, t.duration_ms, t.file_path, t.file_size, t.file_format, t.bitrate, t.mbid, t.genre, t.added_at, t.updated_at, t.tag_source, t.tag_sources_version, t.tag_read_version, t.missing_since, t.mbid_source,
albums.title AS album_title,
artists.name AS artist_name
FROM tracks t
@@ -558,6 +562,7 @@ func (q *Queries) ListRandomTracksForUser(ctx context.Context, arg ListRandomTra
&i.Track.TagSourcesVersion,
&i.Track.TagReadVersion,
&i.Track.MissingSince,
&i.Track.MbidSource,
&i.AlbumTitle,
&i.ArtistName,
); err != nil {
@@ -607,7 +612,7 @@ func (q *Queries) ListTrackPathsForReconcile(ctx context.Context) ([]ListTrackPa
}
const listTracksByAlbum = `-- name: ListTracksByAlbum :many
SELECT id, title, album_id, artist_id, track_number, disc_number, duration_ms, file_path, file_size, file_format, bitrate, mbid, genre, added_at, updated_at, tag_source, tag_sources_version, tag_read_version, missing_since FROM tracks
SELECT id, title, album_id, artist_id, track_number, disc_number, duration_ms, file_path, file_size, file_format, bitrate, mbid, genre, added_at, updated_at, tag_source, tag_sources_version, tag_read_version, missing_since, mbid_source FROM tracks
WHERE album_id = $1
AND NOT EXISTS (
SELECT 1 FROM lidarr_quarantine q
@@ -653,6 +658,7 @@ func (q *Queries) ListTracksByAlbum(ctx context.Context, arg ListTracksByAlbumPa
&i.TagSourcesVersion,
&i.TagReadVersion,
&i.MissingSince,
&i.MbidSource,
); err != nil {
return nil, err
}
@@ -726,7 +732,7 @@ func (q *Queries) MarkTracksMissing(ctx context.Context, ids []pgtype.UUID) (int
}
const searchTracks = `-- name: SearchTracks :many
SELECT id, title, album_id, artist_id, track_number, disc_number, duration_ms, file_path, file_size, file_format, bitrate, mbid, genre, added_at, updated_at, tag_source, tag_sources_version, tag_read_version, missing_since FROM tracks
SELECT id, title, album_id, artist_id, track_number, disc_number, duration_ms, file_path, file_size, file_format, bitrate, mbid, genre, added_at, updated_at, tag_source, tag_sources_version, tag_read_version, missing_since, mbid_source FROM tracks
WHERE title ILIKE '%' || $1::text || '%'
AND NOT EXISTS (
SELECT 1 FROM lidarr_quarantine q
@@ -779,6 +785,7 @@ func (q *Queries) SearchTracks(ctx context.Context, arg SearchTracksParams) ([]T
&i.TagSourcesVersion,
&i.TagReadVersion,
&i.MissingSince,
&i.MbidSource,
); err != nil {
return nil, err
}
@@ -792,7 +799,7 @@ func (q *Queries) SearchTracks(ctx context.Context, arg SearchTracksParams) ([]T
const setTrackMbidIfNull = `-- name: SetTrackMbidIfNull :exec
UPDATE tracks
SET mbid = $2, updated_at = now()
SET mbid = $2, mbid_source = 'tag', updated_at = now()
WHERE id = $1 AND mbid IS NULL
`
@@ -812,8 +819,9 @@ const upsertTrack = `-- name: UpsertTrack :one
INSERT INTO tracks (
title, album_id, artist_id, track_number, disc_number,
duration_ms, file_path, file_size, file_format, bitrate, mbid, genre,
tag_read_version
) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13)
tag_read_version, mbid_source
) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13,
CASE WHEN $11::text IS NOT NULL THEN 'tag' END)
ON CONFLICT (file_path) DO UPDATE SET
title = EXCLUDED.title,
album_id = EXCLUDED.album_id,
@@ -824,13 +832,17 @@ ON CONFLICT (file_path) DO UPDATE SET
file_size = EXCLUDED.file_size,
file_format = EXCLUDED.file_format,
bitrate = EXCLUDED.bitrate,
mbid = EXCLUDED.mbid,
-- Both read the row as it was before this statement.
mbid = CASE WHEN EXCLUDED.mbid IS NULL AND tracks.mbid_source = 'acoustid'
THEN tracks.mbid ELSE EXCLUDED.mbid END,
mbid_source = CASE WHEN EXCLUDED.mbid IS NOT NULL THEN 'tag'
WHEN tracks.mbid_source = 'acoustid' THEN 'acoustid' END,
genre = EXCLUDED.genre,
-- Stamped on update too, so a tag-repair pass marks rows as done and the
-- next scan can short-circuit them again (#2499).
tag_read_version = EXCLUDED.tag_read_version,
updated_at = now()
RETURNING id, title, album_id, artist_id, track_number, disc_number, duration_ms, file_path, file_size, file_format, bitrate, mbid, genre, added_at, updated_at, tag_source, tag_sources_version, tag_read_version, missing_since
RETURNING id, title, album_id, artist_id, track_number, disc_number, duration_ms, file_path, file_size, file_format, bitrate, mbid, genre, added_at, updated_at, tag_source, tag_sources_version, tag_read_version, missing_since, mbid_source
`
type UpsertTrackParams struct {
@@ -850,6 +862,8 @@ type UpsertTrackParams struct {
}
// file_path is the canonical identity for library scan; mbid is secondary.
// mbid is the file's tag. A looked-up id (mbid_source 'acoustid', M401)
// survives a re-read that finds no tag id, and a tag id replaces it (D4).
func (q *Queries) UpsertTrack(ctx context.Context, arg UpsertTrackParams) (Track, error) {
row := q.db.QueryRow(ctx, upsertTrack,
arg.Title,
@@ -887,6 +901,7 @@ func (q *Queries) UpsertTrack(ctx context.Context, arg UpsertTrackParams) (Track
&i.TagSourcesVersion,
&i.TagReadVersion,
&i.MissingSince,
&i.MbidSource,
)
return i, err
}
@@ -0,0 +1,7 @@
DROP TABLE IF EXISTS acoustid_settings;
DROP TABLE IF EXISTS track_acoustid_lookups;
-- Looked-up ids go with the lookups; a tag id stays.
UPDATE tracks SET mbid = NULL WHERE mbid_source = 'acoustid';
ALTER TABLE tracks DROP CONSTRAINT IF EXISTS tracks_mbid_source_pair;
ALTER TABLE tracks DROP CONSTRAINT IF EXISTS tracks_mbid_source_check;
ALTER TABLE tracks DROP COLUMN IF EXISTS mbid_source;
@@ -0,0 +1,69 @@
-- 0069_acoustid_lookup.up.sql — recording MBIDs looked up through AcoustID
-- for tracks whose tags carry none (Scribe milestone #401).
--
-- A track with no recording MBID is invisible to the ListenBrainz similarity
-- arm in both directions, so this fills the ids the tags leave empty.
-- Where tracks.mbid came from. The file's own tag always outranks a lookup
-- (M401 D4): the scan's upsert keeps a looked-up id while the tag carries
-- none, and replaces it the moment the tag carries one.
ALTER TABLE tracks ADD COLUMN mbid_source text;
-- Every id stored so far was read from a tag (the scan or the tag backfill),
-- or inherited from a merged duplicate's tag.
UPDATE tracks SET mbid_source = 'tag' WHERE mbid IS NOT NULL;
ALTER TABLE tracks
ADD CONSTRAINT tracks_mbid_source_check CHECK (mbid_source IN ('tag', 'acoustid')),
ADD CONSTRAINT tracks_mbid_source_pair CHECK ((mbid IS NULL) = (mbid_source IS NULL));
-- What happened the last time a track was looked up. A table of its own, as
-- track_loudness is (0065): tracks is read with SELECT * on the hot path.
--
-- What a row means, which the lookup worker depends on:
-- no row never looked up, or the file changed since (the scan deletes
-- the row): look it up
-- matched one recording cleared the score threshold, or one was left
-- after disambiguating by title and length; recording_mbid is it
-- ambiguous several recordings cleared it and nothing told them apart.
-- Nothing is written: a wrong MBID would feed similarity the
-- wrong neighbours, which is worse than none (M401 D5)
-- no_match AcoustID linked nothing at or above the threshold
-- failed fpcalc rejected the file, or AcoustID rejected its print
-- Every state is settled until the file changes. A failure that says nothing
-- about the file (AcoustID unreachable, a timeout, a cancelled pass) writes no
-- row, so the worker tries again.
CREATE TABLE track_acoustid_lookups (
track_id uuid PRIMARY KEY REFERENCES tracks (id) ON DELETE CASCADE,
state text NOT NULL,
-- The best score any linked recording had; NULL when none was linked.
best_score real,
-- How many distinct recordings were linked, at any score.
candidates integer NOT NULL DEFAULT 0,
recording_mbid text,
-- Why a lookup failed, for the admin list.
detail text,
looked_up_at timestamptz NOT NULL DEFAULT now(),
CONSTRAINT track_acoustid_lookups_state_check
CHECK (state IN ('matched', 'ambiguous', 'no_match', 'failed')),
CONSTRAINT track_acoustid_lookups_matched_has_mbid
CHECK ((state = 'matched') = (recording_mbid IS NOT NULL))
);
CREATE INDEX track_acoustid_lookups_state ON track_acoustid_lookups (state);
-- AcoustID's knobs, in admin Settings (rule 25). Singleton in the style of
-- loudness_settings (0065).
CREATE TABLE acoustid_settings (
id boolean PRIMARY KEY DEFAULT true,
-- Off by default (rule 26): AcoustID needs a key the operator registers,
-- and the lookup sends fingerprints to a third party.
enabled boolean NOT NULL DEFAULT false,
api_key text,
-- The score a recording needs before it is considered at all. AcoustID
-- scores a true match close to 1; 0.85 leaves room for a lossy encode.
min_score real NOT NULL DEFAULT 0.85,
updated_at timestamptz NOT NULL DEFAULT now(),
CONSTRAINT acoustid_settings_singleton CHECK (id = true),
CONSTRAINT acoustid_settings_min_score_range CHECK (min_score >= 0.5 AND min_score <= 1)
);
INSERT INTO acoustid_settings (id) VALUES (true) ON CONFLICT (id) DO NOTHING;
+99
View File
@@ -0,0 +1,99 @@
-- AcoustID recording-id lookup (M401). Migration 0069 says what each
-- lookup state means.
-- name: ListTracksNeedingAcoustIDLookup :many
-- The lookup worker's queue: present tracks with no tag MBID and no lookup
-- since their file last changed. A looked-up id is included once its row is
-- gone (the file changed), so the new bytes are checked rather than trusted.
-- Keyset-paged on id, as the loudness queue is, so a pass ends even when
-- every attempt is inconclusive and writes nothing.
SELECT t.id, t.file_path, t.title, t.duration_ms
FROM tracks t
WHERE (t.mbid IS NULL OR t.mbid_source = 'acoustid')
AND t.missing_since IS NULL
AND t.id > sqlc.arg(after_id)::uuid
AND NOT EXISTS (SELECT 1 FROM track_acoustid_lookups l WHERE l.track_id = t.id)
ORDER BY t.id
LIMIT sqlc.arg(batch_limit);
-- name: RecordAcoustIDLookup :exec
INSERT INTO track_acoustid_lookups (track_id, state, best_score, candidates, recording_mbid, detail)
VALUES (sqlc.arg(track_id), sqlc.arg(state), sqlc.narg(best_score), sqlc.arg(candidates),
sqlc.narg(recording_mbid), sqlc.narg(detail))
ON CONFLICT (track_id) DO UPDATE SET
state = EXCLUDED.state,
best_score = EXCLUDED.best_score,
candidates = EXCLUDED.candidates,
recording_mbid = EXCLUDED.recording_mbid,
detail = EXCLUDED.detail,
looked_up_at = now();
-- name: SetTrackMbidFromAcoustID :execrows
-- Never over a tag id (D4): the guard is here, not only in the worker, so a
-- tag the scan read between the worker's list and this write still wins.
-- updated_at is left alone: it gates the scan's mtime skip, and moving it
-- past a file's mtime would hide a change made just before the lookup.
UPDATE tracks
SET mbid = sqlc.arg(mbid), mbid_source = 'acoustid'
WHERE id = sqlc.arg(id)
AND (mbid IS NULL OR mbid_source = 'acoustid');
-- name: ClearTrackAcoustIDMbid :exec
-- A re-lookup of changed bytes that no longer matches takes back the id an
-- earlier lookup wrote. A tag id is never touched.
UPDATE tracks
SET mbid = NULL, mbid_source = NULL
WHERE id = sqlc.arg(id)
AND mbid_source = 'acoustid';
-- name: DeleteAcoustIDLookup :exec
-- The scan drops the lookup of bytes that are gone, so the worker checks the
-- new ones.
DELETE FROM track_acoustid_lookups WHERE track_id = $1;
-- name: GetMbidCoverage :one
-- The admin gauge: present tracks by where their MBID came from, and the
-- lookup states of those a lookup has seen. pending is what the worker's
-- queue holds.
SELECT count(*)::bigint AS total,
count(*) FILTER (WHERE t.mbid_source = 'tag')::bigint AS from_tag,
count(*) FILTER (WHERE t.mbid_source = 'acoustid')::bigint AS from_acoustid,
count(*) FILTER (WHERE l.state = 'ambiguous')::bigint AS ambiguous,
count(*) FILTER (WHERE l.state = 'no_match')::bigint AS no_match,
count(*) FILTER (WHERE l.state = 'failed')::bigint AS failed,
count(*) FILTER (
WHERE l.track_id IS NULL AND (t.mbid IS NULL OR t.mbid_source = 'acoustid')
)::bigint AS pending
FROM tracks t
LEFT JOIN track_acoustid_lookups l ON l.track_id = t.id
WHERE t.missing_since IS NULL;
-- name: ListUnsettledAcoustIDLookups :many
-- The admin list of tracks a lookup could not settle (ambiguous or failed),
-- so they are findable rather than silently empty.
SELECT t.id, t.title, t.file_path, ar.name AS artist_name, al.title AS album_title,
l.state, l.best_score, l.candidates, l.detail, l.looked_up_at
FROM track_acoustid_lookups l
JOIN tracks t ON t.id = l.track_id
JOIN artists ar ON ar.id = t.artist_id
JOIN albums al ON al.id = t.album_id
WHERE l.state IN ('ambiguous', 'failed')
AND t.missing_since IS NULL
ORDER BY ar.name, al.title, t.disc_number NULLS FIRST, t.track_number NULLS FIRST, t.title
LIMIT sqlc.arg(row_limit);
-- name: GetAcoustIDSettings :one
SELECT * FROM acoustid_settings WHERE id = true;
-- name: UpdateAcoustIDSettings :one
-- The key is write-only from the admin card: set_api_key false keeps the
-- stored one, and an empty api_key with set_api_key true clears it.
UPDATE acoustid_settings
SET enabled = sqlc.arg(enabled),
min_score = sqlc.arg(min_score),
api_key = CASE WHEN sqlc.arg(set_api_key)::boolean
THEN NULLIF(sqlc.arg(api_key)::text, '')
ELSE api_key END,
updated_at = now()
WHERE id = true
RETURNING *;
+1 -1
View File
@@ -88,7 +88,7 @@ ON CONFLICT (track_a_id, track_b_id, source) DO NOTHING;
-- A recording MBID is what the similarity pipeline keys on. If only the removed
-- copy carried one, the survivor takes it rather than going dark to similarity.
UPDATE tracks AS survivor
SET mbid = loser.mbid
SET mbid = loser.mbid, mbid_source = loser.mbid_source
FROM tracks AS loser
WHERE survivor.id = sqlc.arg(survivor_id)::uuid
AND loser.id = sqlc.arg(loser_id)::uuid
+11 -4
View File
@@ -1,10 +1,13 @@
-- name: UpsertTrack :one
-- file_path is the canonical identity for library scan; mbid is secondary.
-- mbid is the file's tag. A looked-up id (mbid_source 'acoustid', M401)
-- survives a re-read that finds no tag id, and a tag id replaces it (D4).
INSERT INTO tracks (
title, album_id, artist_id, track_number, disc_number,
duration_ms, file_path, file_size, file_format, bitrate, mbid, genre,
tag_read_version
) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13)
tag_read_version, mbid_source
) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13,
CASE WHEN $11::text IS NOT NULL THEN 'tag' END)
ON CONFLICT (file_path) DO UPDATE SET
title = EXCLUDED.title,
album_id = EXCLUDED.album_id,
@@ -15,7 +18,11 @@ ON CONFLICT (file_path) DO UPDATE SET
file_size = EXCLUDED.file_size,
file_format = EXCLUDED.file_format,
bitrate = EXCLUDED.bitrate,
mbid = EXCLUDED.mbid,
-- Both read the row as it was before this statement.
mbid = CASE WHEN EXCLUDED.mbid IS NULL AND tracks.mbid_source = 'acoustid'
THEN tracks.mbid ELSE EXCLUDED.mbid END,
mbid_source = CASE WHEN EXCLUDED.mbid IS NOT NULL THEN 'tag'
WHEN tracks.mbid_source = 'acoustid' THEN 'acoustid' END,
genre = EXCLUDED.genre,
-- Stamped on update too, so a tag-repair pass marks rows as done and the
-- next scan can short-circuit them again (#2499).
@@ -39,7 +46,7 @@ SELECT id, file_path
-- Heal a track's recording MBID only while still NULL — idempotent, so
-- re-running the backfill is a no-op for already-healed rows.
UPDATE tracks
SET mbid = $2, updated_at = now()
SET mbid = $2, mbid_source = 'tag', updated_at = now()
WHERE id = $1 AND mbid IS NULL;
-- name: GetTrackByID :one
+454
View File
@@ -0,0 +1,454 @@
package library
import (
"context"
"errors"
"fmt"
"log/slog"
"strings"
"sync"
"time"
"unicode"
"github.com/jackc/pgx/v5"
"github.com/jackc/pgx/v5/pgtype"
"github.com/jackc/pgx/v5/pgxpool"
"git.fabledsword.com/bvandeusen/minstrel/internal/acoustid"
"git.fabledsword.com/bvandeusen/minstrel/internal/db/dbq"
)
// AcoustID lookup worker (M401 #3921).
//
// A track whose tags carry no recording MBID is invisible to the ListenBrainz
// similarity arm: it is never a seed and never comes back as a result. This
// worker fingerprints such tracks and asks AcoustID which recording they are,
// writing an id only when the answer is unambiguous (D5). The file's own tag
// always outranks it (D4).
//
// It never gates anything else (rule 164). With no key, switched off or
// AcoustID unreachable, it idles and says why through Status; scans and
// playback do not notice.
// acoustIDLookupTick is how often the worker looks for work. A settings save
// starts a pass at once, so this only paces new tracks and retries after an
// outage.
const acoustIDLookupTick = 15 * time.Minute
// acoustIDLookupBatch is how many tracks one query hands the worker.
const acoustIDLookupBatch = 50
// acoustIDLookupConcurrency is how many files are fingerprinted at once. The
// client serialises the requests themselves to AcoustID's rate, so this only
// overlaps fpcalc's decodes; low for the reason the loudness backfill's is.
const acoustIDLookupConcurrency = 2
// acoustIDDurationToleranceSec is how far a recording's MusicBrainz length may
// be from the file's and still count as the same take when telling candidates
// apart. A release's length and a rip's differ by a second or two of
// silence; a radio edit or a live take differs by far more.
const acoustIDDurationToleranceSec = 3
// Lookup states, as migration 0069's CHECK has them.
const (
lookupMatched = "matched"
lookupAmbiguous = "ambiguous"
lookupNoMatch = "no_match"
lookupFailed = "failed"
)
// recordingLookup is the AcoustID client as the worker uses it.
type recordingLookup interface {
Lookup(ctx context.Context, apiKey, fingerprint string, durationSec int) ([]acoustid.Candidate, error)
}
// AcoustIDLookupStatus is what the admin card shows about the worker.
type AcoustIDLookupStatus struct {
Running bool
LastPassAt time.Time
// Problem says why the last pass stopped short, in words for the card:
// the key was refused, or AcoustID could not be reached. Empty when the
// last pass ran to the end.
Problem string
}
// AcoustIDLookupResult tallies one pass.
type AcoustIDLookupResult struct {
Processed int
Matched int
Ambiguous int
NoMatch int
Failed int
Inconclusive int // nothing stored; tried again on a later pass
}
func (r *AcoustIDLookupResult) add(state string) {
r.Processed++
switch state {
case lookupMatched:
r.Matched++
case lookupAmbiguous:
r.Ambiguous++
case lookupNoMatch:
r.NoMatch++
case lookupFailed:
r.Failed++
default:
r.Inconclusive++
}
}
// errPassStopped ends a pass early: every further lookup would fail the same
// way, so asking again would only spend the rate limit.
type errPassStopped struct{ reason string }
func (e errPassStopped) Error() string { return e.reason }
// AcoustIDLookupWorker fills recording MBIDs through AcoustID.
type AcoustIDLookupWorker struct {
pool *pgxpool.Pool
logger *slog.Logger
settings *AcoustIDSettingsService
client recordingLookup
tick time.Duration
batch int32
// fingerprint is a field so an integration test pins which tracks a pass
// touches, not what fpcalc prints.
fingerprint func(ctx context.Context, path string) (lookupFingerprint, error)
kick chan struct{}
mu sync.Mutex
status AcoustIDLookupStatus
}
// NewAcoustIDLookupWorker builds a worker with the production cadence.
func NewAcoustIDLookupWorker(
pool *pgxpool.Pool, logger *slog.Logger, settings *AcoustIDSettingsService, client recordingLookup,
) *AcoustIDLookupWorker {
return &AcoustIDLookupWorker{
pool: pool,
logger: logger,
settings: settings,
client: client,
tick: acoustIDLookupTick,
batch: acoustIDLookupBatch,
fingerprint: computeLookupFingerprint,
kick: make(chan struct{}, 1),
}
}
// Kick asks for a pass now, for the admin card's "look up now". A pass
// already running absorbs it.
func (w *AcoustIDLookupWorker) Kick() {
if w == nil {
return
}
select {
case w.kick <- struct{}{}:
default:
}
}
// Settings is the settings service the worker reads, for the admin API. A
// nil worker has none, which serves the defaults.
func (w *AcoustIDLookupWorker) Settings() *AcoustIDSettingsService {
if w == nil {
return nil
}
return w.settings
}
// Status reports the worker's state for the admin card.
func (w *AcoustIDLookupWorker) Status() AcoustIDLookupStatus {
if w == nil {
return AcoustIDLookupStatus{}
}
w.mu.Lock()
defer w.mu.Unlock()
return w.status
}
// Run blocks until ctx is cancelled: one pass at start, then one per tick, on
// a kick, and after every settings save.
func (w *AcoustIDLookupWorker) Run(ctx context.Context) {
w.runOnce(ctx)
t := time.NewTicker(w.tick)
defer t.Stop()
for {
select {
case <-ctx.Done():
return
case <-t.C:
case <-w.kick:
case <-w.settings.Changed():
}
w.runOnce(ctx)
}
}
// runOnce contains a pass so that nothing it does (an error, a panic) can stop
// the next one from starting (rule 157).
func (w *AcoustIDLookupWorker) runOnce(ctx context.Context) {
w.setStatus(func(s *AcoustIDLookupStatus) { s.Running = true })
problem := ""
defer func() {
if r := recover(); r != nil {
w.logger.Error("acoustid lookup: pass panicked", "panic", r)
problem = "The last lookup pass stopped on an internal error; see the server log."
}
w.setStatus(func(s *AcoustIDLookupStatus) {
s.Running, s.LastPassAt, s.Problem = false, time.Now(), problem
})
}()
res, err := w.pass(ctx)
var stopped errPassStopped
switch {
case errors.As(err, &stopped):
problem = stopped.reason
w.logger.Warn("acoustid lookup: pass stopped", "reason", stopped.reason, "processed", res.Processed)
case err != nil && ctx.Err() == nil:
problem = "The last lookup pass failed; see the server log."
w.logger.Warn("acoustid lookup: pass failed", "err", err, "processed", res.Processed)
}
if res.Processed > 0 {
w.logger.Info("acoustid lookup: pass complete",
"processed", res.Processed, "matched", res.Matched, "ambiguous", res.Ambiguous,
"no_match", res.NoMatch, "failed", res.Failed, "inconclusive", res.Inconclusive)
}
}
func (w *AcoustIDLookupWorker) setStatus(f func(*AcoustIDLookupStatus)) {
w.mu.Lock()
f(&w.status)
w.mu.Unlock()
}
// pass walks the queue once, keyset-paged on id so it ends even when every
// attempt is inconclusive. Settings are read before every batch, so switching
// the lookup off or changing the threshold applies at the next batch.
func (w *AcoustIDLookupWorker) pass(ctx context.Context) (AcoustIDLookupResult, error) {
q := dbq.New(w.pool)
var res AcoustIDLookupResult
after := pgtype.UUID{Valid: true}
for {
if err := ctx.Err(); err != nil {
return res, err
}
cfg := w.settings.Get()
if !cfg.Ready() {
return res, nil
}
rows, err := q.ListTracksNeedingAcoustIDLookup(ctx, dbq.ListTracksNeedingAcoustIDLookupParams{
AfterID: after,
BatchLimit: w.batch,
})
if err != nil {
return res, fmt.Errorf("list tracks needing a lookup: %w", err)
}
if len(rows) == 0 {
return res, nil
}
if err := w.lookupBatch(ctx, cfg, rows, &res); err != nil {
return res, err
}
after = rows[len(rows)-1].ID
}
}
// lookupBatch looks up one batch, acoustIDLookupConcurrency at a time. The
// first track that stops the pass stops the batch: the tracks already started
// finish, no new ones start.
func (w *AcoustIDLookupWorker) lookupBatch(
ctx context.Context, cfg AcoustIDSettings, rows []dbq.ListTracksNeedingAcoustIDLookupRow, res *AcoustIDLookupResult,
) error {
ctx, cancel := context.WithCancelCause(ctx)
defer cancel(nil)
var (
mu sync.Mutex
wg sync.WaitGroup
sem = make(chan struct{}, acoustIDLookupConcurrency)
)
for _, row := range rows {
if ctx.Err() != nil {
break
}
sem <- struct{}{}
// The slot may have come free because a lookup just stopped the pass.
if ctx.Err() != nil {
<-sem
break
}
wg.Add(1)
go func(row dbq.ListTracksNeedingAcoustIDLookupRow) {
defer wg.Done()
defer func() { <-sem }()
defer func() {
if r := recover(); r != nil {
w.logger.Error("acoustid lookup: track panicked", "path", row.FilePath, "panic", r)
}
}()
state, err := w.lookupTrack(ctx, cfg, row)
if err != nil {
cancel(err)
}
mu.Lock()
res.add(state)
mu.Unlock()
}(row)
}
wg.Wait()
if cause := context.Cause(ctx); cause != nil && !errors.Is(cause, context.Canceled) {
return cause
}
return nil
}
// lookupTrack fingerprints one file, asks AcoustID and stores the answer. It
// returns the state stored ("" when nothing was), and an errPassStopped when
// no further lookup can succeed this pass.
func (w *AcoustIDLookupWorker) lookupTrack(
ctx context.Context, cfg AcoustIDSettings, row dbq.ListTracksNeedingAcoustIDLookupRow,
) (string, error) {
fp, err := w.fingerprint(ctx, row.FilePath)
if err != nil {
if isInconclusive(err) {
return "", nil
}
// fpcalc ran and rejected the file: a verdict, settled until it changes.
return w.store(ctx, row, lookupDecision{state: lookupFailed, detail: err.Error()}), nil
}
cands, err := w.client.Lookup(ctx, cfg.APIKey, fp.fingerprint, fp.durationSec)
switch {
case err == nil:
case errors.Is(err, acoustid.ErrInvalidKey):
return "", errPassStopped{"AcoustID refused the API key. Check it in Settings."}
case errors.Is(err, acoustid.ErrUnavailable):
return "", errPassStopped{"AcoustID could not be reached; lookups resume on the next pass."}
case errors.Is(err, acoustid.ErrInvalidFingerprint):
return w.store(ctx, row, lookupDecision{state: lookupFailed, detail: err.Error()}), nil
default:
// The caller's cancellation, or a request this client built wrongly.
// Neither is a verdict on the track.
if ctx.Err() == nil {
w.logger.Warn("acoustid lookup: lookup failed", "path", row.FilePath, "err", err)
}
return "", nil
}
return w.store(ctx, row, chooseRecording(cands, cfg.MinScore, row.Title, row.DurationMs)), nil
}
// store records a decision and applies it to tracks.mbid in one transaction,
// so the gauge never shows a match whose id was not written. A failed write
// stores nothing and the track is tried again.
func (w *AcoustIDLookupWorker) store(ctx context.Context, row dbq.ListTracksNeedingAcoustIDLookupRow, d lookupDecision) string {
err := pgx.BeginFunc(ctx, w.pool, func(tx pgx.Tx) error {
q := dbq.New(tx)
params := dbq.RecordAcoustIDLookupParams{
TrackID: row.ID, State: d.state, Candidates: int32(d.candidates),
}
if d.candidates > 0 {
params.BestScore = &d.bestScore
}
if d.recording != "" {
params.RecordingMbid = &d.recording
}
if d.detail != "" {
params.Detail = &d.detail
}
if err := q.RecordAcoustIDLookup(ctx, params); err != nil {
return err
}
if d.state == lookupMatched {
_, err := q.SetTrackMbidFromAcoustID(ctx, dbq.SetTrackMbidFromAcoustIDParams{ID: row.ID, Mbid: &d.recording})
return err
}
// A re-lookup of changed bytes that no longer matches takes back the
// id the earlier lookup wrote.
return q.ClearTrackAcoustIDMbid(ctx, row.ID)
})
if err != nil {
if ctx.Err() == nil {
w.logger.Warn("acoustid lookup: storing the result failed", "path", row.FilePath, "err", err)
}
return ""
}
return d.state
}
// lookupDecision is what one lookup settles on.
type lookupDecision struct {
state string
recording string // set only when matched
bestScore float32
candidates int
detail string
}
// chooseRecording applies D5 to AcoustID's candidates (best score first):
// only recordings at or above minScore count, one of them is a match, and
// several are told apart by the track's title and length or not at all. A
// wrong MBID would feed similarity the wrong neighbours, which is worse than
// none, so anything left unresolved is ambiguous and writes nothing.
func chooseRecording(cands []acoustid.Candidate, minScore float32, title string, durationMs int32) lookupDecision {
d := lookupDecision{candidates: len(cands)}
if len(cands) > 0 {
d.bestScore = float32(cands[0].Score)
}
var above []acoustid.Candidate
for _, c := range cands {
if float32(c.Score) >= minScore {
above = append(above, c)
}
}
switch len(above) {
case 0:
d.state = lookupNoMatch
return d
case 1:
d.state, d.recording = lookupMatched, above[0].ID
return d
}
want := normalizeTitle(title)
var same []acoustid.Candidate
for _, c := range above {
if normalizeTitle(c.Title) == want && sameLength(c.DurationSec, durationMs) {
same = append(same, c)
}
}
if len(same) == 1 {
d.state, d.recording = lookupMatched, same[0].ID
return d
}
d.state = lookupAmbiguous
return d
}
// sameLength reports whether a recording's length (0 when MusicBrainz has
// none) fits the file's. An unknown length on either side tells nothing, so
// it does not rule a candidate out.
func sameLength(recordingSec int, fileMs int32) bool {
if recordingSec <= 0 || fileMs <= 0 {
return true
}
diff := recordingSec*1000 - int(fileMs)
return diff <= acoustIDDurationToleranceSec*1000 && diff >= -acoustIDDurationToleranceSec*1000
}
// normalizeTitle compares titles by their letters and digits alone, case
// folded, so "Don't Stop" and "Dont stop" match while "Song (Radio Edit)"
// and "Song" do not.
func normalizeTitle(s string) string {
var b strings.Builder
for _, r := range strings.ToLower(s) {
if unicode.IsLetter(r) || unicode.IsDigit(r) {
b.WriteRune(r)
}
}
return b.String()
}
// AcoustIDCoverage reports where the library's recording MBIDs came from and
// what the lookups found, for the admin gauge.
func AcoustIDCoverage(ctx context.Context, pool *pgxpool.Pool) (dbq.GetMbidCoverageRow, error) {
return dbq.New(pool).GetMbidCoverage(ctx)
}
+371
View File
@@ -0,0 +1,371 @@
package library
import (
"context"
"errors"
"fmt"
"io"
"log/slog"
"path/filepath"
"strings"
"sync"
"testing"
"time"
"github.com/jackc/pgx/v5/pgtype"
"github.com/jackc/pgx/v5/pgxpool"
"git.fabledsword.com/bvandeusen/minstrel/internal/acoustid"
"git.fabledsword.com/bvandeusen/minstrel/internal/db/dbq"
"git.fabledsword.com/bvandeusen/minstrel/internal/dbtest"
)
func cand(id, title string, sec int, score float64) acoustid.Candidate {
return acoustid.Candidate{Recording: acoustid.Recording{ID: id, Title: title, DurationSec: sec}, Score: score}
}
// D5, as a table over what decides it: how many candidates clear the
// threshold, and whether title and length leave exactly one of them.
func TestChooseRecording(t *testing.T) {
const (
minScore = 0.85
title = "Don't Stop"
fileMs = 200_400
)
cases := []struct {
name string
cands []acoustid.Candidate
state string
rec string
}{
{"no candidates", nil, lookupNoMatch, ""},
{"all below the threshold", []acoustid.Candidate{cand("a", title, 200, 0.84)}, lookupNoMatch, ""},
{"one above the threshold", []acoustid.Candidate{cand("a", "Other title", 999, 0.9), cand("b", title, 200, 0.4)}, lookupMatched, "a"},
{"exactly at the threshold counts", []acoustid.Candidate{cand("a", title, 200, 0.85)}, lookupMatched, "a"},
{"several, one with the title and length",
[]acoustid.Candidate{cand("album", "Dont stop", 201, 0.95), cand("edit", "Don't Stop (Radio Edit)", 181, 0.95)},
lookupMatched, "album"},
{"several, title alike but only one length fits",
[]acoustid.Candidate{cand("live", title, 260, 0.9), cand("studio", title, 198, 0.9)},
lookupMatched, "studio"},
{"several with the same title and length",
[]acoustid.Candidate{cand("r1", title, 200, 0.95), cand("r2", title, 201, 0.95)},
lookupAmbiguous, ""},
{"several, none with the title",
[]acoustid.Candidate{cand("x", "Something", 200, 0.95), cand("y", "Else", 200, 0.95)},
lookupAmbiguous, ""},
// An unknown length rules nothing out, so it cannot break a tie on
// its own: both stay and the lookup is ambiguous.
{"unknown lengths do not disambiguate",
[]acoustid.Candidate{cand("r1", title, 0, 0.95), cand("r2", title, 0, 0.95)},
lookupAmbiguous, ""},
}
for _, c := range cases {
d := chooseRecording(c.cands, minScore, title, fileMs)
if d.state != c.state || d.recording != c.rec {
t.Errorf("%s: got %s %q, want %s %q", c.name, d.state, d.recording, c.state, c.rec)
}
if d.candidates != len(c.cands) {
t.Errorf("%s: candidates = %d, want %d", c.name, d.candidates, len(c.cands))
}
}
if d := chooseRecording([]acoustid.Candidate{cand("a", title, 200, 0.97)}, minScore, title, fileMs); d.bestScore != 0.97 {
t.Errorf("best score = %v, want 0.97", d.bestScore)
}
}
func TestNormalizeTitle(t *testing.T) {
if normalizeTitle("Don't Stop!") != normalizeTitle("dont stop") {
t.Error("punctuation and case should not separate titles")
}
if normalizeTitle("Song (Radio Edit)") == normalizeTitle("Song") {
t.Error("an edit's title must not match the original's")
}
if normalizeTitle("Ænima") != "ænima" {
t.Errorf("non-ASCII letters must survive: %q", normalizeTitle("Ænima"))
}
}
// fakeLookup answers by fingerprint. The worker passes each file's path
// through as its fingerprint (see testWorker), so a test keys answers by file.
type fakeLookup struct {
mu sync.Mutex
answers map[string][]acoustid.Candidate
err error
calls int
}
func (f *fakeLookup) Lookup(_ context.Context, _ string, fingerprint string, _ int) ([]acoustid.Candidate, error) {
f.mu.Lock()
defer f.mu.Unlock()
f.calls++
if f.err != nil {
return nil, f.err
}
return f.answers[fingerprint], nil
}
func testWorker(t *testing.T, pool *pgxpool.Pool, client recordingLookup) *AcoustIDLookupWorker {
t.Helper()
ctx := context.Background()
settings, err := NewAcoustIDSettingsService(ctx, pool)
if err != nil {
t.Fatal(err)
}
key := "test-key"
if _, err := settings.Set(ctx, AcoustIDSettingsUpdate{Enabled: true, MinScore: 0.85, APIKey: &key}); err != nil {
t.Fatal(err)
}
w := NewAcoustIDLookupWorker(pool, slog.New(slog.NewTextHandler(io.Discard, nil)), settings, client)
w.fingerprint = func(_ context.Context, path string) (lookupFingerprint, error) {
return lookupFingerprint{fingerprint: path, durationSec: 200}, nil
}
return w
}
func ptr(s string) *string { return &s }
// mbidOf reads a track's MBID and its source.
func mbidOf(t *testing.T, pool *pgxpool.Pool, id pgtype.UUID) (string, string) {
t.Helper()
var mbid, source *string
if err := pool.QueryRow(context.Background(),
`SELECT mbid, mbid_source FROM tracks WHERE id = $1`, id).Scan(&mbid, &source); err != nil {
t.Fatal(err)
}
deref := func(p *string) string {
if p == nil {
return ""
}
return *p
}
return deref(mbid), deref(source)
}
func addTrack(t *testing.T, q *dbq.Queries, album dbq.Album, artist dbq.Artist, title, path string, mbid *string) dbq.Track {
t.Helper()
tr, err := q.UpsertTrack(context.Background(), dbq.UpsertTrackParams{
Title: title, AlbumID: album.ID, ArtistID: artist.ID, DurationMs: 200_000,
FilePath: path, FileSize: 100, FileFormat: "mp3", Mbid: mbid,
})
if err != nil {
t.Fatalf("track %s: %v", title, err)
}
return tr
}
// D4: the file's tag outranks a lookup, in both directions. Falsified by
// restoring `mbid = EXCLUDED.mbid` in UpsertTrack: the first re-read below
// then erases the looked-up id.
func TestUpsertTrack_KeepsALookedUpMbidUntilATagCarriesOne_Integration(t *testing.T) {
pool := newPool(t)
ctx := context.Background()
q := dbq.New(pool)
dir := t.TempDir()
tr, album, artist := seedTrack(t, pool, filepath.Join(dir, "a.mp3"))
reread := func(mbid *string) {
t.Helper()
if _, err := q.UpsertTrack(ctx, dbq.UpsertTrackParams{
Title: tr.Title, AlbumID: album.ID, ArtistID: artist.ID, DurationMs: 1000,
FilePath: tr.FilePath, FileSize: 100, FileFormat: "mp3", Mbid: mbid,
}); err != nil {
t.Fatal(err)
}
}
if _, err := q.SetTrackMbidFromAcoustID(ctx, dbq.SetTrackMbidFromAcoustIDParams{ID: tr.ID, Mbid: ptr("rec-looked-up")}); err != nil {
t.Fatal(err)
}
reread(nil)
if m, s := mbidOf(t, pool, tr.ID); m != "rec-looked-up" || s != "acoustid" {
t.Errorf("after an untagged re-read: %q from %q, want the looked-up id kept", m, s)
}
reread(ptr("rec-from-tag"))
if m, s := mbidOf(t, pool, tr.ID); m != "rec-from-tag" || s != "tag" {
t.Errorf("after a tagged re-read: %q from %q, want the tag's id", m, s)
}
// A tag id is the file's to take away: removing the tag clears it, and a
// lookup cannot write over one.
if n, err := q.SetTrackMbidFromAcoustID(ctx, dbq.SetTrackMbidFromAcoustIDParams{ID: tr.ID, Mbid: ptr("rec-other")}); err != nil || n != 0 {
t.Errorf("a lookup wrote over a tag id: %d rows, %v", n, err)
}
reread(nil)
if m, s := mbidOf(t, pool, tr.ID); m != "" || s != "" {
t.Errorf("after the tag was removed: %q from %q, want none", m, s)
}
}
func TestAcoustIDLookup_FillsOnlyWhatItCanSettle_Integration(t *testing.T) {
pool := newPool(t)
ctx := context.Background()
q := dbq.New(pool)
dir := t.TempDir()
_, album, artist := seedTrack(t, pool, filepath.Join(dir, "seed.mp3"))
path := func(n string) string { return filepath.Join(dir, n+".mp3") }
matched := addTrack(t, q, album, artist, "Matched", path("matched"), nil)
ambiguous := addTrack(t, q, album, artist, "Twice", path("ambiguous"), nil)
none := addTrack(t, q, album, artist, "Unknown", path("none"), nil)
tagged := addTrack(t, q, album, artist, "Tagged", path("tagged"), ptr("rec-tag"))
fake := &fakeLookup{answers: map[string][]acoustid.Candidate{
path("matched"): {cand("rec-matched", "Matched", 200, 0.97)},
path("ambiguous"): {cand("rec-1", "Twice", 200, 0.95), cand("rec-2", "Twice", 200, 0.95)},
path("tagged"): {cand("rec-wrong", "Tagged", 200, 0.99)},
}}
w := testWorker(t, pool, fake)
res, err := w.pass(ctx)
if err != nil {
t.Fatal(err)
}
// seed.mp3 has no answer, so it settles as no_match beside "Unknown".
if res.Matched != 1 || res.Ambiguous != 1 || res.NoMatch != 2 {
t.Errorf("pass = %+v, want 1 matched, 1 ambiguous, 2 no match", res)
}
if m, s := mbidOf(t, pool, matched.ID); m != "rec-matched" || s != "acoustid" {
t.Errorf("matched track: %q from %q", m, s)
}
for _, tr := range []dbq.Track{ambiguous, none} {
if m, _ := mbidOf(t, pool, tr.ID); m != "" {
t.Errorf("%s got MBID %q from an unsettled lookup", tr.Title, m)
}
}
// The tagged track is never looked up, so the fake's wrong answer for it
// is never seen.
if m, s := mbidOf(t, pool, tagged.ID); m != "rec-tag" || s != "tag" {
t.Errorf("tagged track: %q from %q, want its tag untouched", m, s)
}
// Settled tracks are not looked up again.
before := fake.calls
if res, err := w.pass(ctx); err != nil || res.Processed != 0 || fake.calls != before {
t.Errorf("second pass = %+v (err %v, %d new calls), want nothing to do", res, err, fake.calls-before)
}
// The matched file changes and its new bytes no longer match: the scan
// drops the lookup, and the re-lookup takes the id back.
if err := q.DeleteAcoustIDLookup(ctx, matched.ID); err != nil {
t.Fatal(err)
}
fake.mu.Lock()
fake.answers[path("matched")] = nil
fake.mu.Unlock()
if _, err := w.pass(ctx); err != nil {
t.Fatal(err)
}
if m, s := mbidOf(t, pool, matched.ID); m != "" || s != "" {
t.Errorf("after a non-matching re-lookup: %q from %q, want the looked-up id taken back", m, s)
}
cov, err := AcoustIDCoverage(ctx, pool)
if err != nil {
t.Fatal(err)
}
if cov.Total != 5 || cov.FromTag != 1 || cov.FromAcoustid != 0 || cov.Ambiguous != 1 || cov.NoMatch != 3 || cov.Pending != 0 {
t.Errorf("coverage = %+v", cov)
}
}
func TestAcoustIDLookup_StopsWhenTheServiceCannotAnswer_Integration(t *testing.T) {
pool := newPool(t)
ctx := context.Background()
q := dbq.New(pool)
dir := t.TempDir()
_, album, artist := seedTrack(t, pool, filepath.Join(dir, "seed.mp3"))
for i := range 5 {
addTrack(t, q, album, artist, fmt.Sprint("T", i), filepath.Join(dir, fmt.Sprint(i, ".mp3")), nil)
}
for _, c := range []struct {
err error
problem string
}{
{fmt.Errorf("%w: invalid API key", acoustid.ErrInvalidKey), "refused the API key"},
{fmt.Errorf("%w: status 503", acoustid.ErrUnavailable), "could not be reached"},
} {
fake := &fakeLookup{err: c.err}
w := testWorker(t, pool, fake)
w.runOnce(ctx)
// Stopped at the first answer, not one failed call per track. Two may
// already be in flight at concurrency 2.
if fake.calls > acoustIDLookupConcurrency {
t.Errorf("%v: %d lookups made, want the pass to stop at the first", c.err, fake.calls)
}
if st := w.Status(); st.Running || !strings.Contains(st.Problem, c.problem) {
t.Errorf("%v: status = %+v, want a problem mentioning %q", c.err, st, c.problem)
}
var n int
if err := pool.QueryRow(ctx, `SELECT count(*) FROM track_acoustid_lookups`).Scan(&n); err != nil || n != 0 {
t.Errorf("%v: %d lookups stored (err %v); a service failure says nothing about a track", c.err, n, err)
}
}
}
// The point of the milestone (#3921): once a lookup fills an MBID, the
// similarity worker picks the track as a seed and can map ListenBrainz's
// answers back to it.
func TestAcoustIDLookup_FilledTrackReachesSimilarity_Integration(t *testing.T) {
pool := newPool(t)
ctx := context.Background()
q := dbq.New(pool)
dir := t.TempDir()
tr, _, _ := seedTrack(t, pool, filepath.Join(dir, "played.mp3"))
u, err := q.CreateUser(ctx, dbq.CreateUserParams{
Username: dbtest.TestUserPrefix + "acoustid", PasswordHash: "x", ApiTokenHash: "acoustid-token",
})
if err != nil {
t.Fatal(err)
}
now := pgtype.Timestamptz{Time: time.Now(), Valid: true}
session, err := q.InsertPlaySession(ctx, dbq.InsertPlaySessionParams{UserID: u.ID, StartedAt: now})
if err != nil {
t.Fatal(err)
}
if _, err := q.InsertPlayEvent(ctx, dbq.InsertPlayEventParams{
UserID: u.ID, TrackID: tr.ID, SessionID: session.ID, StartedAt: now,
}); err != nil {
t.Fatal(err)
}
seeds := func() int {
t.Helper()
rows, err := q.ListPlayedTracksNeedingSimilarity(ctx, 100)
if err != nil {
t.Fatal(err)
}
return len(rows)
}
if n := seeds(); n != 0 {
t.Fatalf("an untagged track is already a similarity seed (%d)", n)
}
fake := &fakeLookup{answers: map[string][]acoustid.Candidate{
tr.FilePath: {cand("rec-played", tr.Title, 1, 0.99)},
}}
if _, err := testWorker(t, pool, fake).pass(ctx); err != nil {
t.Fatal(err)
}
if n := seeds(); n != 1 {
t.Errorf("after the lookup, %d similarity seeds, want the played track", n)
}
got, err := q.GetTracksByMBIDs(ctx, []string{"rec-played"})
if err != nil || len(got) != 1 || got[0].ID != tr.ID {
t.Errorf("GetTracksByMBIDs = %+v (err %v), want the played track", got, err)
}
}
func TestAcoustIDLookup_IdleUntilReady(t *testing.T) {
var nilSvc *AcoustIDSettingsService
if nilSvc.Get().Ready() {
t.Error("the defaults are ready; the lookup must ship off")
}
for _, s := range []AcoustIDSettings{{Enabled: true}, {APIKey: "k"}} {
if s.Ready() {
t.Errorf("%+v reads as ready", s)
}
}
if _, err := nilSvc.Set(context.Background(), AcoustIDSettingsUpdate{MinScore: 0.2}); !errors.Is(err, ErrAcoustIDSettingOutOfRange) {
t.Errorf("min_score 0.2: err = %v, want out of range", err)
}
}
+132
View File
@@ -0,0 +1,132 @@
package library
import (
"context"
"errors"
"fmt"
"sync"
"time"
"github.com/jackc/pgx/v5/pgxpool"
"git.fabledsword.com/bvandeusen/minstrel/internal/db/dbq"
)
// AcoustID lookup settings (M401 #3922). Rule 25: a database row, changed
// without a restart, shared by the worker and the admin API.
// AcoustIDSettings mirrors the acoustid_settings row.
type AcoustIDSettings struct {
Enabled bool
// APIKey is the operator's registered application key. Never sent back
// to the browser; the API reports only whether one is set.
APIKey string
// MinScore is the AcoustID score a recording needs to be considered.
MinScore float32
// UpdatedAt is set by the database.
UpdatedAt time.Time
}
// Ready reports whether the worker may call AcoustID at all.
func (s AcoustIDSettings) Ready() bool { return s.Enabled && s.APIKey != "" }
// DefaultAcoustIDSettings mirrors migration 0069's defaults: off, no key.
var DefaultAcoustIDSettings = AcoustIDSettings{MinScore: 0.85}
// The score threshold's bounds, as migration 0069's CHECK has them. Below 0.5
// AcoustID's own docs call a match doubtful; the floor keeps a typo in the
// field from filling the library with guesses.
const (
minAcoustIDScore = 0.5
maxAcoustIDScore = 1
)
// ErrAcoustIDSettingOutOfRange is returned by Set for a value the CHECK would
// reject, so the API answers 400 naming the field.
var ErrAcoustIDSettingOutOfRange = errors.New("acoustid setting out of range")
// AcoustIDSettingsUpdate is the admin save. A nil APIKey keeps the stored key;
// an empty one clears it.
type AcoustIDSettingsUpdate struct {
Enabled bool
MinScore float32
APIKey *string
}
// AcoustIDSettingsService caches the settings and owns their persistence.
type AcoustIDSettingsService struct {
pool *pgxpool.Pool
mu sync.RWMutex
cur AcoustIDSettings
// changed is signalled after every save, so the worker can start a pass
// at once instead of waiting out its tick.
changed chan struct{}
}
// NewAcoustIDSettingsService loads once and caches. It always returns a usable
// service, holding the defaults (off) when the load fails; the error says so.
func NewAcoustIDSettingsService(ctx context.Context, pool *pgxpool.Pool) (*AcoustIDSettingsService, error) {
s := &AcoustIDSettingsService{pool: pool, cur: DefaultAcoustIDSettings, changed: make(chan struct{}, 1)}
row, err := dbq.New(pool).GetAcoustIDSettings(ctx)
if err != nil {
return s, fmt.Errorf("acoustid settings: load: %w", err)
}
s.cur = acoustIDSettingsFromRow(row)
return s, nil
}
// Get returns the cached settings. A nil service answers with the defaults.
func (s *AcoustIDSettingsService) Get() AcoustIDSettings {
if s == nil {
return DefaultAcoustIDSettings
}
s.mu.RLock()
defer s.mu.RUnlock()
return s.cur
}
// Set validates, persists and re-caches.
func (s *AcoustIDSettingsService) Set(ctx context.Context, in AcoustIDSettingsUpdate) (AcoustIDSettings, error) {
if in.MinScore < minAcoustIDScore || in.MinScore > maxAcoustIDScore {
return AcoustIDSettings{}, fmt.Errorf("%w: min_score must be %.2f-%.2f",
ErrAcoustIDSettingOutOfRange, float32(minAcoustIDScore), float32(maxAcoustIDScore))
}
if s == nil {
return AcoustIDSettings{}, errors.New("acoustid settings: no settings service")
}
params := dbq.UpdateAcoustIDSettingsParams{Enabled: in.Enabled, MinScore: in.MinScore}
if in.APIKey != nil {
params.SetApiKey = true
params.ApiKey = *in.APIKey
}
row, err := dbq.New(s.pool).UpdateAcoustIDSettings(ctx, params)
if err != nil {
return AcoustIDSettings{}, fmt.Errorf("acoustid settings: save: %w", err)
}
out := acoustIDSettingsFromRow(row)
s.mu.Lock()
s.cur = out
s.mu.Unlock()
select {
case s.changed <- struct{}{}:
default:
}
return out, nil
}
// Changed is signalled after a save. A nil service never signals.
func (s *AcoustIDSettingsService) Changed() <-chan struct{} {
if s == nil {
return nil
}
return s.changed
}
func acoustIDSettingsFromRow(row dbq.AcoustidSetting) AcoustIDSettings {
out := AcoustIDSettings{Enabled: row.Enabled, MinScore: row.MinScore, UpdatedAt: row.UpdatedAt.Time}
if row.ApiKey != nil {
out.APIKey = *row.ApiKey
}
return out
}
+1 -1
View File
@@ -69,7 +69,7 @@ func newMergeFixture(t *testing.T) mergeFixture {
}
}
// Only the copy being removed carries a recording MBID.
mustExec(`UPDATE tracks SET mbid = 'rec-www' WHERE id = $1`, f.remove.ID)
mustExec(`UPDATE tracks SET mbid = 'rec-www', mbid_source = 'tag' WHERE id = $1`, f.remove.ID)
user := func(name string) dbq.User {
t.Helper()
+12
View File
@@ -270,6 +270,18 @@ func computeChromaprint(ctx context.Context, path string, lengthSec int32) ([]in
return parseFpcalcRaw(out)
}
// computeLookupFingerprint returns the compressed fingerprint and duration an
// AcoustID lookup needs. It fails as runFingerprintTool does, so a stall or a
// missing fpcalc reads as inconclusive (isInconclusive) and a file fpcalc
// rejects as a verdict.
func computeLookupFingerprint(ctx context.Context, path string) (lookupFingerprint, error) {
out, err := runFingerprintTool(ctx, "fpcalc", fpcalcLookupArgs(path))
if err != nil {
return lookupFingerprint{}, err
}
return parseFpcalcCompressed(out)
}
// runFingerprintTool runs one tool under fingerprintTimeout.
//
// Any non-zero exit is an error, and that deliberately includes fpcalc's exit 3:
+6
View File
@@ -409,6 +409,12 @@ func (s *Scanner) scanFile(
if err := q.DeleteTrackLoudness(ctx, track.ID); err != nil {
s.logger.Warn("loudness: clearing stale measurement failed", "path", path, "err", err)
}
// Likewise an AcoustID lookup (M401): the worker looks the new bytes
// up again. A looked-up MBID stays meanwhile and is taken back only if
// the new lookup does not match it.
if err := q.DeleteAcoustIDLookup(ctx, track.ID); err != nil {
s.logger.Warn("acoustid: clearing stale lookup failed", "path", path, "err", err)
}
}
if knownTrack {
+13 -1
View File
@@ -108,6 +108,10 @@ type Server struct {
// LoudnessSettings is the DB-backed loudness analysis policy (M464 #4995),
// shared with the loudness backfill. Nil makes the router load its own.
LoudnessSettings *library.LoudnessSettingsService
// AcoustIDLookup is the AcoustID lookup worker (M401), whose settings,
// status and "look up now" the admin card drives. Nil in test contexts;
// one is built (and never run) so the admin routes still answer.
AcoustIDLookup *library.AcoustIDLookupWorker
// ReacqSettings is the DB-backed missing-file re-acquisition policy
// (milestone #290) — the same instance the sweeper in cmd/minstrel/main.go
// reads, so a save from the admin card reaches it without a restart
@@ -225,7 +229,15 @@ func (s *Server) Router() http.Handler {
s.Logger.Warn("loudness settings unavailable; serving defaults", "err", err)
}
}
api.Mount(r, s.Pool, s.Logger, writer, s.RecommendationCfg, recSettings, lidarrCfg, lidarrReqs, lidarrQuar, tracksSvc, playlistsSvc, s.CoverEnricher, s.CoverSettings, s.TagSettings, s.LibraryScanner, s.ScanCfg, s.DataDir, smtpSender, bus, s.PlaylistScheduler, s.StreamSecret, netSettings, reacqSettings, fpSettings, loudSettings)
acoustIDLookup := s.AcoustIDLookup
if acoustIDLookup == nil {
acoustIDSettings, err := library.NewAcoustIDSettingsService(context.Background(), s.Pool)
if err != nil {
s.Logger.Warn("acoustid settings unavailable; serving defaults", "err", err)
}
acoustIDLookup = library.NewAcoustIDLookupWorker(s.Pool, s.Logger, acoustIDSettings, nil)
}
api.Mount(r, s.Pool, s.Logger, writer, s.RecommendationCfg, recSettings, lidarrCfg, lidarrReqs, lidarrQuar, tracksSvc, playlistsSvc, s.CoverEnricher, s.CoverSettings, s.TagSettings, s.LibraryScanner, s.ScanCfg, s.DataDir, smtpSender, bus, s.PlaylistScheduler, s.StreamSecret, netSettings, reacqSettings, fpSettings, loudSettings, acoustIDLookup)
// /api/admin/scan is the only admin route owned by the server package
// (it needs the Scanner). Register it as a single inline-middleware
// route — using r.Route("/api/admin", ...) here would create a second