bvandeusen/minstrel
1
0
Fork 0
Code Issues 2 Pull Requests Actions Packages Projects Releases 52 Wiki Activity

feat(library): AcoustID lookup client and the compressed fpcalc print (M401 #3919)

Browse Source 
internal/acoustid posts AcoustID's v2 lookup as a gzip form with
meta=recordings, one request per 400ms (their limit is 3/s) and a 20s
deadline. It returns every linked recording with its best score; choosing
among them is the worker's job (D5). The server's error codes map to an
invalid key (stop and say so), a rejected fingerprint (a verdict on the
track) or unavailable (try again later). A cancelled caller stays a
cancellation.

fpcalcLookupArgs and parseFpcalcCompressed read fpcalc's default output,
the compressed string the lookup takes (D1), always over the first 120s.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
bvandeusen
2026-10-06 23:16:52 -04:00
co-authored by Claude Opus 5.5
parent b28cbe0600
commit f13da62797
4 changed files with 621 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/acoustid/client.go
+267
View File
@@ -0,0 +1,267 @@
// Package acoustid looks up MusicBrainz recording ids by acoustic fingerprint
// through the AcoustID web service (M401).
//
// What it relies on, from https://acoustid.org/webservice (read 2026-10-06):
// - POST https://api.acoustid.org/v2/lookup with client, duration (whole
// seconds), fingerprint (fpcalc's compressed string) and meta. A gzip body
// with Content-Encoding: gzip is accepted and preferred, because
// fingerprints are long.
// - meta=recordings returns each recording's title, duration and artists,
// which the worker needs to tell several candidates apart (M401 D5).
// - "Do not make more than 3 requests per second."
// - The service is free for non-commercial use only.
//
// Error codes are the server's own, from acoustid-server's
// acoustid/api/errors.py.
package acoustid
import (
"bytes"
"cmp"
"compress/gzip"
"context"
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
"net/url"
"slices"
"strconv"
"sync"
"time"
)
// DefaultBaseURL is the lookup endpoint.
const DefaultBaseURL = "https://api.acoustid.org/v2/lookup"
const (
// minInterval keeps to AcoustID's 3 requests a second with headroom: one
// every 400ms is 2.5 a second.
minInterval = 400 * time.Millisecond
// lookupTimeout bounds one request, connection included (rule 156). A
// lookup answers in well under a second; 20s is the line past which slow
// has become never.
lookupTimeout = 20 * time.Second
// bodyLimit caps the response read. A lookup with meta=recordings for a
// popular song runs to tens of kilobytes.
bodyLimit = 2 << 20
userAgent = "Minstrel ( https://git.fabledsword.com/bvandeusen/minstrel )"
)
// The server's error codes that the worker treats differently.
const (
codeInvalidFingerprint = 3
codeInvalidAPIKey = 4
codeServiceUnavailable = 13
codeTooManyRequests = 14
codeUnknownApplication = 17
)
var (
// ErrInvalidKey: the operator's application key was refused. No lookup
// can succeed until it is changed, so the worker stops and says so.
ErrInvalidKey = errors.New("acoustid: the API key was refused")
// ErrInvalidFingerprint: AcoustID rejected this track's fingerprint. A
// verdict on the track, not on the service.
ErrInvalidFingerprint = errors.New("acoustid: the fingerprint was rejected")
// ErrUnavailable: the service could not answer now (unreachable, timed
// out, overloaded or rate limiting). Says nothing about the track, so the
// lookup is tried again later.
ErrUnavailable = errors.New("acoustid: the service is unavailable")
)
// Recording is one MusicBrainz recording a fingerprint matched.
type Recording struct {
ID string
Title string
// DurationSec is MusicBrainz's length for the recording; 0 when unknown.
DurationSec int
Artists []string
}
// Candidate is a recording with the score of the best AcoustID result that
// linked to it.
type Candidate struct {
Recording
Score float64
}
// Client calls the lookup endpoint, at most one request per minInterval.
type Client struct {
baseURL string
http *http.Client
mu sync.Mutex
lastCall time.Time
}
// New returns a client for baseURL (DefaultBaseURL in production).
func New(baseURL string) *Client {
return &Client{baseURL: baseURL, http: &http.Client{Timeout: lookupTimeout}}
}
// Lookup asks which recordings match fingerprint, a compressed chromaprint
// of audio durationSec long. It returns every candidate, best score first;
// choosing among them is the caller's job. No match is an empty slice and
// no error.
func (c *Client) Lookup(ctx context.Context, apiKey, fingerprint string, durationSec int) ([]Candidate, error) {
if err := c.wait(ctx); err != nil {
return nil, err
}
body, err := gzipForm(url.Values{
"client": {apiKey},
"duration": {strconv.Itoa(durationSec)},
"fingerprint": {fingerprint},
"meta": {"recordings"},
"format": {"json"},
})
if err != nil {
return nil, err
}
reqCtx, cancel := context.WithTimeout(ctx, lookupTimeout)
defer cancel()
req, err := http.NewRequestWithContext(reqCtx, http.MethodPost, c.baseURL, bytes.NewReader(body))
if err != nil {
return nil, fmt.Errorf("acoustid: build request: %w", err)
}
req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
req.Header.Set("Content-Encoding", "gzip")
req.Header.Set("Accept", "application/json")
req.Header.Set("User-Agent", userAgent)
resp, err := c.http.Do(req)
if err != nil {
// The caller giving up is not the service failing.
if ctx.Err() != nil {
return nil, ctx.Err()
}
return nil, fmt.Errorf("%w: %v", ErrUnavailable, err)
}
defer func() { _ = resp.Body.Close() }()
raw, err := io.ReadAll(io.LimitReader(resp.Body, bodyLimit))
if err != nil {
return nil, fmt.Errorf("%w: read body: %v", ErrUnavailable, err)
}
return parseLookup(resp.StatusCode, raw)
}
// wait blocks until minInterval has passed since the last request, then
// claims the slot.
func (c *Client) wait(ctx context.Context) error {
c.mu.Lock()
defer c.mu.Unlock()
if d := time.Until(c.lastCall.Add(minInterval)); d > 0 {
t := time.NewTimer(d)
defer t.Stop()
select {
case <-ctx.Done():
return ctx.Err()
case <-t.C:
}
}
c.lastCall = time.Now()
return nil
}
func gzipForm(v url.Values) ([]byte, error) {
var buf bytes.Buffer
zw := gzip.NewWriter(&buf)
if _, err := zw.Write([]byte(v.Encode())); err != nil {
return nil, fmt.Errorf("acoustid: compress request: %w", err)
}
if err := zw.Close(); err != nil {
return nil, fmt.Errorf("acoustid: compress request: %w", err)
}
return buf.Bytes(), nil
}
type lookupResponse struct {
Status string `json:"status"`
Error *struct {
Code int `json:"code"`
Message string `json:"message"`
} `json:"error"`
Results []struct {
ID string `json:"id"`
Score float64 `json:"score"`
Recordings []struct {
ID string `json:"id"`
Title string `json:"title"`
Duration float64 `json:"duration"`
Artists []struct {
Name string `json:"name"`
} `json:"artists"`
} `json:"recordings"`
} `json:"results"`
}
// parseLookup reads a lookup response. AcoustID reports its errors in the
// body with a code, alongside a 4xx or 5xx status, so the body is read first
// and the status only decides when the body says nothing usable.
func parseLookup(status int, raw []byte) ([]Candidate, error) {
var r lookupResponse
if err := json.Unmarshal(raw, &r); err != nil {
if status != http.StatusOK {
return nil, fmt.Errorf("%w: status %d", ErrUnavailable, status)
}
return nil, fmt.Errorf("%w: unreadable response: %v", ErrUnavailable, err)
}
if r.Status != "ok" {
return nil, lookupError(status, r)
}
return candidates(r), nil
}
func lookupError(status int, r lookupResponse) error {
if r.Error == nil {
return fmt.Errorf("%w: status %d", ErrUnavailable, status)
}
switch r.Error.Code {
case codeInvalidAPIKey, codeUnknownApplication:
return fmt.Errorf("%w: %s", ErrInvalidKey, r.Error.Message)
case codeInvalidFingerprint:
return fmt.Errorf("%w: %s", ErrInvalidFingerprint, r.Error.Message)
case codeServiceUnavailable, codeTooManyRequests:
return fmt.Errorf("%w: %s", ErrUnavailable, r.Error.Message)
}
if status >= http.StatusInternalServerError {
return fmt.Errorf("%w: %s", ErrUnavailable, r.Error.Message)
}
// Any other code is a request this client built wrongly. Retrying would
// repeat it, so it is named rather than folded into unavailable.
return fmt.Errorf("acoustid: error %d: %s", r.Error.Code, r.Error.Message)
}
// candidates flattens the results into one entry per recording. The same
// recording can hang off several AcoustID results (one audio, fingerprinted
// from different sources); it keeps the best score it was seen with.
// Results with no linked recording are fingerprints AcoustID knows but nobody
// has tied to MusicBrainz, and carry nothing to fill.
func candidates(r lookupResponse) []Candidate {
out := []Candidate{}
at := map[string]int{}
for _, res := range r.Results {
for _, rec := range res.Recordings {
if rec.ID == "" {
continue
}
if i, ok := at[rec.ID]; ok {
out[i].Score = max(out[i].Score, res.Score)
continue
}
c := Candidate{
Recording: Recording{ID: rec.ID, Title: rec.Title, DurationSec: int(rec.Duration + 0.5)},
Score: res.Score,
}
for _, a := range rec.Artists {
c.Artists = append(c.Artists, a.Name)
}
at[rec.ID] = len(out)
out = append(out, c)
}
}
// Best first, keeping AcoustID's order among equals.
slices.SortStableFunc(out, func(a, b Candidate) int { return cmp.Compare(b.Score, a.Score) })
return out
}
internal/acoustid/client_test.go
+220
View File
@@ -0,0 +1,220 @@
package acoustid
import (
"compress/gzip"
"context"
"errors"
"io"
"net/http"
"net/http/httptest"
"net/url"
"strings"
"testing"
"time"
)
// docSample is the meta=recordings response printed in AcoustID's own
// documentation (https://acoustid.org/webservice), artists and release groups
// as given.
const docSample = `{
"status": "ok",
"results": [{
"score": 1.0,
"id": "9ff43b6a-4f16-427c-93c2-92307ca505e0",
"recordings": [{
"duration": 639,
"releasegroups": [{"type": "Album", "id": "ddaa2d4d-314e-3e7c-b1d0-f6d207f5aa2f", "title": "Before the Dawn Heals Us"}],
"title": "Lower Your Eyelids to Die With the Sun",
"id": "cd2e7c47-16f5-46c6-a37c-a1eb7bf599ff",
"artists": [{"id": "6d7b7cd4-254b-4c25-83f6-dd20f98ceacd", "name": "M83"}]
}]
}]
}`
// severalSample is the shape the worker has to disambiguate: one audio linked
// to two recordings (an album and a single release), the album recording
// reached again through a weaker second result, and a third result that
// AcoustID knows but nobody has tied to MusicBrainz.
const severalSample = `{
"status": "ok",
"results": [
{"score": 0.71, "id": "a2", "recordings": [{"id": "rec-album", "title": "Song", "duration": 200.4}]},
{"score": 0.93, "id": "a1", "recordings": [
{"id": "rec-album", "title": "Song", "duration": 200.4, "artists": [{"name": "A"}, {"name": "B"}]},
{"id": "rec-single", "title": "Song (radio edit)", "duration": 181}
]},
{"score": 0.88, "id": "a3"}
]
}`
func TestParseLookup(t *testing.T) {
cases := []struct {
name string
body string
want []Candidate
}{
{"single match", docSample, []Candidate{{
Recording: Recording{
ID: "cd2e7c47-16f5-46c6-a37c-a1eb7bf599ff", Title: "Lower Your Eyelids to Die With the Sun",
DurationSec: 639, Artists: []string{"M83"},
},
Score: 1.0,
}}},
{"several recordings", severalSample, []Candidate{
{Recording: Recording{ID: "rec-album", Title: "Song", DurationSec: 200}, Score: 0.93},
{Recording: Recording{ID: "rec-single", Title: "Song (radio edit)", DurationSec: 181}, Score: 0.93},
}},
// Choosing a threshold is the worker's; the parser carries the score.
{"below threshold", `{"status":"ok","results":[{"score":0.31,"id":"x","recordings":[{"id":"rec"}]}]}`,
[]Candidate{{Recording: Recording{ID: "rec"}, Score: 0.31}}},
{"no results", `{"status":"ok","results":[]}`, []Candidate{}},
{"results with no recordings", `{"status":"ok","results":[{"score":0.99,"id":"x"}]}`, []Candidate{}},
}
for _, c := range cases {
got, err := parseLookup(http.StatusOK, []byte(c.body))
if err != nil {
t.Errorf("%s: err = %v", c.name, err)
continue
}
if !sameCandidates(got, c.want) {
t.Errorf("%s:\n got %+v\nwant %+v", c.name, got, c.want)
}
}
}
// The first result's recording is the album one with both artists; the
// weaker repeat must not replace its fields, only fail to lower its score.
func TestParseLookup_RepeatedRecordingKeepsBestScoreAndFirstFields(t *testing.T) {
got, err := parseLookup(http.StatusOK, []byte(severalSample))
if err != nil {
t.Fatal(err)
}
if got[0].ID != "rec-album" || got[0].Score != 0.93 {
t.Errorf("album recording = %+v, want score 0.93", got[0])
}
}
func TestParseLookup_Errors(t *testing.T) {
cases := []struct {
name string
status int
body string
want error
}{
{"invalid key", 400, `{"status":"error","error":{"code":4,"message":"invalid API key"}}`, ErrInvalidKey},
{"unknown application", 400, `{"status":"error","error":{"code":17,"message":"unknown application"}}`, ErrInvalidKey},
{"bad fingerprint", 400, `{"status":"error","error":{"code":3,"message":"invalid fingerprint"}}`, ErrInvalidFingerprint},
{"rate limited", 429, `{"status":"error","error":{"code":14,"message":"rate limit (3.000000 requests per second) exceeded, try again later"}}`, ErrUnavailable},
{"down", 503, `{"status":"error","error":{"code":13,"message":"service currently unavailable, try again later"}}`, ErrUnavailable},
{"internal error", 500, `{"status":"error","error":{"code":5,"message":"internal error"}}`, ErrUnavailable},
{"proxy page", 502, `<html>Bad Gateway</html>`, ErrUnavailable},
}
for _, c := range cases {
_, err := parseLookup(c.status, []byte(c.body))
if !errors.Is(err, c.want) {
t.Errorf("%s: err = %v, want %v", c.name, err, c.want)
}
}
// A code this client caused (here a missing parameter) is neither the
// key nor the service, so it must not read as either: retrying repeats it.
_, err := parseLookup(400, []byte(`{"status":"error","error":{"code":2,"message":"missing required parameter \"fingerprint\""}}`))
if err == nil || errors.Is(err, ErrUnavailable) || errors.Is(err, ErrInvalidKey) || !strings.Contains(err.Error(), "missing required parameter") {
t.Errorf("missing parameter: err = %v", err)
}
}
// The request is what AcoustID documents: a gzip form POST carrying the key,
// whole-second duration, fingerprint and meta=recordings.
func TestLookup_SendsACompressedForm(t *testing.T) {
var form url.Values
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodPost || r.Header.Get("Content-Encoding") != "gzip" {
t.Errorf("request %s with Content-Encoding %q, want a gzip POST", r.Method, r.Header.Get("Content-Encoding"))
}
zr, err := gzip.NewReader(r.Body)
if err != nil {
t.Fatalf("body is not gzip: %v", err)
}
raw, _ := io.ReadAll(zr)
form, _ = url.ParseQuery(string(raw))
_, _ = io.WriteString(w, docSample)
}))
defer srv.Close()
got, err := New(srv.URL).Lookup(context.Background(), "key123", "AQADtE", 241)
if err != nil || len(got) != 1 {
t.Fatalf("Lookup = %+v, %v", got, err)
}
for k, want := range map[string]string{
"client": "key123", "duration": "241", "fingerprint": "AQADtE", "meta": "recordings", "format": "json",
} {
if form.Get(k) != want {
t.Errorf("form %s = %q, want %q", k, form.Get(k), want)
}
}
}
func TestLookup_DeadlineIsUnavailable(t *testing.T) {
release := make(chan struct{})
srv := httptest.NewServer(http.HandlerFunc(func(http.ResponseWriter, *http.Request) { <-release }))
defer srv.Close()
defer close(release)
c := New(srv.URL)
c.http.Timeout = 50 * time.Millisecond
_, err := c.Lookup(context.Background(), "k", "f", 1)
if !errors.Is(err, ErrUnavailable) {
t.Errorf("err = %v, want ErrUnavailable", err)
}
}
// A cancelled caller is reported as such, never as the service being down.
func TestLookup_CancelledCallerIsNotUnavailable(t *testing.T) {
release := make(chan struct{})
srv := httptest.NewServer(http.HandlerFunc(func(http.ResponseWriter, *http.Request) { <-release }))
defer srv.Close()
defer close(release)
ctx, cancel := context.WithTimeout(context.Background(), 50*time.Millisecond)
defer cancel()
_, err := New(srv.URL).Lookup(ctx, "k", "f", 1)
if !errors.Is(err, context.DeadlineExceeded) || errors.Is(err, ErrUnavailable) {
t.Errorf("err = %v, want the caller's own deadline", err)
}
}
func TestLookup_KeepsToTheRateLimit(t *testing.T) {
var at []time.Time
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
at = append(at, time.Now())
_, _ = io.WriteString(w, `{"status":"ok","results":[]}`)
}))
defer srv.Close()
c := New(srv.URL)
for range 3 {
if _, err := c.Lookup(context.Background(), "k", "f", 1); err != nil {
t.Fatal(err)
}
}
for i := 1; i < len(at); i++ {
// A little slack for the server's own clock reading.
if gap := at[i].Sub(at[i-1]); gap < minInterval-10*time.Millisecond {
t.Errorf("requests %d and %d were %s apart, want at least %s", i-1, i, gap, minInterval)
}
}
}
func sameCandidates(a, b []Candidate) bool {
if len(a) != len(b) {
return false
}
for i := range a {
if a[i].ID != b[i].ID || a[i].Title != b[i].Title || a[i].DurationSec != b[i].DurationSec ||
a[i].Score != b[i].Score || strings.Join(a[i].Artists, "|") != strings.Join(b[i].Artists, "|") {
return false
}
}
return true
}