Files
minstrel/internal/similarity/worker.go
T
bvandeusenandClaude Opus 5.5 55c7497ca7
release / go (push) Successful in 1m54s
release / web (push) Failing after 5s
release / integration (push) Failing after 3m44s
release / govulncheck (push) Successful in 20s
release / Attach APK to the Release (tag releases only) (push) Canceled after 0s
release / Build + push container image (push) Canceled after 0s
release / Verify release artifacts (tag releases only) (push) Canceled after 0s
release / Build signed APK (push) Canceled after 19s
release / android (push) Canceled after 2m37s
feat: Minstrel stops acquiring music and drops Discover
Minstrel is now a viewer of the library it is pointed at. Finding,
downloading and filing music moves out of it.

Removed from the server:
- the request flow (requests, approval, auto-approve) and the wanted list,
  follows, release profiles and release blocks;
- searching indexers, the download clients, the importer, the album
  lifecycle and missing-file re-acquisition;
- the catalogue's discography, search and art (the MusicBrainz release and
  track-list cache stays, for the duplicate resolver);
- Discover: its suggestions of artists not in the library, snoozes, the
  candidate-artist tag enrichment and its tuning scope;
- writes to acquisition tables from moving the library folder and from
  merging duplicates, the want-list join on Suspect sources, the sealing of
  acquisition credentials, and the release-group lookup only
  re-acquisition used.

Settling a wrong-slot duplicate no longer sets the release aside: the
copies go and their slots show as missing until a good copy is in the
library.

Removed from the web: Admin > Acquisition and Requests, /requests,
/discover, Follow on the artist page, the auto-approve toggle, and the
Collection's wanted-list tabs. The Collection is now the library's health:
an overview, Reports, Duplicates, Suspect sources and Missing files. The
README and docs no longer describe getting music.

The tables stay for now; a later migration drops them. The request
notification kinds and the Android screens are still to go.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-21 20:51:14 -05:00

255 lines
7.9 KiB
Go

// Package similarity owns the inbound ListenBrainz similarity ingest
// pipeline. A periodic worker queries the LB Labs API
// (labs.api.listenbrainz.org similar-recordings / similar-artists) for
// tracks the user has played.
//
// For recordings it keeps LB's whole answer in listenbrainz_similar_recordings
// and derives the track_similarity edges from it in SQL, against whatever is
// in the library at the time, so a recording that reaches the library later
// (an import, or an MBID the AcoustID lookup fills in) gains its edges without
// asking LB again. For artists it stores the in-library matches in
// artist_similarity.
//
// Every answer is recorded as a fetch, an empty one included, and the queue
// reads those records rather than the edges: a seed that matched nothing is
// still fresh and does not hold the head of the queue.
package similarity
import (
"context"
"errors"
"fmt"
"log/slog"
"sort"
"time"
"github.com/jackc/pgx/v5"
"github.com/jackc/pgx/v5/pgtype"
"github.com/jackc/pgx/v5/pgxpool"
"git.fabledsword.com/bvandeusen/minstrel/internal/db/dbq"
"git.fabledsword.com/bvandeusen/minstrel/internal/scrobble/listenbrainz"
)
// Worker drains played tracks and artists needing similarity. A transient
// failure is not recorded as a fetch, so the timer retries it next tick.
type Worker struct {
pool *pgxpool.Pool
client *listenbrainz.Client
logger *slog.Logger
tick time.Duration
batch int32
topK int // per artist
}
// NewWorker constructs a worker with production defaults: 1h tick,
// batch=25, topK=20. batch is tracks AND artists processed per tick, well
// under ListenBrainz rate limits (429s abort the tick). With never-fetched
// seeds first, a freshly played library is covered in days.
func NewWorker(pool *pgxpool.Pool, client *listenbrainz.Client, logger *slog.Logger) *Worker {
return &Worker{
pool: pool,
client: client,
logger: logger,
tick: 1 * time.Hour,
batch: 25,
topK: 20,
}
}
// Run blocks until ctx is cancelled, ticking every w.tick.
func (w *Worker) Run(ctx context.Context) {
t := time.NewTicker(w.tick)
defer t.Stop()
for {
select {
case <-ctx.Done():
return
case <-t.C:
if err := w.tickOnce(ctx); err != nil {
w.logger.Error("similarity: tick failed", "err", err)
}
}
}
}
// tickOnce drains one batch of tracks and one batch of artists, then
// re-resolves every cached answer against the library. Per-row errors are
// logged and skipped. 429 aborts the fetching, not the resolve.
func (w *Worker) tickOnce(ctx context.Context) error {
q := dbq.New(w.pool)
if err := w.tickTracks(ctx, q); err != nil {
return err
}
if err := w.tickArtists(ctx, q); err != nil {
return err
}
// Links recordings that reached the library since their seed was fetched:
// a scan, an import, an MBID from the AcoustID lookup.
if err := q.ResolveListenBrainzTrackEdges(ctx, nil); err != nil {
return fmt.Errorf("resolve similarity edges: %w", err)
}
return nil
}
func (w *Worker) tickTracks(ctx context.Context, q *dbq.Queries) error {
rows, err := q.ListPlayedTracksNeedingSimilarity(ctx, w.batch)
if err != nil {
return err
}
for _, r := range rows {
if r.Mbid == nil {
continue // defensive — query already filters NULL
}
results, err := w.client.SimilarRecordings(ctx, *r.Mbid, 100)
switch {
case err == nil:
case isRateLimited(err):
w.logger.Warn("similarity: 429 — aborting tick")
return nil
case errors.Is(err, listenbrainz.ErrPermanent):
// LB refused this MBID outright; asking again next hour won't
// change that. Recorded as an empty answer so it waits its turn.
w.logger.Warn("similarity: similar-recordings refused", "track_id", r.ID, "err", err)
results = nil
default:
w.logger.Warn("similarity: similar-recordings failed", "track_id", r.ID, "err", err)
continue
}
if err := w.storeTrackSimilar(ctx, r.ID, results); err != nil {
w.logger.Warn("similarity: store similar recordings", "track_id", r.ID, "err", err)
}
}
return nil
}
func (w *Worker) tickArtists(ctx context.Context, q *dbq.Queries) error {
rows, err := q.ListPlayedArtistsNeedingSimilarity(ctx, w.batch)
if err != nil {
return err
}
for _, r := range rows {
if r.Mbid == nil {
continue
}
results, err := w.client.SimilarArtists(ctx, *r.Mbid, 100)
switch {
case err == nil:
case isRateLimited(err):
w.logger.Warn("similarity: 429 — aborting tick")
return nil
case errors.Is(err, listenbrainz.ErrPermanent):
w.logger.Warn("similarity: similar-artists refused", "artist_id", r.ID, "err", err)
results = nil
default:
w.logger.Warn("similarity: similar-artists failed", "artist_id", r.ID, "err", err)
continue
}
w.upsertArtistSimilar(ctx, q, r.ID, results)
if err := q.RecordArtistSimilarityFetch(ctx, dbq.RecordArtistSimilarityFetchParams{
ArtistID: r.ID, Returned: int32(len(results)),
}); err != nil {
w.logger.Warn("similarity: record artist fetch", "artist_id", r.ID, "err", err)
}
}
return nil
}
func isRateLimited(err error) bool {
var ra *listenbrainz.RetryAfterError
return errors.As(err, &ra)
}
// storeTrackSimilar replaces the seed's cached answer with this one, records
// the fetch, and resolves the seed's edges, in one transaction so the queue
// never sees a fetch whose answer is not stored.
func (w *Worker) storeTrackSimilar(ctx context.Context, seedID pgtype.UUID, results []listenbrainz.SimilarRecording) error {
mbids := make([]string, 0, len(results))
scores := make([]float64, 0, len(results))
for _, r := range results {
mbids = append(mbids, r.MBID)
scores = append(scores, r.Score)
}
return pgx.BeginFunc(ctx, w.pool, func(tx pgx.Tx) error {
q := dbq.New(tx)
if err := q.DeleteSimilarRecordingsForSeed(ctx, seedID); err != nil {
return fmt.Errorf("clear cached answer: %w", err)
}
if len(mbids) > 0 {
if err := q.InsertSimilarRecordings(ctx, dbq.InsertSimilarRecordingsParams{
SeedTrackID: seedID, Mbids: mbids, Scores: scores,
}); err != nil {
return fmt.Errorf("cache answer: %w", err)
}
}
if err := q.RecordTrackSimilarityFetch(ctx, dbq.RecordTrackSimilarityFetchParams{
TrackID: seedID, Returned: int32(len(results)),
}); err != nil {
return fmt.Errorf("record fetch: %w", err)
}
if err := q.ResolveListenBrainzTrackEdges(ctx, []pgtype.UUID{seedID}); err != nil {
return fmt.Errorf("resolve edges: %w", err)
}
return nil
})
}
func (w *Worker) upsertArtistSimilar(ctx context.Context, q *dbq.Queries, artistAID pgtype.UUID, results []listenbrainz.SimilarArtist) {
if len(results) == 0 {
return
}
sort.Slice(results, func(i, j int) bool { return results[i].Score > results[j].Score })
// Scores are stored relative to the seed's closest artist, as track edges
// are: LB's raw scores are unbounded counts, and the candidate pool reads
// 0 to 1.
best := results[0].Score
mbids := make([]string, 0, len(results))
for _, r := range results {
mbids = append(mbids, r.MBID)
}
rows, err := q.GetArtistsByMBIDs(ctx, mbids)
if err != nil {
w.logger.Warn("similarity: GetArtistsByMBIDs", "err", err)
return
}
idByMBID := make(map[string]pgtype.UUID, len(rows))
for _, r := range rows {
if r.Mbid != nil {
idByMBID[*r.Mbid] = r.ID
}
}
// In-library similars → artist_similarity.
takenMatched := 0
for _, r := range results {
if takenMatched >= w.topK {
break
}
localID, ok := idByMBID[r.MBID]
if !ok {
continue
}
if localID == artistAID {
continue
}
if uerr := q.UpsertArtistSimilarity(ctx, dbq.UpsertArtistSimilarityParams{
ArtistAID: artistAID, ArtistBID: localID, Score: relativeScore(r.Score, best),
}); uerr != nil {
w.logger.Warn("similarity: UpsertArtistSimilarity", "err", uerr)
continue
}
takenMatched++
}
}
// relativeScore is score over the seed's best, in [0, 1]; 0 when the seed has
// no positive score to measure against.
func relativeScore(score, best float64) float64 {
if best <= 0 {
return 0
}
return score / best
}