release / go (push) Successful in 1m54s
release / web (push) Failing after 5s
release / integration (push) Failing after 3m44s
release / govulncheck (push) Successful in 20s
release / Attach APK to the Release (tag releases only) (push) Canceled after 0s
release / Build + push container image (push) Canceled after 0s
release / Verify release artifacts (tag releases only) (push) Canceled after 0s
release / Build signed APK (push) Canceled after 19s
release / android (push) Canceled after 2m37s
Minstrel is now a viewer of the library it is pointed at. Finding, downloading and filing music moves out of it. Removed from the server: - the request flow (requests, approval, auto-approve) and the wanted list, follows, release profiles and release blocks; - searching indexers, the download clients, the importer, the album lifecycle and missing-file re-acquisition; - the catalogue's discography, search and art (the MusicBrainz release and track-list cache stays, for the duplicate resolver); - Discover: its suggestions of artists not in the library, snoozes, the candidate-artist tag enrichment and its tuning scope; - writes to acquisition tables from moving the library folder and from merging duplicates, the want-list join on Suspect sources, the sealing of acquisition credentials, and the release-group lookup only re-acquisition used. Settling a wrong-slot duplicate no longer sets the release aside: the copies go and their slots show as missing until a good copy is in the library. Removed from the web: Admin > Acquisition and Requests, /requests, /discover, Follow on the artist page, the auto-approve toggle, and the Collection's wanted-list tabs. The Collection is now the library's health: an overview, Reports, Duplicates, Suspect sources and Missing files. The README and docs no longer describe getting music. The tables stay for now; a later migration drops them. The request notification kinds and the Android screens are still to go. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
255 lines
7.9 KiB
Go
255 lines
7.9 KiB
Go
// Package similarity owns the inbound ListenBrainz similarity ingest
|
|
// pipeline. A periodic worker queries the LB Labs API
|
|
// (labs.api.listenbrainz.org similar-recordings / similar-artists) for
|
|
// tracks the user has played.
|
|
//
|
|
// For recordings it keeps LB's whole answer in listenbrainz_similar_recordings
|
|
// and derives the track_similarity edges from it in SQL, against whatever is
|
|
// in the library at the time, so a recording that reaches the library later
|
|
// (an import, or an MBID the AcoustID lookup fills in) gains its edges without
|
|
// asking LB again. For artists it stores the in-library matches in
|
|
// artist_similarity.
|
|
//
|
|
// Every answer is recorded as a fetch, an empty one included, and the queue
|
|
// reads those records rather than the edges: a seed that matched nothing is
|
|
// still fresh and does not hold the head of the queue.
|
|
package similarity
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"fmt"
|
|
"log/slog"
|
|
"sort"
|
|
"time"
|
|
|
|
"github.com/jackc/pgx/v5"
|
|
"github.com/jackc/pgx/v5/pgtype"
|
|
"github.com/jackc/pgx/v5/pgxpool"
|
|
|
|
"git.fabledsword.com/bvandeusen/minstrel/internal/db/dbq"
|
|
"git.fabledsword.com/bvandeusen/minstrel/internal/scrobble/listenbrainz"
|
|
)
|
|
|
|
// Worker drains played tracks and artists needing similarity. A transient
|
|
// failure is not recorded as a fetch, so the timer retries it next tick.
|
|
type Worker struct {
|
|
pool *pgxpool.Pool
|
|
client *listenbrainz.Client
|
|
logger *slog.Logger
|
|
tick time.Duration
|
|
batch int32
|
|
topK int // per artist
|
|
}
|
|
|
|
// NewWorker constructs a worker with production defaults: 1h tick,
|
|
// batch=25, topK=20. batch is tracks AND artists processed per tick, well
|
|
// under ListenBrainz rate limits (429s abort the tick). With never-fetched
|
|
// seeds first, a freshly played library is covered in days.
|
|
func NewWorker(pool *pgxpool.Pool, client *listenbrainz.Client, logger *slog.Logger) *Worker {
|
|
return &Worker{
|
|
pool: pool,
|
|
client: client,
|
|
logger: logger,
|
|
tick: 1 * time.Hour,
|
|
batch: 25,
|
|
topK: 20,
|
|
}
|
|
}
|
|
|
|
// Run blocks until ctx is cancelled, ticking every w.tick.
|
|
func (w *Worker) Run(ctx context.Context) {
|
|
t := time.NewTicker(w.tick)
|
|
defer t.Stop()
|
|
for {
|
|
select {
|
|
case <-ctx.Done():
|
|
return
|
|
case <-t.C:
|
|
if err := w.tickOnce(ctx); err != nil {
|
|
w.logger.Error("similarity: tick failed", "err", err)
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// tickOnce drains one batch of tracks and one batch of artists, then
|
|
// re-resolves every cached answer against the library. Per-row errors are
|
|
// logged and skipped. 429 aborts the fetching, not the resolve.
|
|
func (w *Worker) tickOnce(ctx context.Context) error {
|
|
q := dbq.New(w.pool)
|
|
if err := w.tickTracks(ctx, q); err != nil {
|
|
return err
|
|
}
|
|
if err := w.tickArtists(ctx, q); err != nil {
|
|
return err
|
|
}
|
|
// Links recordings that reached the library since their seed was fetched:
|
|
// a scan, an import, an MBID from the AcoustID lookup.
|
|
if err := q.ResolveListenBrainzTrackEdges(ctx, nil); err != nil {
|
|
return fmt.Errorf("resolve similarity edges: %w", err)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (w *Worker) tickTracks(ctx context.Context, q *dbq.Queries) error {
|
|
rows, err := q.ListPlayedTracksNeedingSimilarity(ctx, w.batch)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
for _, r := range rows {
|
|
if r.Mbid == nil {
|
|
continue // defensive — query already filters NULL
|
|
}
|
|
results, err := w.client.SimilarRecordings(ctx, *r.Mbid, 100)
|
|
switch {
|
|
case err == nil:
|
|
case isRateLimited(err):
|
|
w.logger.Warn("similarity: 429 — aborting tick")
|
|
return nil
|
|
case errors.Is(err, listenbrainz.ErrPermanent):
|
|
// LB refused this MBID outright; asking again next hour won't
|
|
// change that. Recorded as an empty answer so it waits its turn.
|
|
w.logger.Warn("similarity: similar-recordings refused", "track_id", r.ID, "err", err)
|
|
results = nil
|
|
default:
|
|
w.logger.Warn("similarity: similar-recordings failed", "track_id", r.ID, "err", err)
|
|
continue
|
|
}
|
|
if err := w.storeTrackSimilar(ctx, r.ID, results); err != nil {
|
|
w.logger.Warn("similarity: store similar recordings", "track_id", r.ID, "err", err)
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (w *Worker) tickArtists(ctx context.Context, q *dbq.Queries) error {
|
|
rows, err := q.ListPlayedArtistsNeedingSimilarity(ctx, w.batch)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
for _, r := range rows {
|
|
if r.Mbid == nil {
|
|
continue
|
|
}
|
|
results, err := w.client.SimilarArtists(ctx, *r.Mbid, 100)
|
|
switch {
|
|
case err == nil:
|
|
case isRateLimited(err):
|
|
w.logger.Warn("similarity: 429 — aborting tick")
|
|
return nil
|
|
case errors.Is(err, listenbrainz.ErrPermanent):
|
|
w.logger.Warn("similarity: similar-artists refused", "artist_id", r.ID, "err", err)
|
|
results = nil
|
|
default:
|
|
w.logger.Warn("similarity: similar-artists failed", "artist_id", r.ID, "err", err)
|
|
continue
|
|
}
|
|
w.upsertArtistSimilar(ctx, q, r.ID, results)
|
|
if err := q.RecordArtistSimilarityFetch(ctx, dbq.RecordArtistSimilarityFetchParams{
|
|
ArtistID: r.ID, Returned: int32(len(results)),
|
|
}); err != nil {
|
|
w.logger.Warn("similarity: record artist fetch", "artist_id", r.ID, "err", err)
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func isRateLimited(err error) bool {
|
|
var ra *listenbrainz.RetryAfterError
|
|
return errors.As(err, &ra)
|
|
}
|
|
|
|
// storeTrackSimilar replaces the seed's cached answer with this one, records
|
|
// the fetch, and resolves the seed's edges, in one transaction so the queue
|
|
// never sees a fetch whose answer is not stored.
|
|
func (w *Worker) storeTrackSimilar(ctx context.Context, seedID pgtype.UUID, results []listenbrainz.SimilarRecording) error {
|
|
mbids := make([]string, 0, len(results))
|
|
scores := make([]float64, 0, len(results))
|
|
for _, r := range results {
|
|
mbids = append(mbids, r.MBID)
|
|
scores = append(scores, r.Score)
|
|
}
|
|
return pgx.BeginFunc(ctx, w.pool, func(tx pgx.Tx) error {
|
|
q := dbq.New(tx)
|
|
if err := q.DeleteSimilarRecordingsForSeed(ctx, seedID); err != nil {
|
|
return fmt.Errorf("clear cached answer: %w", err)
|
|
}
|
|
if len(mbids) > 0 {
|
|
if err := q.InsertSimilarRecordings(ctx, dbq.InsertSimilarRecordingsParams{
|
|
SeedTrackID: seedID, Mbids: mbids, Scores: scores,
|
|
}); err != nil {
|
|
return fmt.Errorf("cache answer: %w", err)
|
|
}
|
|
}
|
|
if err := q.RecordTrackSimilarityFetch(ctx, dbq.RecordTrackSimilarityFetchParams{
|
|
TrackID: seedID, Returned: int32(len(results)),
|
|
}); err != nil {
|
|
return fmt.Errorf("record fetch: %w", err)
|
|
}
|
|
if err := q.ResolveListenBrainzTrackEdges(ctx, []pgtype.UUID{seedID}); err != nil {
|
|
return fmt.Errorf("resolve edges: %w", err)
|
|
}
|
|
return nil
|
|
})
|
|
}
|
|
|
|
func (w *Worker) upsertArtistSimilar(ctx context.Context, q *dbq.Queries, artistAID pgtype.UUID, results []listenbrainz.SimilarArtist) {
|
|
if len(results) == 0 {
|
|
return
|
|
}
|
|
sort.Slice(results, func(i, j int) bool { return results[i].Score > results[j].Score })
|
|
// Scores are stored relative to the seed's closest artist, as track edges
|
|
// are: LB's raw scores are unbounded counts, and the candidate pool reads
|
|
// 0 to 1.
|
|
best := results[0].Score
|
|
|
|
mbids := make([]string, 0, len(results))
|
|
for _, r := range results {
|
|
mbids = append(mbids, r.MBID)
|
|
}
|
|
rows, err := q.GetArtistsByMBIDs(ctx, mbids)
|
|
if err != nil {
|
|
w.logger.Warn("similarity: GetArtistsByMBIDs", "err", err)
|
|
return
|
|
}
|
|
idByMBID := make(map[string]pgtype.UUID, len(rows))
|
|
for _, r := range rows {
|
|
if r.Mbid != nil {
|
|
idByMBID[*r.Mbid] = r.ID
|
|
}
|
|
}
|
|
|
|
// In-library similars → artist_similarity.
|
|
takenMatched := 0
|
|
for _, r := range results {
|
|
if takenMatched >= w.topK {
|
|
break
|
|
}
|
|
localID, ok := idByMBID[r.MBID]
|
|
if !ok {
|
|
continue
|
|
}
|
|
if localID == artistAID {
|
|
continue
|
|
}
|
|
if uerr := q.UpsertArtistSimilarity(ctx, dbq.UpsertArtistSimilarityParams{
|
|
ArtistAID: artistAID, ArtistBID: localID, Score: relativeScore(r.Score, best),
|
|
}); uerr != nil {
|
|
w.logger.Warn("similarity: UpsertArtistSimilarity", "err", uerr)
|
|
continue
|
|
}
|
|
takenMatched++
|
|
}
|
|
|
|
}
|
|
|
|
// relativeScore is score over the seed's best, in [0, 1]; 0 when the seed has
|
|
// no positive score to measure against.
|
|
func relativeScore(score, best float64) float64 {
|
|
if best <= 0 {
|
|
return 0
|
|
}
|
|
return score / best
|
|
}
|