feat: Minstrel stops acquiring music and drops Discover
release / go (push) Successful in 1m54s
release / web (push) Failing after 5s
release / integration (push) Failing after 3m44s
release / govulncheck (push) Successful in 20s
release / Attach APK to the Release (tag releases only) (push) Canceled after 0s
release / Build + push container image (push) Canceled after 0s
release / Verify release artifacts (tag releases only) (push) Canceled after 0s
release / Build signed APK (push) Canceled after 19s
release / android (push) Canceled after 2m37s

Minstrel is now a viewer of the library it is pointed at. Finding,
downloading and filing music moves out of it.

Removed from the server:
- the request flow (requests, approval, auto-approve) and the wanted list,
  follows, release profiles and release blocks;
- searching indexers, the download clients, the importer, the album
  lifecycle and missing-file re-acquisition;
- the catalogue's discography, search and art (the MusicBrainz release and
  track-list cache stays, for the duplicate resolver);
- Discover: its suggestions of artists not in the library, snoozes, the
  candidate-artist tag enrichment and its tuning scope;
- writes to acquisition tables from moving the library folder and from
  merging duplicates, the want-list join on Suspect sources, the sealing of
  acquisition credentials, and the release-group lookup only
  re-acquisition used.

Settling a wrong-slot duplicate no longer sets the release aside: the
copies go and their slots show as missing until a good copy is in the
library.

Removed from the web: Admin > Acquisition and Requests, /requests,
/discover, Follow on the artist page, the auto-approve toggle, and the
Collection's wanted-list tabs. The Collection is now the library's health:
an overview, Reports, Duplicates, Suspect sources and Missing files. The
README and docs no longer describe getting music.

The tables stay for now; a later migration drops them. The request
notification kinds and the Android screens are still to go.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
2026-11-21 20:51:14 -05:00
co-authored by Claude Opus 5.5
parent dabe1e052d
commit 55c7497ca7
277 changed files with 521 additions and 40021 deletions
-214
View File
@@ -1,214 +0,0 @@
package catalogue
import (
"context"
"errors"
"fmt"
"os"
"path/filepath"
"regexp"
"sync"
"time"
"git.fabledsword.com/bvandeusen/minstrel/internal/coverart"
)
// NoArtRetry is how long "no source has art for this" is believed before
// the sources are asked again.
const NoArtRetry = 7 * 24 * time.Hour
// fetchTimeout bounds one look-up across every source.
const fetchTimeout = 30 * time.Second
// ErrNoArt: no enabled source has an image, or the sources could not be
// reached. The caller shows its placeholder.
var ErrNoArt = errors.New("catalogue: no art")
var mbidPattern = regexp.MustCompile(`^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$`)
// ValidMBID reports whether s is shaped like a MusicBrainz id. It is the
// guard that keeps a request path out of the cache directory's parents.
func ValidMBID(s string) bool { return mbidPattern.MatchString(s) }
// Art serves images for artists and release groups that are not in the
// library (search results, Discover suggestions, wanted albums), from the
// enabled cover-art sources. Each image is fetched once and kept
// under the data directory, so the browser only ever talks to Minstrel.
type Art struct {
dir string
artistSources func() []coverart.ArtistArtProvider
albumSources func() []coverart.AlbumCoverProvider
now func() time.Time
mu sync.Mutex
inflight map[string]*artCall
}
type artCall struct {
done chan struct{}
body []byte
err error
}
// ArtSources lists the image sources to ask, in order. The cover-art
// settings service is one: it answers with the sources an admin enabled.
type ArtSources interface {
EnabledArtistProviders() []coverart.ArtistArtProvider
EnabledAlbumProviders() []coverart.AlbumCoverProvider
}
// NewArt keeps images under dataDir/catalogue-art and asks the given
// sources. Nil sources ask nothing, so every image is the placeholder.
func NewArt(dataDir string, sources ArtSources) *Art {
a := &Art{dir: filepath.Join(dataDir, "catalogue-art"), now: time.Now, inflight: map[string]*artCall{}}
if sources != nil {
a.artistSources = sources.EnabledArtistProviders
a.albumSources = sources.EnabledAlbumProviders
}
return a
}
// Artist returns an image of the artist: the first enabled artist source
// with one wins, its thumbnail preferred over its backdrop.
func (a *Art) Artist(ctx context.Context, mbid, name string) ([]byte, error) {
return a.cached(ctx, "artist", mbid, func(ctx context.Context) ([]byte, error) {
if a.artistSources == nil {
return nil, coverart.ErrNotFound
}
ref := coverart.ArtistRef{MBID: mbid, Name: name}
return firstOf(a.artistSources(), func(p coverart.ArtistArtProvider) ([]byte, error) {
thumb, fanart, err := p.FetchArtistArt(ctx, ref)
if err == nil && len(thumb) == 0 {
thumb = fanart
}
return thumb, err
})
})
}
// ReleaseGroup returns a cover for the release group: a source that can
// answer by group (the Cover Art Archive) is asked by its id, the rest by
// artist and title.
func (a *Art) ReleaseGroup(ctx context.Context, mbid, artist, title string) ([]byte, error) {
return a.cached(ctx, "release-group", mbid, func(ctx context.Context) ([]byte, error) {
if a.albumSources == nil {
return nil, coverart.ErrNotFound
}
byName := coverart.AlbumRef{ArtistName: artist, AlbumTitle: title}
return firstOf(a.albumSources(), func(p coverart.AlbumCoverProvider) ([]byte, error) {
if g, ok := p.(coverart.ReleaseGroupCoverProvider); ok {
return g.FetchReleaseGroupCover(ctx, mbid)
}
if artist == "" || title == "" {
return nil, coverart.ErrNotFound
}
return p.FetchAlbumCover(ctx, byName)
})
})
}
// firstOf asks each source in turn. It reports ErrNotFound only when every
// source said so; if any could not be reached, the miss is transient.
func firstOf[P any](sources []P, ask func(P) ([]byte, error)) ([]byte, error) {
transient := false
for _, p := range sources {
body, err := ask(p)
if err == nil && len(body) > 0 {
return body, nil
}
if err != nil && !errors.Is(err, coverart.ErrNotFound) {
transient = true
}
}
if transient {
return nil, coverart.ErrTransient
}
return nil, coverart.ErrNotFound
}
// cached serves kind/mbid from disk, or fetches it once however many
// requests ask at the same moment. A definite "nobody has one" is written
// down and believed for NoArtRetry; an unreachable source is not, so the
// next request asks again.
func (a *Art) cached(ctx context.Context, kind, mbid string, fetch func(context.Context) ([]byte, error)) ([]byte, error) {
if !ValidMBID(mbid) {
return nil, ErrNoArt
}
dir := filepath.Join(a.dir, kind)
imgPath := filepath.Join(dir, mbid)
nonePath := imgPath + ".none"
if body, err := os.ReadFile(imgPath); err == nil && len(body) > 0 {
return body, nil
}
if fi, err := os.Stat(nonePath); err == nil && a.now().Sub(fi.ModTime()) < NoArtRetry {
return nil, ErrNoArt
}
key := kind + "/" + mbid
a.mu.Lock()
if call, ok := a.inflight[key]; ok {
a.mu.Unlock()
select {
case <-call.done:
return call.body, call.err
case <-ctx.Done():
return nil, ctx.Err()
}
}
call := &artCall{done: make(chan struct{})}
a.inflight[key] = call
a.mu.Unlock()
defer func() {
a.mu.Lock()
delete(a.inflight, key)
a.mu.Unlock()
close(call.done)
}()
// The fetch outlives a browser that navigates away: a second request is
// likely, and the image is worth keeping.
fctx, cancel := context.WithTimeout(context.WithoutCancel(ctx), fetchTimeout)
defer cancel()
body, err := fetch(fctx)
switch {
case err == nil && len(body) > 0:
if werr := writeFile(dir, imgPath, body); werr != nil {
// Serving it uncached is still better than a placeholder.
call.body = body
return body, nil
}
_ = os.Remove(nonePath)
call.body = body
return body, nil
case err == nil || errors.Is(err, coverart.ErrNotFound):
_ = writeFile(dir, nonePath, nil)
}
call.err = ErrNoArt
return nil, ErrNoArt
}
func writeFile(dir, path string, body []byte) error {
if err := os.MkdirAll(dir, 0o755); err != nil {
return fmt.Errorf("catalogue art: %w", err)
}
tmp, err := os.CreateTemp(dir, ".part-*")
if err != nil {
return fmt.Errorf("catalogue art: %w", err)
}
if _, err := tmp.Write(body); err != nil {
_ = tmp.Close()
_ = os.Remove(tmp.Name())
return fmt.Errorf("catalogue art: %w", err)
}
if err := tmp.Close(); err != nil {
_ = os.Remove(tmp.Name())
return fmt.Errorf("catalogue art: %w", err)
}
if err := os.Rename(tmp.Name(), path); err != nil {
_ = os.Remove(tmp.Name())
return fmt.Errorf("catalogue art: %w", err)
}
return nil
}
-165
View File
@@ -1,165 +0,0 @@
package catalogue
import (
"context"
"errors"
"os"
"path/filepath"
"sync"
"sync/atomic"
"testing"
"time"
"git.fabledsword.com/bvandeusen/minstrel/internal/coverart"
)
const someMBID = "e21857d5-3256-4547-afb3-4b6ded592596"
// fakeSource is an artist and album source answering with fixed results.
type fakeSource struct {
thumb, fanart, cover []byte
err error
calls atomic.Int32
gate chan struct{} // when set, each fetch waits for it
lastRef coverart.AlbumRef
mu sync.Mutex
}
func (f *fakeSource) ID() string { return "fake" }
func (f *fakeSource) DisplayName() string { return "Fake" }
func (f *fakeSource) RequiresAPIKey() bool { return false }
func (f *fakeSource) DefaultEnabled() bool { return true }
func (f *fakeSource) Configure(_ coverart.ProviderSettings) error { return nil }
func (f *fakeSource) wait() {
f.calls.Add(1)
if f.gate != nil {
<-f.gate
}
}
func (f *fakeSource) FetchArtistArt(_ context.Context, _ coverart.ArtistRef) ([]byte, []byte, error) {
f.wait()
return f.thumb, f.fanart, f.err
}
func (f *fakeSource) FetchAlbumCover(_ context.Context, ref coverart.AlbumRef) ([]byte, error) {
f.wait()
f.mu.Lock()
f.lastRef = ref
f.mu.Unlock()
return f.cover, f.err
}
// groupSource can also answer by release group, as the Cover Art Archive does.
type groupSource struct{ *fakeSource }
func (g groupSource) FetchReleaseGroupCover(_ context.Context, _ string) ([]byte, error) {
g.wait()
return []byte("by-group"), nil
}
type sources struct {
artists []coverart.ArtistArtProvider
albums []coverart.AlbumCoverProvider
}
func (s sources) EnabledArtistProviders() []coverart.ArtistArtProvider { return s.artists }
func (s sources) EnabledAlbumProviders() []coverart.AlbumCoverProvider { return s.albums }
func TestArt_FetchesOnceAndKeepsIt(t *testing.T) {
src := &fakeSource{fanart: []byte("backdrop")}
a := NewArt(t.TempDir(), sources{artists: []coverart.ArtistArtProvider{src}})
body, err := a.Artist(context.Background(), someMBID, "Gorillaz")
if err != nil || string(body) != "backdrop" {
t.Fatalf("with only a backdrop, use it: %q, %v", body, err)
}
if body, _ := a.Artist(context.Background(), someMBID, "Gorillaz"); string(body) != "backdrop" || src.calls.Load() != 1 {
t.Errorf("the second ask should come from disk: %q after %d calls", body, src.calls.Load())
}
}
func TestArt_NobodyHasOneIsBelievedForAWeek(t *testing.T) {
src := &fakeSource{err: coverart.ErrNotFound}
a := NewArt(t.TempDir(), sources{artists: []coverart.ArtistArtProvider{src}})
if _, err := a.Artist(context.Background(), someMBID, "x"); !errors.Is(err, ErrNoArt) {
t.Fatalf("err = %v, want ErrNoArt", err)
}
if _, err := a.Artist(context.Background(), someMBID, "x"); !errors.Is(err, ErrNoArt) || src.calls.Load() != 1 {
t.Errorf("a definite miss is remembered: %d calls", src.calls.Load())
}
a.now = func() time.Time { return time.Now().Add(NoArtRetry + time.Hour) }
src.err, src.thumb = nil, []byte("thumb")
if body, err := a.Artist(context.Background(), someMBID, "x"); err != nil || string(body) != "thumb" {
t.Errorf("after a week the sources are asked again: %q, %v", body, err)
}
}
func TestArt_AnUnreachableSourceIsAskedAgain(t *testing.T) {
src := &fakeSource{err: coverart.ErrTransient}
a := NewArt(t.TempDir(), sources{artists: []coverart.ArtistArtProvider{src}})
for range 2 {
if _, err := a.Artist(context.Background(), someMBID, "x"); !errors.Is(err, ErrNoArt) {
t.Fatalf("err = %v", err)
}
}
if src.calls.Load() != 2 {
t.Errorf("calls = %d: an outage must not be written down as 'no art'", src.calls.Load())
}
}
func TestArt_ReleaseGroupAsksByGroupThenByName(t *testing.T) {
byName := &fakeSource{cover: []byte("by-name")}
caa := groupSource{&fakeSource{}}
a := NewArt(t.TempDir(), sources{albums: []coverart.AlbumCoverProvider{caa, byName}})
if body, _ := a.ReleaseGroup(context.Background(), someMBID, "Gorillaz", "Demon Days"); string(body) != "by-group" {
t.Errorf("a source that answers by group is asked by group: %q", body)
}
other := "0c4d5b5f-2f3c-3d31-9c4e-4b4f6b4c1a01"
a = NewArt(t.TempDir(), sources{albums: []coverart.AlbumCoverProvider{byName}})
if body, _ := a.ReleaseGroup(context.Background(), other, "Gorillaz", "Demon Days"); string(body) != "by-name" {
t.Errorf("by name: %q", body)
}
if byName.lastRef.MBID != "" || byName.lastRef.AlbumTitle != "Demon Days" {
t.Errorf("a release-group id is not a release id, so it is not passed as one: %+v", byName.lastRef)
}
}
func TestArt_ConcurrentAsksShareOneFetch(t *testing.T) {
src := &fakeSource{thumb: []byte("t"), gate: make(chan struct{})}
a := NewArt(t.TempDir(), sources{artists: []coverart.ArtistArtProvider{src}})
var wg sync.WaitGroup
for range 5 {
wg.Add(1)
go func() {
defer wg.Done()
if body, err := a.Artist(context.Background(), someMBID, "x"); err != nil || string(body) != "t" {
t.Errorf("%q, %v", body, err)
}
}()
}
// Let the fetch through once every caller has had time to queue on it.
time.Sleep(50 * time.Millisecond)
close(src.gate)
wg.Wait()
if src.calls.Load() != 1 {
t.Errorf("calls = %d, want one fetch for five asks", src.calls.Load())
}
}
func TestArt_RefusesWhatIsNotAnMBID(t *testing.T) {
dir := t.TempDir()
a := NewArt(dir, sources{artists: []coverart.ArtistArtProvider{&fakeSource{thumb: []byte("t")}}})
if _, err := a.Artist(context.Background(), "../../etc/passwd", "x"); !errors.Is(err, ErrNoArt) {
t.Fatalf("err = %v", err)
}
if _, err := os.Stat(filepath.Join(dir, "catalogue-art")); !os.IsNotExist(err) {
t.Error("nothing should be written for a refused id")
}
if _, err := NewArt(dir, nil).Artist(context.Background(), someMBID, "x"); !errors.Is(err, ErrNoArt) {
t.Errorf("no sources: err = %v", err)
}
}
+6 -211
View File
@@ -1,7 +1,6 @@
// Package catalogue is Minstrel's own record of what artists have released,
// read from MusicBrainz and kept in the database. The wanted
// list, search and import read it; MusicBrainz is asked only when what is
// held is missing or old.
// Package catalogue keeps MusicBrainz's releases and track lists for the
// albums the library holds. The duplicate resolver reads them to tell a song
// filed in the wrong slot from one legitimately repeated.
package catalogue
import (
@@ -9,34 +8,22 @@ import (
"errors"
"fmt"
"log/slog"
"time"
"github.com/jackc/pgx/v5"
"github.com/jackc/pgx/v5/pgtype"
"github.com/jackc/pgx/v5/pgxpool"
"git.fabledsword.com/bvandeusen/minstrel/internal/db/dbq"
"git.fabledsword.com/bvandeusen/minstrel/internal/musicbrainz"
)
const (
// DiscographyMaxAge is how long an artist's list of release groups is
// trusted before it is read again. New releases appear on this cadence.
DiscographyMaxAge = 7 * 24 * time.Hour
// ReleasesMaxAge is how long a release group's editions and track lists
// are trusted. They change rarely once a group is out.
ReleasesMaxAge = 30 * 24 * time.Hour
)
// ErrNotFound: MusicBrainz has no artist or release group under that id.
// ErrNotFound: MusicBrainz has no release group under that id.
var ErrNotFound = errors.New("catalogue: not in MusicBrainz")
// Service reads and refreshes the catalogue.
// Service reads release groups from MusicBrainz into the catalogue.
type Service struct {
pool *pgxpool.Pool
mb *musicbrainz.Client
logger *slog.Logger
now func() time.Time
}
// New makes a Service. A nil client talks to the public MusicBrainz.
@@ -47,91 +34,7 @@ func New(pool *pgxpool.Pool, mb *musicbrainz.Client, logger *slog.Logger) *Servi
if logger == nil {
logger = slog.Default()
}
return &Service{pool: pool, mb: mb, logger: logger, now: time.Now}
}
// MusicBrainz is the client the service asks, for callers that search.
func (s *Service) MusicBrainz() *musicbrainz.Client { return s.mb }
// Discography is an artist and the release groups credited to them.
type Discography struct {
Artist dbq.MbArtist
Groups []dbq.MbReleaseGroup
// Stale: MusicBrainz could not be asked, so this is the last copy read.
Stale bool
}
// Discography returns the artist's release groups, reading them from
// MusicBrainz first when they have never been read or are older than
// DiscographyMaxAge. When MusicBrainz is unreachable, a copy already held is
// returned marked Stale; with nothing held, the error is returned.
func (s *Service) Discography(ctx context.Context, artistMBID string) (Discography, error) {
return s.DiscographyWithin(ctx, artistMBID, DiscographyMaxAge)
}
// DiscographyWithin is Discography with the age past which the held copy is
// read again: a followed artist is checked for new releases more often than
// one the catalogue merely knows.
func (s *Service) DiscographyWithin(ctx context.Context, artistMBID string, maxAge time.Duration) (Discography, error) {
q := dbq.New(s.pool)
artist, err := q.GetMBArtist(ctx, artistMBID)
held := err == nil && artist.DiscographyRefreshedAt.Valid
if err != nil && !errors.Is(err, pgx.ErrNoRows) {
return Discography{}, fmt.Errorf("catalogue: read artist: %w", err)
}
stale := false
if !held || s.now().Sub(artist.DiscographyRefreshedAt.Time) > maxAge {
if rerr := s.RefreshArtist(ctx, artistMBID); rerr != nil {
if !held || !errors.Is(rerr, musicbrainz.ErrUnreachable) {
return Discography{}, rerr
}
stale = true
}
if artist, err = q.GetMBArtist(ctx, artistMBID); err != nil {
return Discography{}, fmt.Errorf("catalogue: read artist: %w", err)
}
}
groups, err := q.ListMBArtistReleaseGroups(ctx, artistMBID)
if err != nil {
return Discography{}, fmt.Errorf("catalogue: read discography: %w", err)
}
return Discography{Artist: artist, Groups: groups, Stale: stale}, nil
}
// RefreshArtist reads the artist and every release group credited to them,
// and replaces the held discography in one transaction.
func (s *Service) RefreshArtist(ctx context.Context, artistMBID string) error {
artist, err := s.mb.Artist(ctx, artistMBID)
if err != nil {
return mbErr(err)
}
groups, err := s.mb.ArtistReleaseGroups(ctx, artistMBID)
if err != nil {
return mbErr(err)
}
return pgx.BeginFunc(ctx, s.pool, func(tx pgx.Tx) error {
q := dbq.New(tx)
if err := q.UpsertMBArtist(ctx, dbq.UpsertMBArtistParams{
Mbid: artist.MBID, Name: artist.Name, SortName: artist.SortName,
Disambiguation: artist.Disambiguation, ArtistType: artist.Type, Country: artist.Country,
}); err != nil {
return fmt.Errorf("catalogue: save artist: %w", err)
}
if err := q.DeleteMBArtistReleaseGroups(ctx, artist.MBID); err != nil {
return fmt.Errorf("catalogue: clear discography: %w", err)
}
for _, g := range groups {
if err := upsertGroup(ctx, q, g); err != nil {
return err
}
if err := q.AddMBArtistReleaseGroup(ctx, dbq.AddMBArtistReleaseGroupParams{
ArtistMbid: artist.MBID, ReleaseGroupMbid: g.MBID,
}); err != nil {
return fmt.Errorf("catalogue: link release group: %w", err)
}
}
return q.MarkMBArtistDiscographyRefreshed(ctx, artist.MBID)
})
return &Service{pool: pool, mb: mb, logger: logger}
}
func upsertGroup(ctx context.Context, q *dbq.Queries, g musicbrainz.ReleaseGroup) error {
@@ -148,60 +51,6 @@ func upsertGroup(ctx context.Context, q *dbq.Queries, g musicbrainz.ReleaseGroup
return nil
}
// Edition is one release of a group with its track list.
type Edition struct {
Release dbq.MbRelease
Tracks []dbq.MbReleaseTrack
}
// Editions is a release group and its releases.
type Editions struct {
Group dbq.MbReleaseGroup
Editions []Edition
Stale bool
}
// Releases returns a release group's releases with their track lists, read
// from MusicBrainz first when never read or older than ReleasesMaxAge. Like
// Discography, an unreachable MusicBrainz serves the held copy as Stale.
func (s *Service) Releases(ctx context.Context, groupMBID string) (Editions, error) {
q := dbq.New(s.pool)
group, err := q.GetMBReleaseGroup(ctx, groupMBID)
if err != nil && !errors.Is(err, pgx.ErrNoRows) {
return Editions{}, fmt.Errorf("catalogue: read release group: %w", err)
}
held := err == nil && group.ReleasesRefreshedAt.Valid
stale := false
if !held || s.now().Sub(group.ReleasesRefreshedAt.Time) > ReleasesMaxAge {
if rerr := s.RefreshReleaseGroup(ctx, groupMBID); rerr != nil {
if !held || !errors.Is(rerr, musicbrainz.ErrUnreachable) {
return Editions{}, rerr
}
stale = true
}
if group, err = q.GetMBReleaseGroup(ctx, groupMBID); err != nil {
return Editions{}, fmt.Errorf("catalogue: read release group: %w", err)
}
}
releases, err := q.ListMBReleasesForGroup(ctx, groupMBID)
if err != nil {
return Editions{}, fmt.Errorf("catalogue: read releases: %w", err)
}
tracks, err := q.ListMBReleaseTracksForGroup(ctx, groupMBID)
if err != nil {
return Editions{}, fmt.Errorf("catalogue: read track lists: %w", err)
}
byRelease := make(map[string][]dbq.MbReleaseTrack, len(releases))
for _, t := range tracks {
byRelease[t.ReleaseMbid] = append(byRelease[t.ReleaseMbid], t)
}
out := Editions{Group: group, Stale: stale}
for _, r := range releases {
out.Editions = append(out.Editions, Edition{Release: r, Tracks: byRelease[r.Mbid]})
}
return out, nil
}
// RefreshReleaseGroup reads a group's releases and track lists and replaces
// the held copy in one transaction. A group not yet held is looked up first.
func (s *Service) RefreshReleaseGroup(ctx context.Context, groupMBID string) error {
@@ -254,60 +103,6 @@ func (s *Service) RefreshReleaseGroup(ctx context.Context, groupMBID string) err
})
}
// RefreshDue re-reads up to limit discographies that are older than
// DiscographyMaxAge, never-read first. It stops at the first unreachable
// MusicBrainz, since the rest would fail the same way.
func (s *Service) RefreshDue(ctx context.Context, limit int) (refreshed int, err error) {
due, err := dbq.New(s.pool).ListMBArtistsDueForRefresh(ctx, dbq.ListMBArtistsDueForRefreshParams{
Before: pgtype.Timestamptz{Time: s.now().Add(-DiscographyMaxAge), Valid: true},
MaxRows: int32(limit),
})
if err != nil {
return 0, fmt.Errorf("catalogue: list due artists: %w", err)
}
for _, mbid := range due {
if err := s.RefreshArtist(ctx, mbid); err != nil {
if errors.Is(err, musicbrainz.ErrUnreachable) || ctx.Err() != nil {
return refreshed, err
}
// One artist MusicBrainz cannot describe (merged away, say)
// must not stop the rest.
s.logger.Warn("catalogue: refresh artist", "mbid", mbid, "err", err)
continue
}
refreshed++
}
return refreshed, nil
}
// RefreshInterval is how often Run looks for discographies due a re-read,
// and RefreshBatch how many it reads per look: 20 artists is about a minute
// of MusicBrainz's budget.
const (
RefreshInterval = time.Hour
RefreshBatch = 20
)
// Run refreshes due discographies every RefreshInterval until ctx ends.
func (s *Service) Run(ctx context.Context) {
ticker := time.NewTicker(RefreshInterval)
defer ticker.Stop()
for {
select {
case <-ctx.Done():
return
case <-ticker.C:
}
n, err := s.RefreshDue(ctx, RefreshBatch)
switch {
case err != nil && ctx.Err() == nil:
s.logger.Warn("catalogue: refresh pass stopped", "refreshed", n, "err", err)
case n > 0:
s.logger.Info("catalogue: refresh pass complete", "refreshed", n)
}
}
}
// mbErr maps MusicBrainz's not-found to the catalogue's; anything else passes
// through so callers can tell an unreachable MusicBrainz apart.
func mbErr(err error) error {
+32 -128
View File
@@ -9,14 +9,13 @@ import (
"net/http"
"net/http/httptest"
"os"
"strings"
"sync/atomic"
"testing"
"time"
"github.com/jackc/pgx/v5/pgxpool"
"git.fabledsword.com/bvandeusen/minstrel/internal/db"
"git.fabledsword.com/bvandeusen/minstrel/internal/db/dbq"
"git.fabledsword.com/bvandeusen/minstrel/internal/dbtest"
"git.fabledsword.com/bvandeusen/minstrel/internal/musicbrainz"
)
@@ -44,15 +43,13 @@ func newPool(t *testing.T) *pgxpool.Pool {
const (
gorillaz = "e21857d5-3256-4547-afb3-4b6ded592596"
deLaSoul = "4f2a6b9e-9ae6-4f34-9a41-3e9f2f7c4c1d"
demonDays = "0c4d5b5f-2f3c-3d31-9c4e-4b4f6b4c1a01"
feelGood = "0c4d5b5f-2f3c-3d31-9c4e-4b4f6b4c1a02" // credited to both
demonDaysR = "1a2b3c4d-0000-4000-8000-000000000001"
demonDaysJ = "1a2b3c4d-0000-4000-8000-000000000002"
)
// fakeMB is a MusicBrainz with two artists, a collaboration between them,
// and one album with two editions. down makes every request a 503.
// fakeMB is a MusicBrainz with one album in two editions. down makes every
// request a 503.
type fakeMB struct {
srv *httptest.Server
calls atomic.Int32
@@ -70,21 +67,6 @@ func newFakeMB(t *testing.T) *fakeMB {
}
q := r.URL.Query()
switch {
case r.URL.Path == "/artist/"+gorillaz:
_, _ = fmt.Fprintf(w, `{"id": %q, "name": "Gorillaz", "sort-name": "Gorillaz", "type": "Group", "country": "GB"}`, gorillaz)
case r.URL.Path == "/artist/"+deLaSoul:
_, _ = fmt.Fprintf(w, `{"id": %q, "name": "De La Soul", "type": "Group", "country": "US"}`, deLaSoul)
case r.URL.Path == "/release-group" && q.Get("artist") == gorillaz:
_, _ = fmt.Fprintf(w, `{"release-group-count": 2, "release-groups": [
{"id": %q, "title": "Demon Days", "primary-type": "Album", "first-release-date": "2005-05-11",
"artist-credit": [{"name": "Gorillaz", "artist": {"id": %q}}]},
{"id": %q, "title": "Feel Good Inc.", "primary-type": "Single", "secondary-types": [], "first-release-date": "2005-05-09",
"artist-credit": [{"name": "Gorillaz", "joinphrase": " feat. ", "artist": {"id": %q}}, {"name": "De La Soul", "artist": {"id": %q}}]}
]}`, demonDays, gorillaz, feelGood, gorillaz, deLaSoul)
case r.URL.Path == "/release-group" && q.Get("artist") == deLaSoul:
_, _ = fmt.Fprintf(w, `{"release-group-count": 1, "release-groups": [
{"id": %q, "title": "Feel Good Inc.", "primary-type": "Single", "first-release-date": "2005-05-09"}
]}`, feelGood)
case r.URL.Path == "/release-group/"+demonDays:
_, _ = fmt.Fprintf(w, `{"id": %q, "title": "Demon Days", "primary-type": "Album", "first-release-date": "2005-05-11",
"artist-credit": [{"name": "Gorillaz", "artist": {"id": %q}}]}`, demonDays, gorillaz)
@@ -110,138 +92,60 @@ func (f *fakeMB) client() *musicbrainz.Client {
return &musicbrainz.Client{BaseURL: f.srv.URL, Pace: &musicbrainz.Pacer{}, Retries: -1}
}
func TestDiscography_ReadsOnceThenServesTheHeldCopy_Integration(t *testing.T) {
ctx := context.Background()
mb := newFakeMB(t)
s := New(newPool(t), mb.client(), nil)
d, err := s.Discography(ctx, gorillaz)
if err != nil {
t.Fatal(err)
}
if d.Artist.Name != "Gorillaz" || d.Artist.Country != "GB" || len(d.Groups) != 2 || d.Stale {
t.Fatalf("discography = %+v", d)
}
// Oldest first: the single came out two days before the album.
if d.Groups[0].Mbid != feelGood || d.Groups[1].PrimaryType != "Album" {
t.Errorf("order = %s, %s", d.Groups[0].Title, d.Groups[1].Title)
}
if d.Groups[0].ArtistCredit != "Gorillaz feat. De La Soul" {
t.Errorf("credit = %q", d.Groups[0].ArtistCredit)
}
before := mb.calls.Load()
if _, err := s.Discography(ctx, gorillaz); err != nil {
t.Fatal(err)
}
if mb.calls.Load() != before {
t.Error("a fresh discography should be served without asking MusicBrainz")
}
// The collaboration belongs to both artists.
dls, err := s.Discography(ctx, deLaSoul)
if err != nil {
t.Fatal(err)
}
if len(dls.Groups) != 1 || dls.Groups[0].Mbid != feelGood {
t.Errorf("De La Soul's discography = %+v", dls.Groups)
}
if again, _ := s.Discography(ctx, gorillaz); len(again.Groups) != 2 {
t.Error("refreshing one artist must not take a shared group from the other")
}
}
func TestDiscography_UnreachableServesStaleOrFails_Integration(t *testing.T) {
func TestRefreshReleaseGroup_ReadsEditionsAndTrackLists_Integration(t *testing.T) {
ctx := context.Background()
mb := newFakeMB(t)
pool := newPool(t)
s := New(pool, mb.client(), nil)
mb.down.Store(true)
if _, err := s.Discography(ctx, gorillaz); !errors.Is(err, musicbrainz.ErrUnreachable) {
t.Fatalf("nothing held and MusicBrainz down: err = %v, want ErrUnreachable", err)
}
mb.down.Store(false)
if _, err := s.Discography(ctx, gorillaz); err != nil {
t.Fatal(err)
}
// A week and a day later, MusicBrainz is down: the held copy is served.
s.now = func() time.Time { return time.Now().Add(DiscographyMaxAge + 24*time.Hour) }
mb.down.Store(true)
d, err := s.Discography(ctx, gorillaz)
if err != nil || !d.Stale || len(d.Groups) != 2 {
t.Fatalf("stale copy: %+v, %v", d, err)
}
}
func TestDiscography_UnknownArtistIsNotFound_Integration(t *testing.T) {
s := New(newPool(t), newFakeMB(t).client(), nil)
if _, err := s.Discography(context.Background(), "00000000-0000-4000-8000-000000000000"); !errors.Is(err, ErrNotFound) {
t.Fatalf("err = %v, want ErrNotFound", err)
}
}
func TestReleases_ReadsEditionsAndTrackLists_Integration(t *testing.T) {
ctx := context.Background()
mb := newFakeMB(t)
s := New(newPool(t), mb.client(), nil)
q := dbq.New(pool)
// The group isn't held yet: it is looked up first.
e, err := s.Releases(ctx, demonDays)
if err := s.RefreshReleaseGroup(ctx, demonDays); err != nil {
t.Fatal(err)
}
group, err := q.GetMBReleaseGroup(ctx, demonDays)
if err != nil {
t.Fatal(err)
}
if e.Group.Title != "Demon Days" || len(e.Editions) != 2 {
t.Fatalf("editions = %+v", e)
if group.Title != "Demon Days" || !group.ReleasesRefreshedAt.Valid {
t.Fatalf("group = %+v", group)
}
official := e.Editions[0]
if official.Release.Mbid != demonDaysR || official.Release.TrackCount != 2 || official.Release.Formats != "CD" {
t.Errorf("the official edition should come first: %+v", official.Release)
}
if len(official.Tracks) != 2 || official.Tracks[1].RecordingMbid != "rec-lls" || official.Tracks[1].LengthMs != 190000 {
t.Errorf("tracks = %+v", official.Tracks)
}
before := mb.calls.Load()
if _, err := s.Releases(ctx, demonDays); err != nil {
tracks, err := q.ListMBReleaseTracksForGroup(ctx, demonDays)
if err != nil {
t.Fatal(err)
}
if mb.calls.Load() != before {
t.Error("fresh editions should be served without asking MusicBrainz")
if len(tracks) != 3 {
t.Fatalf("want 3 tracks across both editions, got %+v", tracks)
}
var lls *dbq.MbReleaseTrack
for i := range tracks {
if tracks[i].RecordingMbid == "rec-lls" {
lls = &tracks[i]
}
}
if lls == nil || lls.ReleaseMbid != demonDaysR || lls.LengthMs != 190000 {
t.Errorf("Last Living Souls = %+v", lls)
}
// Refreshing replaces rather than adds.
if err := s.RefreshReleaseGroup(ctx, demonDays); err != nil {
t.Fatal(err)
}
if again, _ := s.Releases(ctx, demonDays); len(again.Editions) != 2 || len(again.Editions[0].Tracks) != 2 {
t.Errorf("after a refresh: %+v", again)
if again, _ := q.ListMBReleaseTracksForGroup(ctx, demonDays); len(again) != 3 {
t.Errorf("after a refresh: %d tracks, want 3", len(again))
}
}
func TestRefreshDue_ReadsWhatIsOldAndStopsWhenMusicBrainzIsDown_Integration(t *testing.T) {
func TestRefreshReleaseGroup_UnknownAndUnreachable_Integration(t *testing.T) {
ctx := context.Background()
mb := newFakeMB(t)
s := New(newPool(t), mb.client(), nil)
for _, a := range []string{gorillaz, deLaSoul} {
if err := s.RefreshArtist(ctx, a); err != nil {
t.Fatal(err)
}
}
if n, err := s.RefreshDue(ctx, 10); err != nil || n != 0 {
t.Fatalf("nothing is due yet: %d, %v", n, err)
}
s.now = func() time.Time { return time.Now().Add(DiscographyMaxAge + time.Hour) }
if n, err := s.RefreshDue(ctx, 1); err != nil || n != 1 {
t.Errorf("one due, limit one: %d, %v", n, err)
if err := s.RefreshReleaseGroup(ctx, "00000000-0000-4000-8000-000000000000"); !errors.Is(err, ErrNotFound) {
t.Errorf("unknown group: err = %v, want ErrNotFound", err)
}
mb.down.Store(true)
n, err := s.RefreshDue(ctx, 10)
if !errors.Is(err, musicbrainz.ErrUnreachable) || n != 0 {
t.Errorf("MusicBrainz down: %d, %v", n, err)
}
if !strings.Contains(err.Error(), "503") {
t.Errorf("the error should say what MusicBrainz answered: %v", err)
if err := s.RefreshReleaseGroup(ctx, demonDays); !errors.Is(err, musicbrainz.ErrUnreachable) {
t.Errorf("MusicBrainz down: err = %v, want ErrUnreachable", err)
}
}