feat: Minstrel stops acquiring music and drops Discover
release / go (push) Successful in 1m54s
release / web (push) Failing after 5s
release / integration (push) Failing after 3m44s
release / govulncheck (push) Successful in 20s
release / Attach APK to the Release (tag releases only) (push) Canceled after 0s
release / Build + push container image (push) Canceled after 0s
release / Verify release artifacts (tag releases only) (push) Canceled after 0s
release / Build signed APK (push) Canceled after 19s
release / android (push) Canceled after 2m37s
release / go (push) Successful in 1m54s
release / web (push) Failing after 5s
release / integration (push) Failing after 3m44s
release / govulncheck (push) Successful in 20s
release / Attach APK to the Release (tag releases only) (push) Canceled after 0s
release / Build + push container image (push) Canceled after 0s
release / Verify release artifacts (tag releases only) (push) Canceled after 0s
release / Build signed APK (push) Canceled after 19s
release / android (push) Canceled after 2m37s
Minstrel is now a viewer of the library it is pointed at. Finding, downloading and filing music moves out of it. Removed from the server: - the request flow (requests, approval, auto-approve) and the wanted list, follows, release profiles and release blocks; - searching indexers, the download clients, the importer, the album lifecycle and missing-file re-acquisition; - the catalogue's discography, search and art (the MusicBrainz release and track-list cache stays, for the duplicate resolver); - Discover: its suggestions of artists not in the library, snoozes, the candidate-artist tag enrichment and its tuning scope; - writes to acquisition tables from moving the library folder and from merging duplicates, the want-list join on Suspect sources, the sealing of acquisition credentials, and the release-group lookup only re-acquisition used. Settling a wrong-slot duplicate no longer sets the release aside: the copies go and their slots show as missing until a good copy is in the library. Removed from the web: Admin > Acquisition and Requests, /requests, /discover, Follow on the artist page, the auto-approve toggle, and the Collection's wanted-list tabs. The Collection is now the library's health: an overview, Reports, Duplicates, Suspect sources and Missing files. The README and docs no longer describe getting music. The tables stay for now; a later migration drops them. The request notification kinds and the Android screens are still to go. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -1,214 +0,0 @@
|
||||
package catalogue
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"git.fabledsword.com/bvandeusen/minstrel/internal/coverart"
|
||||
)
|
||||
|
||||
// NoArtRetry is how long "no source has art for this" is believed before
|
||||
// the sources are asked again.
|
||||
const NoArtRetry = 7 * 24 * time.Hour
|
||||
|
||||
// fetchTimeout bounds one look-up across every source.
|
||||
const fetchTimeout = 30 * time.Second
|
||||
|
||||
// ErrNoArt: no enabled source has an image, or the sources could not be
|
||||
// reached. The caller shows its placeholder.
|
||||
var ErrNoArt = errors.New("catalogue: no art")
|
||||
|
||||
var mbidPattern = regexp.MustCompile(`^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$`)
|
||||
|
||||
// ValidMBID reports whether s is shaped like a MusicBrainz id. It is the
|
||||
// guard that keeps a request path out of the cache directory's parents.
|
||||
func ValidMBID(s string) bool { return mbidPattern.MatchString(s) }
|
||||
|
||||
// Art serves images for artists and release groups that are not in the
|
||||
// library (search results, Discover suggestions, wanted albums), from the
|
||||
// enabled cover-art sources. Each image is fetched once and kept
|
||||
// under the data directory, so the browser only ever talks to Minstrel.
|
||||
type Art struct {
|
||||
dir string
|
||||
artistSources func() []coverart.ArtistArtProvider
|
||||
albumSources func() []coverart.AlbumCoverProvider
|
||||
now func() time.Time
|
||||
|
||||
mu sync.Mutex
|
||||
inflight map[string]*artCall
|
||||
}
|
||||
|
||||
type artCall struct {
|
||||
done chan struct{}
|
||||
body []byte
|
||||
err error
|
||||
}
|
||||
|
||||
// ArtSources lists the image sources to ask, in order. The cover-art
|
||||
// settings service is one: it answers with the sources an admin enabled.
|
||||
type ArtSources interface {
|
||||
EnabledArtistProviders() []coverart.ArtistArtProvider
|
||||
EnabledAlbumProviders() []coverart.AlbumCoverProvider
|
||||
}
|
||||
|
||||
// NewArt keeps images under dataDir/catalogue-art and asks the given
|
||||
// sources. Nil sources ask nothing, so every image is the placeholder.
|
||||
func NewArt(dataDir string, sources ArtSources) *Art {
|
||||
a := &Art{dir: filepath.Join(dataDir, "catalogue-art"), now: time.Now, inflight: map[string]*artCall{}}
|
||||
if sources != nil {
|
||||
a.artistSources = sources.EnabledArtistProviders
|
||||
a.albumSources = sources.EnabledAlbumProviders
|
||||
}
|
||||
return a
|
||||
}
|
||||
|
||||
// Artist returns an image of the artist: the first enabled artist source
|
||||
// with one wins, its thumbnail preferred over its backdrop.
|
||||
func (a *Art) Artist(ctx context.Context, mbid, name string) ([]byte, error) {
|
||||
return a.cached(ctx, "artist", mbid, func(ctx context.Context) ([]byte, error) {
|
||||
if a.artistSources == nil {
|
||||
return nil, coverart.ErrNotFound
|
||||
}
|
||||
ref := coverart.ArtistRef{MBID: mbid, Name: name}
|
||||
return firstOf(a.artistSources(), func(p coverart.ArtistArtProvider) ([]byte, error) {
|
||||
thumb, fanart, err := p.FetchArtistArt(ctx, ref)
|
||||
if err == nil && len(thumb) == 0 {
|
||||
thumb = fanart
|
||||
}
|
||||
return thumb, err
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
// ReleaseGroup returns a cover for the release group: a source that can
|
||||
// answer by group (the Cover Art Archive) is asked by its id, the rest by
|
||||
// artist and title.
|
||||
func (a *Art) ReleaseGroup(ctx context.Context, mbid, artist, title string) ([]byte, error) {
|
||||
return a.cached(ctx, "release-group", mbid, func(ctx context.Context) ([]byte, error) {
|
||||
if a.albumSources == nil {
|
||||
return nil, coverart.ErrNotFound
|
||||
}
|
||||
byName := coverart.AlbumRef{ArtistName: artist, AlbumTitle: title}
|
||||
return firstOf(a.albumSources(), func(p coverart.AlbumCoverProvider) ([]byte, error) {
|
||||
if g, ok := p.(coverart.ReleaseGroupCoverProvider); ok {
|
||||
return g.FetchReleaseGroupCover(ctx, mbid)
|
||||
}
|
||||
if artist == "" || title == "" {
|
||||
return nil, coverart.ErrNotFound
|
||||
}
|
||||
return p.FetchAlbumCover(ctx, byName)
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
// firstOf asks each source in turn. It reports ErrNotFound only when every
|
||||
// source said so; if any could not be reached, the miss is transient.
|
||||
func firstOf[P any](sources []P, ask func(P) ([]byte, error)) ([]byte, error) {
|
||||
transient := false
|
||||
for _, p := range sources {
|
||||
body, err := ask(p)
|
||||
if err == nil && len(body) > 0 {
|
||||
return body, nil
|
||||
}
|
||||
if err != nil && !errors.Is(err, coverart.ErrNotFound) {
|
||||
transient = true
|
||||
}
|
||||
}
|
||||
if transient {
|
||||
return nil, coverart.ErrTransient
|
||||
}
|
||||
return nil, coverart.ErrNotFound
|
||||
}
|
||||
|
||||
// cached serves kind/mbid from disk, or fetches it once however many
|
||||
// requests ask at the same moment. A definite "nobody has one" is written
|
||||
// down and believed for NoArtRetry; an unreachable source is not, so the
|
||||
// next request asks again.
|
||||
func (a *Art) cached(ctx context.Context, kind, mbid string, fetch func(context.Context) ([]byte, error)) ([]byte, error) {
|
||||
if !ValidMBID(mbid) {
|
||||
return nil, ErrNoArt
|
||||
}
|
||||
dir := filepath.Join(a.dir, kind)
|
||||
imgPath := filepath.Join(dir, mbid)
|
||||
nonePath := imgPath + ".none"
|
||||
|
||||
if body, err := os.ReadFile(imgPath); err == nil && len(body) > 0 {
|
||||
return body, nil
|
||||
}
|
||||
if fi, err := os.Stat(nonePath); err == nil && a.now().Sub(fi.ModTime()) < NoArtRetry {
|
||||
return nil, ErrNoArt
|
||||
}
|
||||
|
||||
key := kind + "/" + mbid
|
||||
a.mu.Lock()
|
||||
if call, ok := a.inflight[key]; ok {
|
||||
a.mu.Unlock()
|
||||
select {
|
||||
case <-call.done:
|
||||
return call.body, call.err
|
||||
case <-ctx.Done():
|
||||
return nil, ctx.Err()
|
||||
}
|
||||
}
|
||||
call := &artCall{done: make(chan struct{})}
|
||||
a.inflight[key] = call
|
||||
a.mu.Unlock()
|
||||
|
||||
defer func() {
|
||||
a.mu.Lock()
|
||||
delete(a.inflight, key)
|
||||
a.mu.Unlock()
|
||||
close(call.done)
|
||||
}()
|
||||
|
||||
// The fetch outlives a browser that navigates away: a second request is
|
||||
// likely, and the image is worth keeping.
|
||||
fctx, cancel := context.WithTimeout(context.WithoutCancel(ctx), fetchTimeout)
|
||||
defer cancel()
|
||||
body, err := fetch(fctx)
|
||||
switch {
|
||||
case err == nil && len(body) > 0:
|
||||
if werr := writeFile(dir, imgPath, body); werr != nil {
|
||||
// Serving it uncached is still better than a placeholder.
|
||||
call.body = body
|
||||
return body, nil
|
||||
}
|
||||
_ = os.Remove(nonePath)
|
||||
call.body = body
|
||||
return body, nil
|
||||
case err == nil || errors.Is(err, coverart.ErrNotFound):
|
||||
_ = writeFile(dir, nonePath, nil)
|
||||
}
|
||||
call.err = ErrNoArt
|
||||
return nil, ErrNoArt
|
||||
}
|
||||
|
||||
func writeFile(dir, path string, body []byte) error {
|
||||
if err := os.MkdirAll(dir, 0o755); err != nil {
|
||||
return fmt.Errorf("catalogue art: %w", err)
|
||||
}
|
||||
tmp, err := os.CreateTemp(dir, ".part-*")
|
||||
if err != nil {
|
||||
return fmt.Errorf("catalogue art: %w", err)
|
||||
}
|
||||
if _, err := tmp.Write(body); err != nil {
|
||||
_ = tmp.Close()
|
||||
_ = os.Remove(tmp.Name())
|
||||
return fmt.Errorf("catalogue art: %w", err)
|
||||
}
|
||||
if err := tmp.Close(); err != nil {
|
||||
_ = os.Remove(tmp.Name())
|
||||
return fmt.Errorf("catalogue art: %w", err)
|
||||
}
|
||||
if err := os.Rename(tmp.Name(), path); err != nil {
|
||||
_ = os.Remove(tmp.Name())
|
||||
return fmt.Errorf("catalogue art: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -1,165 +0,0 @@
|
||||
package catalogue
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.fabledsword.com/bvandeusen/minstrel/internal/coverart"
|
||||
)
|
||||
|
||||
const someMBID = "e21857d5-3256-4547-afb3-4b6ded592596"
|
||||
|
||||
// fakeSource is an artist and album source answering with fixed results.
|
||||
type fakeSource struct {
|
||||
thumb, fanart, cover []byte
|
||||
err error
|
||||
calls atomic.Int32
|
||||
gate chan struct{} // when set, each fetch waits for it
|
||||
lastRef coverart.AlbumRef
|
||||
mu sync.Mutex
|
||||
}
|
||||
|
||||
func (f *fakeSource) ID() string { return "fake" }
|
||||
func (f *fakeSource) DisplayName() string { return "Fake" }
|
||||
func (f *fakeSource) RequiresAPIKey() bool { return false }
|
||||
func (f *fakeSource) DefaultEnabled() bool { return true }
|
||||
func (f *fakeSource) Configure(_ coverart.ProviderSettings) error { return nil }
|
||||
|
||||
func (f *fakeSource) wait() {
|
||||
f.calls.Add(1)
|
||||
if f.gate != nil {
|
||||
<-f.gate
|
||||
}
|
||||
}
|
||||
|
||||
func (f *fakeSource) FetchArtistArt(_ context.Context, _ coverart.ArtistRef) ([]byte, []byte, error) {
|
||||
f.wait()
|
||||
return f.thumb, f.fanart, f.err
|
||||
}
|
||||
|
||||
func (f *fakeSource) FetchAlbumCover(_ context.Context, ref coverart.AlbumRef) ([]byte, error) {
|
||||
f.wait()
|
||||
f.mu.Lock()
|
||||
f.lastRef = ref
|
||||
f.mu.Unlock()
|
||||
return f.cover, f.err
|
||||
}
|
||||
|
||||
// groupSource can also answer by release group, as the Cover Art Archive does.
|
||||
type groupSource struct{ *fakeSource }
|
||||
|
||||
func (g groupSource) FetchReleaseGroupCover(_ context.Context, _ string) ([]byte, error) {
|
||||
g.wait()
|
||||
return []byte("by-group"), nil
|
||||
}
|
||||
|
||||
type sources struct {
|
||||
artists []coverart.ArtistArtProvider
|
||||
albums []coverart.AlbumCoverProvider
|
||||
}
|
||||
|
||||
func (s sources) EnabledArtistProviders() []coverart.ArtistArtProvider { return s.artists }
|
||||
func (s sources) EnabledAlbumProviders() []coverart.AlbumCoverProvider { return s.albums }
|
||||
|
||||
func TestArt_FetchesOnceAndKeepsIt(t *testing.T) {
|
||||
src := &fakeSource{fanart: []byte("backdrop")}
|
||||
a := NewArt(t.TempDir(), sources{artists: []coverart.ArtistArtProvider{src}})
|
||||
|
||||
body, err := a.Artist(context.Background(), someMBID, "Gorillaz")
|
||||
if err != nil || string(body) != "backdrop" {
|
||||
t.Fatalf("with only a backdrop, use it: %q, %v", body, err)
|
||||
}
|
||||
if body, _ := a.Artist(context.Background(), someMBID, "Gorillaz"); string(body) != "backdrop" || src.calls.Load() != 1 {
|
||||
t.Errorf("the second ask should come from disk: %q after %d calls", body, src.calls.Load())
|
||||
}
|
||||
}
|
||||
|
||||
func TestArt_NobodyHasOneIsBelievedForAWeek(t *testing.T) {
|
||||
src := &fakeSource{err: coverart.ErrNotFound}
|
||||
a := NewArt(t.TempDir(), sources{artists: []coverart.ArtistArtProvider{src}})
|
||||
|
||||
if _, err := a.Artist(context.Background(), someMBID, "x"); !errors.Is(err, ErrNoArt) {
|
||||
t.Fatalf("err = %v, want ErrNoArt", err)
|
||||
}
|
||||
if _, err := a.Artist(context.Background(), someMBID, "x"); !errors.Is(err, ErrNoArt) || src.calls.Load() != 1 {
|
||||
t.Errorf("a definite miss is remembered: %d calls", src.calls.Load())
|
||||
}
|
||||
a.now = func() time.Time { return time.Now().Add(NoArtRetry + time.Hour) }
|
||||
src.err, src.thumb = nil, []byte("thumb")
|
||||
if body, err := a.Artist(context.Background(), someMBID, "x"); err != nil || string(body) != "thumb" {
|
||||
t.Errorf("after a week the sources are asked again: %q, %v", body, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestArt_AnUnreachableSourceIsAskedAgain(t *testing.T) {
|
||||
src := &fakeSource{err: coverart.ErrTransient}
|
||||
a := NewArt(t.TempDir(), sources{artists: []coverart.ArtistArtProvider{src}})
|
||||
for range 2 {
|
||||
if _, err := a.Artist(context.Background(), someMBID, "x"); !errors.Is(err, ErrNoArt) {
|
||||
t.Fatalf("err = %v", err)
|
||||
}
|
||||
}
|
||||
if src.calls.Load() != 2 {
|
||||
t.Errorf("calls = %d: an outage must not be written down as 'no art'", src.calls.Load())
|
||||
}
|
||||
}
|
||||
|
||||
func TestArt_ReleaseGroupAsksByGroupThenByName(t *testing.T) {
|
||||
byName := &fakeSource{cover: []byte("by-name")}
|
||||
caa := groupSource{&fakeSource{}}
|
||||
a := NewArt(t.TempDir(), sources{albums: []coverart.AlbumCoverProvider{caa, byName}})
|
||||
if body, _ := a.ReleaseGroup(context.Background(), someMBID, "Gorillaz", "Demon Days"); string(body) != "by-group" {
|
||||
t.Errorf("a source that answers by group is asked by group: %q", body)
|
||||
}
|
||||
|
||||
other := "0c4d5b5f-2f3c-3d31-9c4e-4b4f6b4c1a01"
|
||||
a = NewArt(t.TempDir(), sources{albums: []coverart.AlbumCoverProvider{byName}})
|
||||
if body, _ := a.ReleaseGroup(context.Background(), other, "Gorillaz", "Demon Days"); string(body) != "by-name" {
|
||||
t.Errorf("by name: %q", body)
|
||||
}
|
||||
if byName.lastRef.MBID != "" || byName.lastRef.AlbumTitle != "Demon Days" {
|
||||
t.Errorf("a release-group id is not a release id, so it is not passed as one: %+v", byName.lastRef)
|
||||
}
|
||||
}
|
||||
|
||||
func TestArt_ConcurrentAsksShareOneFetch(t *testing.T) {
|
||||
src := &fakeSource{thumb: []byte("t"), gate: make(chan struct{})}
|
||||
a := NewArt(t.TempDir(), sources{artists: []coverart.ArtistArtProvider{src}})
|
||||
var wg sync.WaitGroup
|
||||
for range 5 {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
if body, err := a.Artist(context.Background(), someMBID, "x"); err != nil || string(body) != "t" {
|
||||
t.Errorf("%q, %v", body, err)
|
||||
}
|
||||
}()
|
||||
}
|
||||
// Let the fetch through once every caller has had time to queue on it.
|
||||
time.Sleep(50 * time.Millisecond)
|
||||
close(src.gate)
|
||||
wg.Wait()
|
||||
if src.calls.Load() != 1 {
|
||||
t.Errorf("calls = %d, want one fetch for five asks", src.calls.Load())
|
||||
}
|
||||
}
|
||||
|
||||
func TestArt_RefusesWhatIsNotAnMBID(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
a := NewArt(dir, sources{artists: []coverart.ArtistArtProvider{&fakeSource{thumb: []byte("t")}}})
|
||||
if _, err := a.Artist(context.Background(), "../../etc/passwd", "x"); !errors.Is(err, ErrNoArt) {
|
||||
t.Fatalf("err = %v", err)
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(dir, "catalogue-art")); !os.IsNotExist(err) {
|
||||
t.Error("nothing should be written for a refused id")
|
||||
}
|
||||
if _, err := NewArt(dir, nil).Artist(context.Background(), someMBID, "x"); !errors.Is(err, ErrNoArt) {
|
||||
t.Errorf("no sources: err = %v", err)
|
||||
}
|
||||
}
|
||||
@@ -1,7 +1,6 @@
|
||||
// Package catalogue is Minstrel's own record of what artists have released,
|
||||
// read from MusicBrainz and kept in the database. The wanted
|
||||
// list, search and import read it; MusicBrainz is asked only when what is
|
||||
// held is missing or old.
|
||||
// Package catalogue keeps MusicBrainz's releases and track lists for the
|
||||
// albums the library holds. The duplicate resolver reads them to tell a song
|
||||
// filed in the wrong slot from one legitimately repeated.
|
||||
package catalogue
|
||||
|
||||
import (
|
||||
@@ -9,34 +8,22 @@ import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v5"
|
||||
"github.com/jackc/pgx/v5/pgtype"
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
|
||||
"git.fabledsword.com/bvandeusen/minstrel/internal/db/dbq"
|
||||
"git.fabledsword.com/bvandeusen/minstrel/internal/musicbrainz"
|
||||
)
|
||||
|
||||
const (
|
||||
// DiscographyMaxAge is how long an artist's list of release groups is
|
||||
// trusted before it is read again. New releases appear on this cadence.
|
||||
DiscographyMaxAge = 7 * 24 * time.Hour
|
||||
// ReleasesMaxAge is how long a release group's editions and track lists
|
||||
// are trusted. They change rarely once a group is out.
|
||||
ReleasesMaxAge = 30 * 24 * time.Hour
|
||||
)
|
||||
|
||||
// ErrNotFound: MusicBrainz has no artist or release group under that id.
|
||||
// ErrNotFound: MusicBrainz has no release group under that id.
|
||||
var ErrNotFound = errors.New("catalogue: not in MusicBrainz")
|
||||
|
||||
// Service reads and refreshes the catalogue.
|
||||
// Service reads release groups from MusicBrainz into the catalogue.
|
||||
type Service struct {
|
||||
pool *pgxpool.Pool
|
||||
mb *musicbrainz.Client
|
||||
logger *slog.Logger
|
||||
now func() time.Time
|
||||
}
|
||||
|
||||
// New makes a Service. A nil client talks to the public MusicBrainz.
|
||||
@@ -47,91 +34,7 @@ func New(pool *pgxpool.Pool, mb *musicbrainz.Client, logger *slog.Logger) *Servi
|
||||
if logger == nil {
|
||||
logger = slog.Default()
|
||||
}
|
||||
return &Service{pool: pool, mb: mb, logger: logger, now: time.Now}
|
||||
}
|
||||
|
||||
// MusicBrainz is the client the service asks, for callers that search.
|
||||
func (s *Service) MusicBrainz() *musicbrainz.Client { return s.mb }
|
||||
|
||||
// Discography is an artist and the release groups credited to them.
|
||||
type Discography struct {
|
||||
Artist dbq.MbArtist
|
||||
Groups []dbq.MbReleaseGroup
|
||||
// Stale: MusicBrainz could not be asked, so this is the last copy read.
|
||||
Stale bool
|
||||
}
|
||||
|
||||
// Discography returns the artist's release groups, reading them from
|
||||
// MusicBrainz first when they have never been read or are older than
|
||||
// DiscographyMaxAge. When MusicBrainz is unreachable, a copy already held is
|
||||
// returned marked Stale; with nothing held, the error is returned.
|
||||
func (s *Service) Discography(ctx context.Context, artistMBID string) (Discography, error) {
|
||||
return s.DiscographyWithin(ctx, artistMBID, DiscographyMaxAge)
|
||||
}
|
||||
|
||||
// DiscographyWithin is Discography with the age past which the held copy is
|
||||
// read again: a followed artist is checked for new releases more often than
|
||||
// one the catalogue merely knows.
|
||||
func (s *Service) DiscographyWithin(ctx context.Context, artistMBID string, maxAge time.Duration) (Discography, error) {
|
||||
q := dbq.New(s.pool)
|
||||
artist, err := q.GetMBArtist(ctx, artistMBID)
|
||||
held := err == nil && artist.DiscographyRefreshedAt.Valid
|
||||
if err != nil && !errors.Is(err, pgx.ErrNoRows) {
|
||||
return Discography{}, fmt.Errorf("catalogue: read artist: %w", err)
|
||||
}
|
||||
stale := false
|
||||
if !held || s.now().Sub(artist.DiscographyRefreshedAt.Time) > maxAge {
|
||||
if rerr := s.RefreshArtist(ctx, artistMBID); rerr != nil {
|
||||
if !held || !errors.Is(rerr, musicbrainz.ErrUnreachable) {
|
||||
return Discography{}, rerr
|
||||
}
|
||||
stale = true
|
||||
}
|
||||
if artist, err = q.GetMBArtist(ctx, artistMBID); err != nil {
|
||||
return Discography{}, fmt.Errorf("catalogue: read artist: %w", err)
|
||||
}
|
||||
}
|
||||
groups, err := q.ListMBArtistReleaseGroups(ctx, artistMBID)
|
||||
if err != nil {
|
||||
return Discography{}, fmt.Errorf("catalogue: read discography: %w", err)
|
||||
}
|
||||
return Discography{Artist: artist, Groups: groups, Stale: stale}, nil
|
||||
}
|
||||
|
||||
// RefreshArtist reads the artist and every release group credited to them,
|
||||
// and replaces the held discography in one transaction.
|
||||
func (s *Service) RefreshArtist(ctx context.Context, artistMBID string) error {
|
||||
artist, err := s.mb.Artist(ctx, artistMBID)
|
||||
if err != nil {
|
||||
return mbErr(err)
|
||||
}
|
||||
groups, err := s.mb.ArtistReleaseGroups(ctx, artistMBID)
|
||||
if err != nil {
|
||||
return mbErr(err)
|
||||
}
|
||||
return pgx.BeginFunc(ctx, s.pool, func(tx pgx.Tx) error {
|
||||
q := dbq.New(tx)
|
||||
if err := q.UpsertMBArtist(ctx, dbq.UpsertMBArtistParams{
|
||||
Mbid: artist.MBID, Name: artist.Name, SortName: artist.SortName,
|
||||
Disambiguation: artist.Disambiguation, ArtistType: artist.Type, Country: artist.Country,
|
||||
}); err != nil {
|
||||
return fmt.Errorf("catalogue: save artist: %w", err)
|
||||
}
|
||||
if err := q.DeleteMBArtistReleaseGroups(ctx, artist.MBID); err != nil {
|
||||
return fmt.Errorf("catalogue: clear discography: %w", err)
|
||||
}
|
||||
for _, g := range groups {
|
||||
if err := upsertGroup(ctx, q, g); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := q.AddMBArtistReleaseGroup(ctx, dbq.AddMBArtistReleaseGroupParams{
|
||||
ArtistMbid: artist.MBID, ReleaseGroupMbid: g.MBID,
|
||||
}); err != nil {
|
||||
return fmt.Errorf("catalogue: link release group: %w", err)
|
||||
}
|
||||
}
|
||||
return q.MarkMBArtistDiscographyRefreshed(ctx, artist.MBID)
|
||||
})
|
||||
return &Service{pool: pool, mb: mb, logger: logger}
|
||||
}
|
||||
|
||||
func upsertGroup(ctx context.Context, q *dbq.Queries, g musicbrainz.ReleaseGroup) error {
|
||||
@@ -148,60 +51,6 @@ func upsertGroup(ctx context.Context, q *dbq.Queries, g musicbrainz.ReleaseGroup
|
||||
return nil
|
||||
}
|
||||
|
||||
// Edition is one release of a group with its track list.
|
||||
type Edition struct {
|
||||
Release dbq.MbRelease
|
||||
Tracks []dbq.MbReleaseTrack
|
||||
}
|
||||
|
||||
// Editions is a release group and its releases.
|
||||
type Editions struct {
|
||||
Group dbq.MbReleaseGroup
|
||||
Editions []Edition
|
||||
Stale bool
|
||||
}
|
||||
|
||||
// Releases returns a release group's releases with their track lists, read
|
||||
// from MusicBrainz first when never read or older than ReleasesMaxAge. Like
|
||||
// Discography, an unreachable MusicBrainz serves the held copy as Stale.
|
||||
func (s *Service) Releases(ctx context.Context, groupMBID string) (Editions, error) {
|
||||
q := dbq.New(s.pool)
|
||||
group, err := q.GetMBReleaseGroup(ctx, groupMBID)
|
||||
if err != nil && !errors.Is(err, pgx.ErrNoRows) {
|
||||
return Editions{}, fmt.Errorf("catalogue: read release group: %w", err)
|
||||
}
|
||||
held := err == nil && group.ReleasesRefreshedAt.Valid
|
||||
stale := false
|
||||
if !held || s.now().Sub(group.ReleasesRefreshedAt.Time) > ReleasesMaxAge {
|
||||
if rerr := s.RefreshReleaseGroup(ctx, groupMBID); rerr != nil {
|
||||
if !held || !errors.Is(rerr, musicbrainz.ErrUnreachable) {
|
||||
return Editions{}, rerr
|
||||
}
|
||||
stale = true
|
||||
}
|
||||
if group, err = q.GetMBReleaseGroup(ctx, groupMBID); err != nil {
|
||||
return Editions{}, fmt.Errorf("catalogue: read release group: %w", err)
|
||||
}
|
||||
}
|
||||
releases, err := q.ListMBReleasesForGroup(ctx, groupMBID)
|
||||
if err != nil {
|
||||
return Editions{}, fmt.Errorf("catalogue: read releases: %w", err)
|
||||
}
|
||||
tracks, err := q.ListMBReleaseTracksForGroup(ctx, groupMBID)
|
||||
if err != nil {
|
||||
return Editions{}, fmt.Errorf("catalogue: read track lists: %w", err)
|
||||
}
|
||||
byRelease := make(map[string][]dbq.MbReleaseTrack, len(releases))
|
||||
for _, t := range tracks {
|
||||
byRelease[t.ReleaseMbid] = append(byRelease[t.ReleaseMbid], t)
|
||||
}
|
||||
out := Editions{Group: group, Stale: stale}
|
||||
for _, r := range releases {
|
||||
out.Editions = append(out.Editions, Edition{Release: r, Tracks: byRelease[r.Mbid]})
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// RefreshReleaseGroup reads a group's releases and track lists and replaces
|
||||
// the held copy in one transaction. A group not yet held is looked up first.
|
||||
func (s *Service) RefreshReleaseGroup(ctx context.Context, groupMBID string) error {
|
||||
@@ -254,60 +103,6 @@ func (s *Service) RefreshReleaseGroup(ctx context.Context, groupMBID string) err
|
||||
})
|
||||
}
|
||||
|
||||
// RefreshDue re-reads up to limit discographies that are older than
|
||||
// DiscographyMaxAge, never-read first. It stops at the first unreachable
|
||||
// MusicBrainz, since the rest would fail the same way.
|
||||
func (s *Service) RefreshDue(ctx context.Context, limit int) (refreshed int, err error) {
|
||||
due, err := dbq.New(s.pool).ListMBArtistsDueForRefresh(ctx, dbq.ListMBArtistsDueForRefreshParams{
|
||||
Before: pgtype.Timestamptz{Time: s.now().Add(-DiscographyMaxAge), Valid: true},
|
||||
MaxRows: int32(limit),
|
||||
})
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("catalogue: list due artists: %w", err)
|
||||
}
|
||||
for _, mbid := range due {
|
||||
if err := s.RefreshArtist(ctx, mbid); err != nil {
|
||||
if errors.Is(err, musicbrainz.ErrUnreachable) || ctx.Err() != nil {
|
||||
return refreshed, err
|
||||
}
|
||||
// One artist MusicBrainz cannot describe (merged away, say)
|
||||
// must not stop the rest.
|
||||
s.logger.Warn("catalogue: refresh artist", "mbid", mbid, "err", err)
|
||||
continue
|
||||
}
|
||||
refreshed++
|
||||
}
|
||||
return refreshed, nil
|
||||
}
|
||||
|
||||
// RefreshInterval is how often Run looks for discographies due a re-read,
|
||||
// and RefreshBatch how many it reads per look: 20 artists is about a minute
|
||||
// of MusicBrainz's budget.
|
||||
const (
|
||||
RefreshInterval = time.Hour
|
||||
RefreshBatch = 20
|
||||
)
|
||||
|
||||
// Run refreshes due discographies every RefreshInterval until ctx ends.
|
||||
func (s *Service) Run(ctx context.Context) {
|
||||
ticker := time.NewTicker(RefreshInterval)
|
||||
defer ticker.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-ticker.C:
|
||||
}
|
||||
n, err := s.RefreshDue(ctx, RefreshBatch)
|
||||
switch {
|
||||
case err != nil && ctx.Err() == nil:
|
||||
s.logger.Warn("catalogue: refresh pass stopped", "refreshed", n, "err", err)
|
||||
case n > 0:
|
||||
s.logger.Info("catalogue: refresh pass complete", "refreshed", n)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// mbErr maps MusicBrainz's not-found to the catalogue's; anything else passes
|
||||
// through so callers can tell an unreachable MusicBrainz apart.
|
||||
func mbErr(err error) error {
|
||||
|
||||
@@ -9,14 +9,13 @@ import (
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
|
||||
"git.fabledsword.com/bvandeusen/minstrel/internal/db"
|
||||
"git.fabledsword.com/bvandeusen/minstrel/internal/db/dbq"
|
||||
"git.fabledsword.com/bvandeusen/minstrel/internal/dbtest"
|
||||
"git.fabledsword.com/bvandeusen/minstrel/internal/musicbrainz"
|
||||
)
|
||||
@@ -44,15 +43,13 @@ func newPool(t *testing.T) *pgxpool.Pool {
|
||||
|
||||
const (
|
||||
gorillaz = "e21857d5-3256-4547-afb3-4b6ded592596"
|
||||
deLaSoul = "4f2a6b9e-9ae6-4f34-9a41-3e9f2f7c4c1d"
|
||||
demonDays = "0c4d5b5f-2f3c-3d31-9c4e-4b4f6b4c1a01"
|
||||
feelGood = "0c4d5b5f-2f3c-3d31-9c4e-4b4f6b4c1a02" // credited to both
|
||||
demonDaysR = "1a2b3c4d-0000-4000-8000-000000000001"
|
||||
demonDaysJ = "1a2b3c4d-0000-4000-8000-000000000002"
|
||||
)
|
||||
|
||||
// fakeMB is a MusicBrainz with two artists, a collaboration between them,
|
||||
// and one album with two editions. down makes every request a 503.
|
||||
// fakeMB is a MusicBrainz with one album in two editions. down makes every
|
||||
// request a 503.
|
||||
type fakeMB struct {
|
||||
srv *httptest.Server
|
||||
calls atomic.Int32
|
||||
@@ -70,21 +67,6 @@ func newFakeMB(t *testing.T) *fakeMB {
|
||||
}
|
||||
q := r.URL.Query()
|
||||
switch {
|
||||
case r.URL.Path == "/artist/"+gorillaz:
|
||||
_, _ = fmt.Fprintf(w, `{"id": %q, "name": "Gorillaz", "sort-name": "Gorillaz", "type": "Group", "country": "GB"}`, gorillaz)
|
||||
case r.URL.Path == "/artist/"+deLaSoul:
|
||||
_, _ = fmt.Fprintf(w, `{"id": %q, "name": "De La Soul", "type": "Group", "country": "US"}`, deLaSoul)
|
||||
case r.URL.Path == "/release-group" && q.Get("artist") == gorillaz:
|
||||
_, _ = fmt.Fprintf(w, `{"release-group-count": 2, "release-groups": [
|
||||
{"id": %q, "title": "Demon Days", "primary-type": "Album", "first-release-date": "2005-05-11",
|
||||
"artist-credit": [{"name": "Gorillaz", "artist": {"id": %q}}]},
|
||||
{"id": %q, "title": "Feel Good Inc.", "primary-type": "Single", "secondary-types": [], "first-release-date": "2005-05-09",
|
||||
"artist-credit": [{"name": "Gorillaz", "joinphrase": " feat. ", "artist": {"id": %q}}, {"name": "De La Soul", "artist": {"id": %q}}]}
|
||||
]}`, demonDays, gorillaz, feelGood, gorillaz, deLaSoul)
|
||||
case r.URL.Path == "/release-group" && q.Get("artist") == deLaSoul:
|
||||
_, _ = fmt.Fprintf(w, `{"release-group-count": 1, "release-groups": [
|
||||
{"id": %q, "title": "Feel Good Inc.", "primary-type": "Single", "first-release-date": "2005-05-09"}
|
||||
]}`, feelGood)
|
||||
case r.URL.Path == "/release-group/"+demonDays:
|
||||
_, _ = fmt.Fprintf(w, `{"id": %q, "title": "Demon Days", "primary-type": "Album", "first-release-date": "2005-05-11",
|
||||
"artist-credit": [{"name": "Gorillaz", "artist": {"id": %q}}]}`, demonDays, gorillaz)
|
||||
@@ -110,138 +92,60 @@ func (f *fakeMB) client() *musicbrainz.Client {
|
||||
return &musicbrainz.Client{BaseURL: f.srv.URL, Pace: &musicbrainz.Pacer{}, Retries: -1}
|
||||
}
|
||||
|
||||
func TestDiscography_ReadsOnceThenServesTheHeldCopy_Integration(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
mb := newFakeMB(t)
|
||||
s := New(newPool(t), mb.client(), nil)
|
||||
|
||||
d, err := s.Discography(ctx, gorillaz)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if d.Artist.Name != "Gorillaz" || d.Artist.Country != "GB" || len(d.Groups) != 2 || d.Stale {
|
||||
t.Fatalf("discography = %+v", d)
|
||||
}
|
||||
// Oldest first: the single came out two days before the album.
|
||||
if d.Groups[0].Mbid != feelGood || d.Groups[1].PrimaryType != "Album" {
|
||||
t.Errorf("order = %s, %s", d.Groups[0].Title, d.Groups[1].Title)
|
||||
}
|
||||
if d.Groups[0].ArtistCredit != "Gorillaz feat. De La Soul" {
|
||||
t.Errorf("credit = %q", d.Groups[0].ArtistCredit)
|
||||
}
|
||||
|
||||
before := mb.calls.Load()
|
||||
if _, err := s.Discography(ctx, gorillaz); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if mb.calls.Load() != before {
|
||||
t.Error("a fresh discography should be served without asking MusicBrainz")
|
||||
}
|
||||
|
||||
// The collaboration belongs to both artists.
|
||||
dls, err := s.Discography(ctx, deLaSoul)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(dls.Groups) != 1 || dls.Groups[0].Mbid != feelGood {
|
||||
t.Errorf("De La Soul's discography = %+v", dls.Groups)
|
||||
}
|
||||
if again, _ := s.Discography(ctx, gorillaz); len(again.Groups) != 2 {
|
||||
t.Error("refreshing one artist must not take a shared group from the other")
|
||||
}
|
||||
}
|
||||
|
||||
func TestDiscography_UnreachableServesStaleOrFails_Integration(t *testing.T) {
|
||||
func TestRefreshReleaseGroup_ReadsEditionsAndTrackLists_Integration(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
mb := newFakeMB(t)
|
||||
pool := newPool(t)
|
||||
s := New(pool, mb.client(), nil)
|
||||
|
||||
mb.down.Store(true)
|
||||
if _, err := s.Discography(ctx, gorillaz); !errors.Is(err, musicbrainz.ErrUnreachable) {
|
||||
t.Fatalf("nothing held and MusicBrainz down: err = %v, want ErrUnreachable", err)
|
||||
}
|
||||
|
||||
mb.down.Store(false)
|
||||
if _, err := s.Discography(ctx, gorillaz); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// A week and a day later, MusicBrainz is down: the held copy is served.
|
||||
s.now = func() time.Time { return time.Now().Add(DiscographyMaxAge + 24*time.Hour) }
|
||||
mb.down.Store(true)
|
||||
d, err := s.Discography(ctx, gorillaz)
|
||||
if err != nil || !d.Stale || len(d.Groups) != 2 {
|
||||
t.Fatalf("stale copy: %+v, %v", d, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDiscography_UnknownArtistIsNotFound_Integration(t *testing.T) {
|
||||
s := New(newPool(t), newFakeMB(t).client(), nil)
|
||||
if _, err := s.Discography(context.Background(), "00000000-0000-4000-8000-000000000000"); !errors.Is(err, ErrNotFound) {
|
||||
t.Fatalf("err = %v, want ErrNotFound", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestReleases_ReadsEditionsAndTrackLists_Integration(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
mb := newFakeMB(t)
|
||||
s := New(newPool(t), mb.client(), nil)
|
||||
q := dbq.New(pool)
|
||||
|
||||
// The group isn't held yet: it is looked up first.
|
||||
e, err := s.Releases(ctx, demonDays)
|
||||
if err := s.RefreshReleaseGroup(ctx, demonDays); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
group, err := q.GetMBReleaseGroup(ctx, demonDays)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if e.Group.Title != "Demon Days" || len(e.Editions) != 2 {
|
||||
t.Fatalf("editions = %+v", e)
|
||||
if group.Title != "Demon Days" || !group.ReleasesRefreshedAt.Valid {
|
||||
t.Fatalf("group = %+v", group)
|
||||
}
|
||||
official := e.Editions[0]
|
||||
if official.Release.Mbid != demonDaysR || official.Release.TrackCount != 2 || official.Release.Formats != "CD" {
|
||||
t.Errorf("the official edition should come first: %+v", official.Release)
|
||||
}
|
||||
if len(official.Tracks) != 2 || official.Tracks[1].RecordingMbid != "rec-lls" || official.Tracks[1].LengthMs != 190000 {
|
||||
t.Errorf("tracks = %+v", official.Tracks)
|
||||
}
|
||||
|
||||
before := mb.calls.Load()
|
||||
if _, err := s.Releases(ctx, demonDays); err != nil {
|
||||
tracks, err := q.ListMBReleaseTracksForGroup(ctx, demonDays)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if mb.calls.Load() != before {
|
||||
t.Error("fresh editions should be served without asking MusicBrainz")
|
||||
if len(tracks) != 3 {
|
||||
t.Fatalf("want 3 tracks across both editions, got %+v", tracks)
|
||||
}
|
||||
var lls *dbq.MbReleaseTrack
|
||||
for i := range tracks {
|
||||
if tracks[i].RecordingMbid == "rec-lls" {
|
||||
lls = &tracks[i]
|
||||
}
|
||||
}
|
||||
if lls == nil || lls.ReleaseMbid != demonDaysR || lls.LengthMs != 190000 {
|
||||
t.Errorf("Last Living Souls = %+v", lls)
|
||||
}
|
||||
|
||||
// Refreshing replaces rather than adds.
|
||||
if err := s.RefreshReleaseGroup(ctx, demonDays); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if again, _ := s.Releases(ctx, demonDays); len(again.Editions) != 2 || len(again.Editions[0].Tracks) != 2 {
|
||||
t.Errorf("after a refresh: %+v", again)
|
||||
if again, _ := q.ListMBReleaseTracksForGroup(ctx, demonDays); len(again) != 3 {
|
||||
t.Errorf("after a refresh: %d tracks, want 3", len(again))
|
||||
}
|
||||
}
|
||||
|
||||
func TestRefreshDue_ReadsWhatIsOldAndStopsWhenMusicBrainzIsDown_Integration(t *testing.T) {
|
||||
func TestRefreshReleaseGroup_UnknownAndUnreachable_Integration(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
mb := newFakeMB(t)
|
||||
s := New(newPool(t), mb.client(), nil)
|
||||
for _, a := range []string{gorillaz, deLaSoul} {
|
||||
if err := s.RefreshArtist(ctx, a); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if n, err := s.RefreshDue(ctx, 10); err != nil || n != 0 {
|
||||
t.Fatalf("nothing is due yet: %d, %v", n, err)
|
||||
}
|
||||
|
||||
s.now = func() time.Time { return time.Now().Add(DiscographyMaxAge + time.Hour) }
|
||||
if n, err := s.RefreshDue(ctx, 1); err != nil || n != 1 {
|
||||
t.Errorf("one due, limit one: %d, %v", n, err)
|
||||
if err := s.RefreshReleaseGroup(ctx, "00000000-0000-4000-8000-000000000000"); !errors.Is(err, ErrNotFound) {
|
||||
t.Errorf("unknown group: err = %v, want ErrNotFound", err)
|
||||
}
|
||||
mb.down.Store(true)
|
||||
n, err := s.RefreshDue(ctx, 10)
|
||||
if !errors.Is(err, musicbrainz.ErrUnreachable) || n != 0 {
|
||||
t.Errorf("MusicBrainz down: %d, %v", n, err)
|
||||
}
|
||||
if !strings.Contains(err.Error(), "503") {
|
||||
t.Errorf("the error should say what MusicBrainz answered: %v", err)
|
||||
if err := s.RefreshReleaseGroup(ctx, demonDays); !errors.Is(err, musicbrainz.ErrUnreachable) {
|
||||
t.Errorf("MusicBrainz down: err = %v, want ErrUnreachable", err)
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user