feat: search and grab — music release names, a quality ladder, a paced search schedule and one live grab per want (M502 #5521)

- internal/release: reads a release name (format, bitrate, bit depth and sample rate, source, year, video, discography) and judges it against the album wanted: the artist and title must be in it; video rips, discographies, other versions (drumless, instrumental, karaoke, remix, live unless the album is live…) and singles or EPs for an album are refused. Quality tiers on Lidarr's ladder; a .torrent's file list is read for format, video files and a track count short of the album. - internal/acquire: fan-out across every search source with a narrowed search kept, a process-wide pacer per source, every candidate judged and kept with its reasons, and the grab: one live grab per want, tie-broken (a person's grab beats the schedule's; a better release replaces a worse one unless it is nearly done). Each download lands in its own folder under the client's downloads folder. - internal/wanted: the search schedule, every ten minutes, never-searched first, then requests, missing files and followed artists, taking the best release when auto-grab is on (the default). - Migration 0082: acquisition_settings (auto-grab, interval, allowed qualities seeded from Lidarr's "Any"), want search columns, want_candidates, grabs, release_blocks. - Web: the Wanted page shows each want's last search, a Search button and every release found with its verdict (Grab, Grab anyway, Never this one, Allow again). Integrations gains "Searching and grabbing". Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-10 08:56:47 -04:00
parent 1a1646a920
commit 89b3b44779
40 changed files with 5332 additions and 49 deletions
@@ -0,0 +1,322 @@
+package wanted
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"log/slog"
+	"strconv"
+	"time"
+
+	"github.com/jackc/pgx/v5/pgtype"
+	"github.com/jackc/pgx/v5/pgxpool"
+
+	"git.fabledsword.com/bvandeusen/minstrel/internal/acquire"
+	"git.fabledsword.com/bvandeusen/minstrel/internal/catalogue"
+	"git.fabledsword.com/bvandeusen/minstrel/internal/db/dbq"
+	"git.fabledsword.com/bvandeusen/minstrel/internal/indexer"
+	"git.fabledsword.com/bvandeusen/minstrel/internal/release"
+)
+
+// Searching for wants on a schedule (M502 #5521), ported from Fabled
+// Librarian's internal/wanted. The schedule decides which wants and when;
+// internal/acquire judges what a search finds and grabs.
+
+// Search outcomes, matching wants_search_outcome_check.
+const (
+	OutcomeGrabbed = "grabbed"
+	OutcomeWaiting = "waiting"
+	OutcomeNothing = "nothing"
+	OutcomeError   = "error"
+)
+
+const (
+	// searchTick is how often the schedule looks for wants due; the
+	// operator's setting is how long a want waits between searches.
+	searchTick = 10 * time.Minute
+	// searchWindow is how long one pass searches: most of a tick, so a pass
+	// ends before the next begins.
+	searchWindow = 9 * time.Minute
+	// duePage bounds what one pass reads. The sources' pace is the real
+	// limit; this caps a pass whose sources are all unpaced.
+	duePage = 200
+	// grabTries is how many grabbable releases one search tries in turn
+	// when the first is refused for its files.
+	grabTries = 3
+	// passTimeout backs up the window for a call none of the clients'
+	// own timeouts cover, such as a wedged database read.
+	passTimeout = 15 * time.Minute
+)
+
+// ErrWantHeld: the want is already answered by the library.
+var ErrWantHeld = errors.New("wanted: the library already holds this album")
+
+// Searcher searches for wants and grabs what they need.
+type Searcher struct {
+	pool   *pgxpool.Pool
+	cat    *catalogue.Service
+	logger *slog.Logger
+}
+
+// NewSearcher returns a Searcher reading editions through cat (nil reads
+// only what the catalogue already holds).
+func NewSearcher(pool *pgxpool.Pool, cat *catalogue.Service, logger *slog.Logger) *Searcher {
+	if logger == nil {
+		logger = slog.Default()
+	}
+	return &Searcher{pool: pool, cat: cat, logger: logger}
+}
+
+// Report is one pass.
+type Report struct {
+	Looked  int `json:"looked"`
+	Grabbed int `json:"grabbed"`
+	Waiting int `json:"waiting"`
+	Nothing int `json:"nothing"`
+	Failed  int `json:"failed"`
+	// Paced: the pass stopped because the next search would wait past its
+	// window for a source's pace.
+	Paced bool `json:"paced"`
+	// Off: the schedule is switched off. Unconfigured: nothing to search
+	// with. Both are states, not faults.
+	Off          bool   `json:"off"`
+	Unconfigured bool   `json:"unconfigured"`
+	AutoGrab     bool   `json:"auto_grab"`
+	Error        string `json:"error,omitempty"`
+}
+
+// Run searches every searchTick until ctx ends.
+func (s *Searcher) Run(ctx context.Context) {
+	ticker := time.NewTicker(searchTick)
+	defer ticker.Stop()
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		case <-ticker.C:
+		}
+		rep := s.Pass(ctx)
+		if rep.Looked > 0 || rep.Error != "" {
+			s.logger.Info("wanted: search pass", "looked", rep.Looked, "grabbed", rep.Grabbed,
+				"waiting", rep.Waiting, "nothing", rep.Nothing, "failed", rep.Failed,
+				"paced", rep.Paced, "error", rep.Error)
+		}
+	}
+}
+
+// Pass searches for the wants due, paced by the sources, until the window
+// closes, and records what it did.
+func (s *Searcher) Pass(ctx context.Context) Report {
+	rep := s.pass(ctx)
+	if body, err := json.Marshal(rep); err == nil {
+		if err := dbq.New(s.pool).RecordSearchPass(ctx, body); err != nil && ctx.Err() == nil {
+			s.logger.Warn("wanted: record search pass", "err", err)
+		}
+	}
+	return rep
+}
+
+func (s *Searcher) pass(ctx context.Context) Report {
+	var rep Report
+	ctx, cancel := context.WithTimeout(ctx, passTimeout)
+	defer cancel()
+	svc, settings, err := acquire.Build(ctx, s.pool)
+	if err != nil {
+		rep.Error = indexer.FirstLine(err.Error())
+		return rep
+	}
+	rep.AutoGrab = settings.AutoGrab
+	switch {
+	case settings.SearchIntervalHours <= 0:
+		rep.Off = true
+		return rep
+	case svc.Search == nil:
+		rep.Unconfigured = true
+		return rep
+	}
+	deadline := time.Now().Add(searchWindow)
+	due, err := dbq.New(s.pool).ListWantsDueForSearch(ctx, dbq.ListWantsDueForSearchParams{
+		Hours: settings.SearchIntervalHours, MaxRows: duePage,
+	})
+	if err != nil {
+		rep.Error = indexer.FirstLine(err.Error())
+		return rep
+	}
+	for _, row := range due {
+		if !svc.Ready(ctx, deadline) {
+			rep.Paced = ctx.Err() == nil
+			return rep
+		}
+		want := s.wantFor(ctx, row.ID, row.ArtistName, row.Title, row.PrimaryType, row.SecondaryTypes,
+			row.FirstReleaseDate, row.ReleaseGroupMbid)
+		out := s.one(ctx, svc, want, settings.AutoGrab)
+		rep.Looked++
+		switch out.Outcome {
+		case OutcomeGrabbed:
+			rep.Grabbed++
+		case OutcomeWaiting:
+			rep.Waiting++
+		case OutcomeNothing:
+			rep.Nothing++
+		default:
+			rep.Failed++
+		}
+	}
+	return rep
+}
+
+// wantFor describes a want for the matcher, with its editions' track counts
+// from the catalogue (read from MusicBrainz the first time when it can be).
+func (s *Searcher) wantFor(ctx context.Context, id pgtype.UUID, artist, title, primary string, secondary []string, date, group string) acquire.Want {
+	w := acquire.Want{ID: id, Want: release.Want{
+		Artist: artist, Title: title, Year: yearOf(date), PrimaryType: primary, SecondaryTypes: secondary,
+	}}
+	q := dbq.New(s.pool)
+	span, err := q.EditionTrackSpan(ctx, group)
+	if err == nil && span.MinTracks == 0 && s.cat != nil {
+		if _, rerr := s.cat.Releases(ctx, group); rerr == nil {
+			span, err = q.EditionTrackSpan(ctx, group)
+		}
+	}
+	if err == nil {
+		w.MinTracks, w.MaxTracks = int(span.MinTracks), int(span.MaxTracks)
+	}
+	return w
+}
+
+func yearOf(date string) int {
+	if len(date) < 4 {
+		return 0
+	}
+	y, _ := strconv.Atoi(date[:4])
+	return y
+}
+
+// Searched is what one want's search came to.
+type Searched struct {
+	Outcome    string              `json:"outcome"`
+	Considered int                 `json:"considered"`
+	Best       string              `json:"best_release"`
+	Verdict    string              `json:"best_verdict"`
+	Grabbed    string              `json:"grabbed,omitempty"`
+	Narrowed   []acquire.Narrowing `json:"narrowed"`
+	Error      string              `json:"error,omitempty"`
+}
+
+// one searches for a want, takes the best release when auto-grab is on, and
+// records what came of it on the want.
+func (s *Searcher) one(ctx context.Context, svc *acquire.Service, want acquire.Want, autoGrab bool) Searched {
+	found := svc.For(ctx, want)
+	if found.KeepError != "" {
+		s.logger.Warn("wanted: searched, but could not keep the candidates", "want", want.Title, "err", found.KeepError)
+	}
+	out := Searched{Considered: found.Considered, Narrowed: found.Narrowed}
+	if out.Narrowed == nil {
+		out.Narrowed = []acquire.Narrowing{}
+	}
+	if len(found.Candidates) > 0 {
+		out.Best, out.Verdict = found.Candidates[0].Name, string(found.Candidates[0].Verdict)
+	}
+	switch {
+	case found.Unconfigured:
+		out.Outcome, out.Error = OutcomeError, "No indexer is set up, so there is nothing to search with."
+	case found.Error != "":
+		out.Outcome, out.Error = OutcomeError, found.Error
+	case !found.ForAPerson():
+		out.Outcome = OutcomeNothing
+	case len(found.Grabbable()) == 0 || !autoGrab:
+		out.Outcome = OutcomeWaiting
+	default:
+		out.Outcome = OutcomeWaiting
+		s.grabBest(ctx, svc, want, found, &out)
+	}
+	s.record(ctx, want.ID, out)
+	return out
+}
+
+// grabBest takes the best grabbable release, trying the next when one is
+// refused for what its files hold.
+func (s *Searcher) grabBest(ctx context.Context, svc *acquire.Service, want acquire.Want, found acquire.Result, out *Searched) {
+	for i, c := range found.Grabbable() {
+		if i == grabTries {
+			return
+		}
+		took := svc.Grab(ctx, want, &c, found.Candidates, false, acquire.Origin{By: acquire.OriginSchedule})
+		switch {
+		case took.Added:
+			out.Outcome, out.Grabbed, out.Best, out.Verdict = OutcomeGrabbed, c.Name, c.Name, string(acquire.Grab)
+			if took.Error != "" {
+				s.logger.Warn("wanted: took a release but could not record it", "want", want.Title, "err", took.Error)
+			}
+			s.logger.Info("wanted: took a release", "want", want.Title, "artist", want.Artist,
+				"release", c.Name, "quality", c.Tier, "save_path", took.SavePath)
+			return
+		case took.Refused:
+			continue
+		case took.Unconfigured:
+			out.Outcome, out.Error = OutcomeError, "No download client is set up, so nothing was taken."
+			return
+		case took.Contest != nil:
+			return
+		case took.Error != "":
+			out.Outcome, out.Error = OutcomeError, took.Error
+			return
+		}
+	}
+}
+
+func (s *Searcher) record(ctx context.Context, id pgtype.UUID, out Searched) {
+	if err := dbq.New(s.pool).RecordWantSearch(ctx, dbq.RecordWantSearchParams{
+		ID: id, Outcome: out.Outcome, Considered: int32(out.Considered),
+		BestRelease: out.Best, BestVerdict: out.Verdict, SearchError: out.Error,
+	}); err != nil && ctx.Err() == nil {
+		s.logger.Warn("wanted: record search", "err", err)
+	}
+}
+
+// SearchNow searches for one want at once, as the schedule would, for a
+// person pressing Search: it spends the sources' pace but never waits on it.
+func (s *Searcher) SearchNow(ctx context.Context, id pgtype.UUID) (Searched, error) {
+	want, err := s.load(ctx, id)
+	if err != nil {
+		return Searched{}, err
+	}
+	svc, settings, err := acquire.Build(ctx, s.pool)
+	if err != nil {
+		return Searched{}, err
+	}
+	return s.one(ctx, svc, want, settings.AutoGrab), nil
+}
+
+// Take grabs the release named by key for a person, searching again first.
+func (s *Searcher) Take(ctx context.Context, id pgtype.UUID, key string, force bool, by pgtype.UUID) (acquire.Taken, error) {
+	want, err := s.load(ctx, id)
+	if err != nil {
+		return acquire.Taken{}, err
+	}
+	svc, _, err := acquire.Build(ctx, s.pool)
+	if err != nil {
+		return acquire.Taken{}, err
+	}
+	took := svc.Take(ctx, want, key, force, acquire.Origin{By: acquire.OriginOperator, User: by})
+	if took.Added && took.Chosen != nil {
+		s.record(ctx, id, Searched{
+			Outcome: OutcomeGrabbed, Best: took.Chosen.Name, Verdict: string(took.Chosen.Verdict),
+			Considered: 0, Grabbed: took.Chosen.Name,
+		})
+	}
+	return took, nil
+}
+
+// load reads an open want for a search.
+func (s *Searcher) load(ctx context.Context, id pgtype.UUID) (acquire.Want, error) {
+	row, err := dbq.New(s.pool).GetWantForSearch(ctx, id)
+	if err != nil {
+		return acquire.Want{}, err
+	}
+	if row.FulfilledAt.Valid {
+		return acquire.Want{}, ErrWantHeld
+	}
+	return s.wantFor(ctx, row.ID, row.ArtistName, row.Title, row.PrimaryType, row.SecondaryTypes,
+		row.FirstReleaseDate, row.ReleaseGroupMbid), nil
+}
@@ -0,0 +1,195 @@
+package wanted
+
+import (
+	"context"
+	"encoding/json"
+	"io"
+	"log/slog"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"sync/atomic"
+	"testing"
+
+	"git.fabledsword.com/bvandeusen/minstrel/internal/acquire"
+	"git.fabledsword.com/bvandeusen/minstrel/internal/db/dbq"
+)
+
+const ramHash = "0123456789abcdef0123456789abcdef01234567"
+
+const ramFeed = `<?xml version="1.0"?>
+<rss xmlns:torznab="http://torznab.com/schemas/2015/feed"><channel>
+<item><title>Daft Punk - Random Access Memories (2013) [FLAC]</title>
+  <link>magnet:?xt=urn:btih:` + ramHash + `</link><size>400000000</size>
+  <torznab:attr name="seeders" value="12"/><torznab:attr name="indexer" value="Knaben"/></item>
+<item><title>Daft Punk Random Access Memories (Drumless Edition 2023) [Flac 24-88]</title>
+  <link>magnet:?xt=urn:btih:fedcba9876543210fedcba9876543210fedcba98</link><size>900000000</size>
+  <torznab:attr name="seeders" value="40"/></item>
+</channel></rss>`
+
+// searchStack is a fake indexer feed and a fake FabledExchange download
+// client, configured the way the operator would in Integrations.
+func searchStack(t *testing.T, ctx context.Context, store *acquire.Store) *atomic.Int32 {
+	t.Helper()
+	var added atomic.Int32
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		switch {
+		case r.URL.Path == "/feed":
+			if q := r.URL.Query().Get("q"); q != "Daft Punk Random Access Memories" {
+				t.Errorf("searched for %q", q)
+			}
+			_, _ = io.WriteString(w, ramFeed)
+		case r.URL.Path == "/api/downloads" && r.Method == http.MethodPost:
+			var body struct {
+				Magnet   string `json:"magnet"`
+				SavePath string `json:"save_path"`
+			}
+			_ = json.NewDecoder(r.Body).Decode(&body)
+			if !strings.Contains(body.Magnet, ramHash) || body.SavePath != "/downloads/"+ramHash {
+				t.Errorf("added %+v", body)
+			}
+			added.Add(1)
+			_, _ = io.WriteString(w, "{}")
+		default:
+			http.NotFound(w, r)
+		}
+	}))
+	t.Cleanup(srv.Close)
+
+	feed, err := store.Create(ctx, acquire.ProtocolTorznab)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if _, err := store.Save(ctx, feed.Code, acquire.Update{
+		Label: "Feed", Enabled: true, BaseURL: srv.URL + "/feed", TimeoutMs: 5000, LocalPath: "/downloads",
+	}); err != nil {
+		t.Fatal(err)
+	}
+	ex, err := store.Create(ctx, acquire.ProtocolExchange)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if _, err := store.Save(ctx, ex.Code, acquire.Update{
+		Label: "Exchange", Enabled: true, BaseURL: srv.URL, TimeoutMs: 5000, ServesDownloads: true, LocalPath: "/downloads",
+	}); err != nil {
+		t.Fatal(err)
+	}
+	return &added
+}
+
+func TestSearcher_FindsAndTakesAWant_Integration(t *testing.T) {
+	ctx := context.Background()
+	pool := newPool(t)
+	q := dbq.New(pool)
+	added := searchStack(t, ctx, acquire.NewStore(pool))
+	w, err := WantWith(ctx, q, WantParams{
+		GroupMBID: "rg-ram", ArtistMBID: "mb-daft-punk", ArtistName: "Daft Punk", Title: "Random Access Memories",
+		PrimaryType: "Album", FirstReleaseDate: "2013-05-17", Reason: ReasonRequest,
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+	s := NewSearcher(pool, nil, slog.New(slog.NewTextHandler(io.Discard, nil)))
+
+	rep := s.Pass(ctx)
+	if rep.Looked != 1 || rep.Grabbed != 1 || !rep.AutoGrab || added.Load() != 1 {
+		t.Fatalf("pass = %+v, added %d", rep, added.Load())
+	}
+	got, err := q.GetWantByReleaseGroup(ctx, "rg-ram")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if got.SearchOutcome != OutcomeGrabbed || got.SearchConsidered != 2 || !strings.Contains(got.BestRelease, "(2013) [FLAC]") {
+		t.Errorf("want after the pass: %s %d %q", got.SearchOutcome, got.SearchConsidered, got.BestRelease)
+	}
+	cands, err := q.ListWantCandidates(ctx, w.ID)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(cands) != 2 || cands[0].Verdict != "grab" || cands[1].Verdict != "rejected" || !strings.Contains(cands[1].Reason, "drumless") {
+		t.Errorf("candidates = %+v", cands)
+	}
+	g, err := q.GetLiveGrab(ctx, w.ID)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if g.InfoHash != ramHash || g.Client != "exchange" || g.Origin != acquire.OriginSchedule || g.Quality != "lossless" {
+		t.Errorf("grab = %+v", g)
+	}
+	settings, _ := q.GetAcquisitionSettings(ctx)
+	if !settings.LastPassAt.Valid || !strings.Contains(string(settings.LastPass), `"grabbed":1`) {
+		t.Errorf("last pass = %s", settings.LastPass)
+	}
+
+	// A want with a download running is not searched again.
+	if rep := s.Pass(ctx); rep.Looked != 0 {
+		t.Errorf("searched a want already downloading: %+v", rep)
+	}
+
+	// Setting the drumless edition aside marks it on every kept candidate;
+	// allowing it again leaves it for a person until the next search.
+	mem := acquire.NewMemory(pool)
+	if err := mem.Block(ctx, dbq.BlockReleaseParams{
+		InfoHash: cands[1].InfoHash, Name: cands[1].Name, Reason: acquire.BlockOperator, WantID: w.ID,
+	}); err != nil {
+		t.Fatal(err)
+	}
+	c, _ := q.GetWantCandidate(ctx, dbq.GetWantCandidateParams{WantID: w.ID, Key: cands[1].Key})
+	if c.Verdict != "blocked" {
+		t.Errorf("blocked candidate = %s", c.Verdict)
+	}
+	if ok, err := mem.Unblock(ctx, cands[1].InfoHash); err != nil || !ok {
+		t.Fatalf("unblock: %v %v", ok, err)
+	}
+	c, _ = q.GetWantCandidate(ctx, dbq.GetWantCandidateParams{WantID: w.ID, Key: cands[1].Key})
+	if c.Verdict != "review" {
+		t.Errorf("unblocked candidate = %s", c.Verdict)
+	}
+}
+
+func TestSearcher_OffAndUnconfigured_Integration(t *testing.T) {
+	ctx := context.Background()
+	pool := newPool(t)
+	s := NewSearcher(pool, nil, slog.New(slog.NewTextHandler(io.Discard, nil)))
+	if rep := s.Pass(ctx); !rep.Unconfigured {
+		t.Errorf("no indexer: %+v", rep)
+	}
+	if _, err := dbq.New(pool).UpdateAcquisitionSettings(ctx, dbq.UpdateAcquisitionSettingsParams{
+		AutoGrab: true, SearchIntervalHours: 0, AllowedQualities: []string{"lossless"},
+	}); err != nil {
+		t.Fatal(err)
+	}
+	if rep := s.Pass(ctx); !rep.Off {
+		t.Errorf("interval 0: %+v", rep)
+	}
+}
+
+func TestGrabs_OneLivePerWant_Integration(t *testing.T) {
+	ctx := context.Background()
+	pool := newPool(t)
+	q := dbq.New(pool)
+	w, err := WantWith(ctx, q, WantParams{GroupMBID: "rg-x", ArtistMBID: "mb-x", ArtistName: "X", Title: "Y", Reason: ReasonFollow})
+	if err != nil {
+		t.Fatal(err)
+	}
+	mem := acquire.NewMemory(pool)
+	grab := func(hash string) dbq.InsertGrabParams {
+		return dbq.InsertGrabParams{WantID: w.ID, InfoHash: hash, Name: hash, Key: hash, Client: "qbittorrent",
+			SavePath: "/downloads/" + hash, Origin: acquire.OriginSchedule}
+	}
+	first, err := mem.Grabbed(ctx, grab("aaaa"), dbq.Grab{}.ID, "")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if _, err := mem.Grabbed(ctx, grab("bbbb"), dbq.Grab{}.ID, ""); err == nil {
+		t.Error("a second live grab for one want was recorded")
+	}
+	second, err := mem.Grabbed(ctx, grab("bbbb"), first.ID, "Replaced by bbbb.")
+	if err != nil {
+		t.Fatalf("replacing: %v", err)
+	}
+	live, ok, err := mem.LiveGrab(ctx, w.ID)
+	if err != nil || !ok || live.ID != second.ID {
+		t.Errorf("live = %+v %v %v", live, ok, err)
+	}
+}