feat: search and grab — music release names, a quality ladder, a paced search schedule and one live grab per want (M502 #5521)
release / govulncheck (push) Successful in 21s
release / go (push) Failing after 33s
release / web (push) Successful in 56s
release / integration (push) Failing after 4m48s
release / android (push) Successful in 5m6s
release / Build signed APK (releases and dev) (push) Successful in 5m16s
release / Attach APK to the Release (tag releases only) (push) Skipped
release / Build + push container image (push) Skipped
release / Verify release artifacts (tag releases only) (push) Skipped

- internal/release: reads a release name (format, bitrate, bit depth and
  sample rate, source, year, video, discography) and judges it against the
  album wanted: the artist and title must be in it; video rips,
  discographies, other versions (drumless, instrumental, karaoke, remix,
  live unless the album is live…) and singles or EPs for an album are
  refused. Quality tiers on Lidarr's ladder; a .torrent's file list is read
  for format, video files and a track count short of the album.
- internal/acquire: fan-out across every search source with a narrowed
  search kept, a process-wide pacer per source, every candidate judged and
  kept with its reasons, and the grab: one live grab per want, tie-broken
  (a person's grab beats the schedule's; a better release replaces a worse
  one unless it is nearly done). Each download lands in its own folder
  under the client's downloads folder.
- internal/wanted: the search schedule, every ten minutes, never-searched
  first, then requests, missing files and followed artists, taking the
  best release when auto-grab is on (the default).
- Migration 0082: acquisition_settings (auto-grab, interval, allowed
  qualities seeded from Lidarr's "Any"), want search columns,
  want_candidates, grabs, release_blocks.
- Web: the Wanted page shows each want's last search, a Search button and
  every release found with its verdict (Grab, Grab anyway, Never this one,
  Allow again). Integrations gains "Searching and grabbing".

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
2026-10-10 08:56:47 -04:00
co-authored by Claude Opus 5.5
parent 1a1646a920
commit 89b3b44779
40 changed files with 5332 additions and 49 deletions
+6
View File
@@ -322,6 +322,12 @@ func run() error {
// daily for new releases and marks wants held once the library has them.
go wanted.New(pool, cat, logger.With("component", "wanted")).Run(ctx)
// Search and grab (M502 #5521): every ten minutes, searches the
// configured indexers for wants due, paced by each source's
// searches-per-hour, and hands the best release to the download client
// when auto-grab is on.
go wanted.NewSearcher(pool, cat, logger.With("component", "search")).Run(ctx)
// Duplicate resolver (M498, M502): classifies the sweep's groups and, with
// the operator's auto-resolve on, merges the copies that are safe to merge.
// Started after SetNotifier so its summaries reach admins. See
+45
View File
@@ -0,0 +1,45 @@
package acquire
import (
"context"
"fmt"
"github.com/jackc/pgx/v5/pgxpool"
"git.fabledsword.com/bvandeusen/minstrel/internal/db/dbq"
"git.fabledsword.com/bvandeusen/minstrel/internal/release"
)
// Build makes a Service from the settings as they stand: every enabled
// search source behind one paced Fanout, the first enabled download client,
// and the quality profile. Built afresh for each pass and each request, so a
// changed setting takes effect on the next one.
func Build(ctx context.Context, pool *pgxpool.Pool) (*Service, dbq.AcquisitionSetting, error) {
q := dbq.New(pool)
settings, err := q.GetAcquisitionSettings(ctx)
if err != nil {
return nil, settings, fmt.Errorf("acquire: read settings: %w", err)
}
sources, err := Sources(ctx, pool)
if err != nil {
return nil, settings, fmt.Errorf("acquire: read search sources: %w", err)
}
clients, err := q.ListUsableAcquisitionServices(ctx, JobDownload)
if err != nil {
return nil, settings, fmt.Errorf("acquire: read download clients: %w", err)
}
s := &Service{Profile: release.NewProfile(settings.AllowedQualities), Memory: NewMemory(pool)}
fan := &Fanout{Sources: sources, Pace: SearchPace}
if len(sources) > 0 {
s.Search = fan
}
if len(clients) > 0 {
row := clients[0]
client, err := DownloadClient(row)
if err != nil {
return nil, settings, fmt.Errorf("acquire: download client %s: %w", row.Code, err)
}
s.Grabber = &Downloader{Client: client, Code: row.Code, Landing: row.LocalPath, Fetch: fan}
}
return s, settings, nil
}
+165
View File
@@ -0,0 +1,165 @@
package acquire
import (
"context"
"errors"
"fmt"
"strings"
"sync"
"git.fabledsword.com/bvandeusen/minstrel/internal/indexer"
)
// Searching every source at once (M502 #5521), ported from Fabled Librarian
// (#4182). A source that is down narrows a search rather than failing it: the
// results come back with the list of who did not answer.
// SourceFailure is one source that could not be reached.
type SourceFailure struct {
Code string
Label string
Err error
}
// PartialError is a search that reached some sources and not others. It is
// returned alongside the results, not instead of them.
type PartialError struct {
Failures []SourceFailure
// Answered is how many sources did answer. Zero means the search failed.
Answered int
}
func (e *PartialError) Error() string {
if e == nil || len(e.Failures) == 0 {
return "some sources did not answer"
}
parts := make([]string, 0, len(e.Failures))
for _, f := range e.Failures {
parts = append(parts, fmt.Sprintf("%s: %s", f.Label, indexer.FirstLine(f.Err.Error())))
}
return "some sources did not answer — " + strings.Join(parts, "; ")
}
// Unwrap lets errors.Is find indexer.ErrUnreachable through a partial failure.
func (e *PartialError) Unwrap() []error {
if e == nil {
return nil
}
out := make([]error, 0, len(e.Failures))
for _, f := range e.Failures {
out = append(out, f.Err)
}
return out
}
// Fanout searches every source and merges what comes back. It is both the
// Searcher and the Fetcher, routing a .torrent fetch to the source that
// offered the result.
type Fanout struct {
// Sources in code order: the fold keeps a duplicate's first arrival, so
// the order decides which source a grab is fetched through.
Sources []Source
// Pace spaces each source's searches. Nil paces nothing.
Pace *Pacer
}
// Search asks every source concurrently. Each client bounds its own call by
// its configured timeout, so there is no shared deadline here.
func (f *Fanout) Search(ctx context.Context, query indexer.Query) ([]indexer.Result, error) {
if f == nil || len(f.Sources) == 0 {
return nil, indexer.ErrUnconfigured
}
found := make([][]indexer.Result, len(f.Sources))
errs := make([]error, len(f.Sources))
var wg sync.WaitGroup
for i, src := range f.Sources {
if src.Search == nil {
continue
}
f.Pace.spend(src)
wg.Add(1)
go func(i int, src Source) {
defer wg.Done()
found[i], errs[i] = src.Search.Search(ctx, query)
}(i, src)
}
wg.Wait()
out := make([]indexer.Result, 0, 32)
var failures []SourceFailure
answered, unconfigured := 0, 0
for i, src := range f.Sources {
if src.Search == nil {
continue
}
if errors.Is(errs[i], indexer.ErrUnconfigured) {
unconfigured++
continue
}
if rl, ok := indexer.AsRateLimited(errs[i]); ok {
f.Pace.hold(src.Code, rl)
}
if errs[i] != nil {
failures = append(failures, SourceFailure{Code: src.Code, Label: src.Label, Err: errs[i]})
continue
}
answered++
for j := range found[i] {
if found[i][j].Source == "" {
found[i][j].Source = src.Code
}
}
out = append(out, found[i]...)
}
switch {
case answered == 0 && len(failures) > 0:
return nil, &PartialError{Failures: failures}
case len(failures) > 0:
return out, &PartialError{Failures: failures, Answered: answered}
case answered == 0 && unconfigured > 0:
return nil, indexer.ErrUnconfigured
}
return out, nil
}
// Download fetches a result's .torrent from the source that offered it: a
// Prowlarr link carries Prowlarr's key and only Prowlarr can redeem it.
func (f *Fanout) Download(ctx context.Context, r indexer.Result) ([]byte, error) {
if f == nil {
return nil, indexer.ErrUnconfigured
}
for _, src := range f.Sources {
if src.Code != r.Source {
continue
}
if src.Fetch == nil {
return nil, fmt.Errorf("%w: %s cannot fetch torrent files", indexer.ErrUnconfigured, src.Label)
}
return src.Fetch.Download(ctx, r)
}
if r.Source == "" {
return nil, fmt.Errorf("%w: this result does not say which source it came from", indexer.ErrUnconfigured)
}
return nil, fmt.Errorf("%w: %q is no longer a configured source", indexer.ErrUnconfigured, r.Source)
}
// Narrowing is one source that did not answer, as a page shows it.
type Narrowing struct {
Source string `json:"source"`
Label string `json:"label"`
Reason string `json:"reason"`
}
// narrowings reads a PartialError out of err, and whether any source
// answered. Nil when err is not a partial failure.
func narrowings(err error) ([]Narrowing, bool) {
var partial *PartialError
if !errors.As(err, &partial) || len(partial.Failures) == 0 {
return nil, false
}
out := make([]Narrowing, 0, len(partial.Failures))
for _, f := range partial.Failures {
out = append(out, Narrowing{Source: f.Code, Label: f.Label, Reason: indexer.FirstLine(f.Err.Error())})
}
return out, partial.Answered > 0
}
+307
View File
@@ -0,0 +1,307 @@
package acquire
import (
"context"
"errors"
"path"
"strings"
"sync"
"github.com/jackc/pgx/v5/pgtype"
"git.fabledsword.com/bvandeusen/minstrel/internal/db/dbq"
"git.fabledsword.com/bvandeusen/minstrel/internal/downloader"
"git.fabledsword.com/bvandeusen/minstrel/internal/indexer"
"git.fabledsword.com/bvandeusen/minstrel/internal/release"
)
// Taking a release (M502 #5521), ported from Fabled Librarian's grab.go and
// rivals.go. One live grab per want: a second is tie-broken against the
// download already running (FL #4470).
// Tag goes on every torrent Minstrel adds, so the operator can tell them
// from their own in the client.
const Tag = "minstrel"
// MissingClient is Taken.Missing when no download client is set up.
const MissingClient = "download_client"
// nearlyDone is how far along a running download stays whatever would
// replace it.
const nearlyDone = 0.9
// Downloader is the grab half of a Service.
type Downloader struct {
Client downloader.Driver
// Code is the client's acquisition_services row.
Code string
// Landing is the client's downloads folder as Minstrel sees it; each
// grab lands in a folder of its own under it.
Landing string
// Fetch gets a result's .torrent from the source that offered it.
Fetch Fetcher
}
// Origin says who took a release.
type Origin struct {
By string // OriginSchedule or OriginOperator
User pgtype.UUID
}
// Contest is how a grab fared against the download already running.
type Contest struct {
// Kept: the running one stays and this grab was not made. Replaced:
// this grab took its place.
Kept, Replaced bool
Reason string
Rival dbq.Grab
}
// Taken is what came of a grab.
type Taken struct {
Chosen *Candidate
// Added is the only field that means a download started.
Added bool
SavePath string
Grab dbq.Grab
// Gone: the search no longer offers the release.
Gone bool
// Refused: not taken; the reason is on Chosen.
Refused bool
// Unconfigured: Missing names what is not set up ("" for search).
Unconfigured bool
Missing string
Contest *Contest
Error string
}
// Take searches again and grabs the candidate named by key: a person's
// grab. Searching again keeps the indexer's download link (which can carry
// its key) on the server, and finds a release gone since it was shown.
// force takes what the verdict would not.
func (s *Service) Take(ctx context.Context, want Want, key string, force bool, origin Origin) Taken {
if s.Grabber == nil {
return Taken{Unconfigured: true, Missing: MissingClient}
}
found := s.For(ctx, want)
switch {
case found.Unconfigured:
return Taken{Unconfigured: true}
case found.Error != "":
return Taken{Error: found.Error}
}
chosen := pick(found.Candidates, key)
if chosen == nil {
return Taken{Gone: true}
}
return s.Grab(ctx, want, chosen, found.Candidates, force, origin)
}
// Grab hands a judged candidate to the download client. The schedule calls
// it with the candidate it has just judged; Take calls it after searching
// again.
func (s *Service) Grab(ctx context.Context, want Want, chosen *Candidate, all []Candidate, force bool, origin Origin) Taken {
out := Taken{Chosen: chosen}
if s.Grabber == nil || s.Grabber.Client == nil {
out.Unconfigured, out.Missing = true, MissingClient
return out
}
if chosen.Verdict == Blocked || (chosen.Verdict != Grab && !force) {
out.Refused = true
return out
}
unlock := lockWant(want.ID)
defer unlock()
contest := s.contest(ctx, want, chosen, all, origin)
if contest != nil && contest.Kept {
out.Contest = contest
return out
}
add := downloader.Add{Magnet: chosen.Source.Magnet, Tags: []string{Tag}}
hash := chosen.InfoHash
if add.Magnet == "" {
if s.Grabber.Fetch == nil {
out.Error = "This release is a .torrent link, and nothing is set up to fetch it."
return out
}
body, err := s.Grabber.Fetch.Download(ctx, chosen.Source)
if magnet, ok := indexer.AsMagnet(err); ok {
add.Magnet, err = magnet, nil
}
if err != nil {
out.Error = indexer.FirstLine(err.Error())
return out
}
if add.Magnet == "" {
add.Torrent = body
if meta, err := release.ParseTorrent(body); err == nil {
if hash == "" {
hash = meta.InfoHash
}
if refused := s.judgeFiles(ctx, want, chosen, hash, meta.Files, force); refused {
out.Refused = true
return out
}
}
}
}
if hash == "" && add.Magnet != "" {
hash = indexer.MagnetHash(add.Magnet)
}
add.SavePath = landingFor(s.Grabber.Landing, hash, chosen.Name)
if err := s.Grabber.Client.AddTorrent(ctx, add); err != nil && !errors.Is(err, downloader.ErrDuplicate) {
if errors.Is(err, downloader.ErrUnconfigured) {
out.Unconfigured, out.Missing = true, MissingClient
} else {
out.Error = indexer.FirstLine(err.Error())
}
return out
}
out.Added, out.SavePath = true, add.SavePath
if s.Memory == nil {
return out
}
var replaced pgtype.UUID
why := ""
if contest != nil && contest.Replaced {
out.Contest = contest
replaced, why = contest.Rival.ID, "Replaced by "+chosen.Name+": "+contest.Reason+"."
}
g, err := s.Memory.Grabbed(ctx, dbq.InsertGrabParams{
WantID: want.ID, InfoHash: hash, Name: chosen.Name, Key: Key(*chosen), Indexer: chosen.Indexer,
Source: chosen.Source.Source, Client: s.Grabber.Code, SavePath: add.SavePath,
Quality: string(chosen.Tier), Origin: origin.By, Forced: force, GrabbedBy: origin.User,
}, replaced, why)
if err != nil {
out.Error = "The download started, but could not be recorded: " + indexer.FirstLine(err.Error())
return out
}
out.Grab = g
// Removed only after the new download was added and recorded, so a
// failure never leaves the want with nothing running.
if replaced.Valid && contest.Rival.InfoHash != "" {
if err := s.Grabber.Client.Delete(ctx, true, contest.Rival.InfoHash); err != nil {
contest.Reason += ". The download it replaced could not be removed from the client: " + indexer.FirstLine(err.Error())
}
}
return out
}
// judgeFiles judges a chosen release again on its .torrent's file list, and
// sets it aside when the files say it is not the album, so no later search
// offers it again. force still takes it.
func (s *Service) judgeFiles(ctx context.Context, want Want, chosen *Candidate, hash string, files []release.File, force bool) bool {
chosen.Files = files
if chosen.FilesCount == 0 {
chosen.FilesCount = len(files)
}
veto, reason, _ := release.JudgeFiles(want.Want, files)
if veto == "" || force {
return false
}
set(chosen, Rejected, Hard, veto, reason)
if s.Memory != nil && hash != "" {
_ = s.Memory.Block(ctx, dbq.BlockReleaseParams{
InfoHash: hash, Name: chosen.Name, Reason: BlockFiles, Detail: reason, WantID: want.ID,
})
}
return true
}
var wantLocks sync.Map // want id → *sync.Mutex
// lockWant settles two grabs for one want one after the other. The
// one-live-grab index is the backstop for anything outside this process.
func lockWant(id pgtype.UUID) func() {
if !id.Valid {
return func() {}
}
m, _ := wantLocks.LoadOrStore(id.Bytes, &sync.Mutex{})
mu := m.(*sync.Mutex)
mu.Lock()
return mu.Unlock
}
// contest settles a grab against the download running for its want: a
// person's grab beats the schedule's; between two of a kind the better
// candidate wins; a tie, a download nearly done, or one that cannot be
// removed stays. Nil when nothing is running.
func (s *Service) contest(ctx context.Context, want Want, chosen *Candidate, all []Candidate, origin Origin) *Contest {
if s.Memory == nil || !want.ID.Valid {
return nil
}
running, ok, err := s.Memory.LiveGrab(ctx, want.ID)
if err != nil || !ok {
return nil
}
c := &Contest{Rival: running}
if running.InfoHash != "" && running.InfoHash == chosen.InfoHash {
c.Kept, c.Reason = true, "That release is already downloading for this album."
return c
}
switch {
case origin.By == OriginOperator && running.Origin == OriginSchedule:
c.Reason = "you chose it over the one the schedule started"
case origin.By == OriginSchedule && running.Origin == OriginOperator:
c.Kept, c.Reason = true, "A download you started for this album is already running."
return c
default:
var theirs *Candidate
for i := range all {
if (running.InfoHash != "" && all[i].InfoHash == running.InfoHash) || Key(all[i]) == running.Key {
theirs = &all[i]
break
}
}
if theirs == nil || !better(*chosen, *theirs) {
c.Kept, c.Reason = true, "A download for this album is already running, and this release is no better."
return c
}
c.Reason = "it ranks above the one that was running"
}
if running.InfoHash == "" {
c.Kept, c.Reason = true, "A download for this album is already running, and it cannot be identified to replace it."
return c
}
if ts, err := s.Grabber.Client.Torrents(ctx, running.InfoHash); err == nil && len(ts) > 0 && ts[0].Progress >= nearlyDone {
c.Kept, c.Reason = true, "A download for this album is already running and nearly done."
return c
}
c.Replaced = true
return c
}
// landingFor is the folder a download lands in: one per torrent, named by
// its infohash, so two grabs never unpack into the same place.
func landingFor(root, hash, name string) string {
dir := hash
if dir == "" {
dir = slug(name)
}
if dir == "" {
dir = "unnamed"
}
return path.Join(root, dir)
}
// slug keeps a folder name to characters every filesystem in the chain
// takes.
func slug(s string) string {
var b strings.Builder
for _, r := range strings.ToLower(strings.TrimSpace(s)) {
if b.Len() >= 80 {
break
}
switch {
case r >= 'a' && r <= 'z', r >= '0' && r <= '9':
b.WriteRune(r)
case r == '-', r == '_', r == ' ', r == '.':
b.WriteRune('-')
}
}
return strings.Trim(b.String(), "-")
}
+168
View File
@@ -0,0 +1,168 @@
package acquire
import (
"context"
"encoding/json"
"errors"
"fmt"
"github.com/jackc/pgx/v5"
"github.com/jackc/pgx/v5/pgtype"
"github.com/jackc/pgx/v5/pgxpool"
"git.fabledsword.com/bvandeusen/minstrel/internal/db/dbq"
"git.fabledsword.com/bvandeusen/minstrel/internal/indexer"
"git.fabledsword.com/bvandeusen/minstrel/internal/release"
)
// Block reasons, matching release_blocks_reason_check.
const (
BlockOperator = "operator"
BlockFiles = "files"
)
// Grab origins, matching grabs_origin_check.
const (
OriginSchedule = "schedule"
OriginOperator = "operator"
)
// Memory is what searches and grabs leave behind: every candidate of a
// want's last search, the grabs, and the releases set aside.
type Memory struct {
pool *pgxpool.Pool
}
// NewMemory returns a Memory over pool.
func NewMemory(pool *pgxpool.Pool) *Memory { return &Memory{pool: pool} }
// Keep replaces a want's candidates with this search's, in order.
func (m *Memory) Keep(ctx context.Context, wantID pgtype.UUID, cands []Candidate) error {
return pgx.BeginFunc(ctx, m.pool, func(tx pgx.Tx) error {
q := dbq.New(tx)
if err := q.DeleteWantCandidates(ctx, wantID); err != nil {
return err
}
for i, c := range cands {
reasons, err := json.Marshal(c.Reasons)
if err != nil {
return err
}
if c.Reasons == nil {
reasons = []byte("[]")
}
folded := c.FoldedFrom
if folded == nil {
folded = []string{}
}
if err := q.InsertWantCandidate(ctx, dbq.InsertWantCandidateParams{
WantID: wantID, Key: Key(c), Name: c.Name, InfoHash: c.InfoHash, Indexer: c.Indexer,
Source: c.Source.Source, SizeBytes: c.SizeBytes, Seeders: int32(c.Seeders),
FilesCount: int32(c.FilesCount), Format: release.Describe(c.Release), Quality: string(c.Tier),
Year: int32(c.Year), Verdict: string(c.Verdict), Reason: c.Reason, Reasons: reasons,
FoldedFrom: folded, Rank: int32(i),
}); err != nil {
return err
}
}
return nil
})
}
// blocked is every release set aside, by infohash, with the sentence a
// candidate shows. A failed read blocks nothing: offering a release set
// aside costs one grab a person can undo, while blocking everything on a
// database blip would hide every result.
func (m *Memory) blocked(ctx context.Context) map[string]string {
if m == nil {
return nil
}
rows, err := dbq.New(m.pool).ListReleaseBlocks(ctx)
if err != nil {
return nil
}
out := make(map[string]string, len(rows))
for _, r := range rows {
out[indexer.NormalizeHash(r.InfoHash)] = BlockSentence(r.Reason, r.Detail)
}
return out
}
// BlockSentence is what a blocked candidate says.
func BlockSentence(reason, detail string) string {
if reason == BlockFiles {
if detail != "" {
return "Set aside for what its files hold: " + detail
}
return "Set aside for what its files hold."
}
return "You set this release aside."
}
// Block sets a release aside and marks every kept candidate of it blocked.
func (m *Memory) Block(ctx context.Context, p dbq.BlockReleaseParams) error {
p.InfoHash = indexer.NormalizeHash(p.InfoHash)
if p.InfoHash == "" {
return errors.New("a release without an infohash cannot be set aside")
}
return pgx.BeginFunc(ctx, m.pool, func(tx pgx.Tx) error {
q := dbq.New(tx)
if err := q.BlockRelease(ctx, p); err != nil {
return err
}
return q.MarkCandidatesBlocked(ctx, dbq.MarkCandidatesBlockedParams{
InfoHash: p.InfoHash, Reason: BlockSentence(p.Reason, p.Detail),
})
})
}
// Unblock lets a release be taken again. Its kept candidates wait for a
// person until the next search judges them afresh.
func (m *Memory) Unblock(ctx context.Context, hash string) (bool, error) {
hash = indexer.NormalizeHash(hash)
var n int64
err := pgx.BeginFunc(ctx, m.pool, func(tx pgx.Tx) error {
q := dbq.New(tx)
var err error
if n, err = q.UnblockRelease(ctx, hash); err != nil {
return err
}
return q.MarkCandidatesUnblocked(ctx, dbq.MarkCandidatesUnblockedParams{
InfoHash: hash, Reason: "Allowed again. The next search judges it afresh.",
})
})
return n > 0, err
}
// LiveGrab is the download running for a want, if there is one.
func (m *Memory) LiveGrab(ctx context.Context, wantID pgtype.UUID) (dbq.Grab, bool, error) {
g, err := dbq.New(m.pool).GetLiveGrab(ctx, wantID)
if errors.Is(err, pgx.ErrNoRows) {
return g, false, nil
}
if err != nil {
return g, false, err
}
return g, true, nil
}
// Grabbed records a download started, ending the grab it replaced first so
// the one-live-grab index holds.
func (m *Memory) Grabbed(ctx context.Context, p dbq.InsertGrabParams, replaced pgtype.UUID, why string) (dbq.Grab, error) {
var g dbq.Grab
err := pgx.BeginFunc(ctx, m.pool, func(tx pgx.Tx) error {
q := dbq.New(tx)
if replaced.Valid {
if err := q.EndGrab(ctx, dbq.EndGrabParams{ID: replaced, EndReason: "superseded", EndDetail: why}); err != nil {
return err
}
}
var err error
g, err = q.InsertGrab(ctx, p)
return err
})
if err != nil {
return g, fmt.Errorf("record grab: %w", err)
}
return g, nil
}
+145
View File
@@ -0,0 +1,145 @@
package acquire
import (
"context"
"sync"
"time"
"git.fabledsword.com/bvandeusen/minstrel/internal/indexer"
)
// SearchPace is the one Pacer every search shares, ported from Fabled
// Librarian (#4448). Process-wide because a Fanout is built fresh for each
// search; a Pacer per Fanout would start empty and pace nothing.
var SearchPace = &Pacer{}
// holdWithoutRetryAfter is how long a source that answered 429 without saying
// for how long is left alone: one schedule tick.
const holdWithoutRetryAfter = 10 * time.Minute
// Pacer spaces the searches sent to each source by its searches_per_hour: a
// spacing, not a burst bucket, because Prowlarr answers 429 to a few requests
// in quick succession (FL #4304). Every search spends, the schedule's and a
// person's alike; only the schedule waits.
type Pacer struct {
// Now and Sleep are the clock, replaceable in tests.
Now func() time.Time
Sleep func(context.Context, time.Duration) error
mu sync.Mutex
next map[string]time.Time
}
func (p *Pacer) now() time.Time {
if p.Now != nil {
return p.Now()
}
return time.Now()
}
func (p *Pacer) sleep(ctx context.Context, d time.Duration) error {
if p.Sleep != nil {
return p.Sleep(ctx, d)
}
t := time.NewTimer(d)
defer t.Stop()
select {
case <-t.C:
return nil
case <-ctx.Done():
return ctx.Err()
}
}
// gap is the spacing a source's number asks for; zero is unpaced.
func gap(perHour int) time.Duration {
if perHour <= 0 {
return 0
}
return time.Hour / time.Duration(perHour)
}
// readyAt is when every source may next be asked.
func (p *Pacer) readyAt(sources []Source) time.Time {
p.mu.Lock()
defer p.mu.Unlock()
var at time.Time
for _, s := range sources {
if s.Search == nil {
continue
}
if n := p.next[s.Code]; n.After(at) {
at = n
}
}
return at
}
// Wait holds until every source may be asked, and answers false, without
// waiting, when that is after the deadline or ctx ends first. All of them,
// not any: a search while one is held back would be narrowed, and the want
// recorded as looked for.
func (p *Pacer) Wait(ctx context.Context, sources []Source, deadline time.Time) bool {
if p == nil {
return true
}
at := p.readyAt(sources)
if !deadline.IsZero() && at.After(deadline) {
return false
}
if d := at.Sub(p.now()); d > 0 {
return p.sleep(ctx, d) == nil
}
return ctx.Err() == nil
}
// spend takes one search's slot from a source.
func (p *Pacer) spend(s Source) {
g := gap(s.PerHour)
if p == nil || g == 0 {
return
}
p.mu.Lock()
defer p.mu.Unlock()
if p.next == nil {
p.next = map[string]time.Time{}
}
at := p.next[s.Code]
if t := p.now(); at.Before(t) {
at = t
}
p.next[s.Code] = at.Add(g)
}
// hold leaves a source alone after it answered 429, for as long as it asked,
// whatever its searches_per_hour says.
func (p *Pacer) hold(code string, rl *indexer.RateLimited) {
if p == nil {
return
}
d := rl.RetryAfter
if d <= 0 {
d = holdWithoutRetryAfter
}
p.mu.Lock()
defer p.mu.Unlock()
if p.next == nil {
p.next = map[string]time.Time{}
}
if until := p.now().Add(d); until.After(p.next[code]) {
p.next[code] = until
}
}
// Ready holds until every source this service searches may be asked again,
// and answers false when that is past the deadline. The schedule calls it
// before each want; a person's search never does.
func (s *Service) Ready(ctx context.Context, deadline time.Time) bool {
if s == nil {
return ctx.Err() == nil
}
if f, ok := s.Search.(*Fanout); ok && f != nil {
return f.Pace.Wait(ctx, f.Sources, deadline)
}
return ctx.Err() == nil
}
+390
View File
@@ -0,0 +1,390 @@
package acquire
import (
"context"
"errors"
"sort"
"strings"
"github.com/jackc/pgx/v5/pgtype"
"git.fabledsword.com/bvandeusen/minstrel/internal/indexer"
"git.fabledsword.com/bvandeusen/minstrel/internal/release"
)
// One search for one want, and what could be done about each result (M502
// #5521), ported from Fabled Librarian's internal/acquire. internal/release
// says whether a name is the album and what quality it is; this package
// asks in the right order and says why in words a person can act on.
// Verdict is what can be done about one result.
type Verdict string
const (
// Grab: the album, in a quality the profile takes.
Grab Verdict = "grab"
// Review: maybe the album; a person decides.
Review Verdict = "review"
// Refused: the album, in a quality the profile does not take. The only
// verdict that names a setting.
Refused Verdict = "refused"
// Rejected: not the album.
Rejected Verdict = "rejected"
// Blocked: a release set aside, by a person or by its files.
Blocked Verdict = "blocked"
)
// order sorts verdicts: what can be taken first, what was given up on last.
var order = map[Verdict]int{Grab: 0, Review: 1, Refused: 2, Rejected: 3, Blocked: 4}
// Reason kinds: a hard reason is a fact about the bytes, a soft one about the
// moment or a setting.
const (
Hard = "hard"
Soft = "soft"
)
// Reason codes beyond internal/release's vetoes.
const (
CodeBlocked = "blocked"
CodeProfile = "profile"
CodeNoSeeders = "no_seeders"
CodeDoubt = "doubt"
)
// Reason is one thing a verdict rests on.
type Reason struct {
Kind string `json:"kind"`
Code string `json:"code"`
Text string `json:"text"`
}
// Want is the album searched for: its wants row and how the catalogue
// describes it.
type Want struct {
ID pgtype.UUID
release.Want
}
// Candidate is one result with everything decided about it.
type Candidate struct {
release.Release
// Source is the indexer's own result. It never leaves the server: its
// DownloadURL can carry the indexer's key.
Source indexer.Result
InfoHash string
Indexer string
SizeBytes int64
Seeders int
SwarmKnown bool
Category int
FilesCount int
// Files is the torrent's file list once its .torrent has been read.
Files []release.File
FoldedFrom []string
Tier release.Tier
YearAgrees bool
Verdict Verdict
Reason string
Reasons []Reason
}
// Key is how a candidate is named across a search and a grab: its infohash,
// or its name when the source gave none.
func Key(c Candidate) string {
if c.InfoHash != "" {
return c.InfoHash
}
return c.Name
}
// Result is one search.
type Result struct {
Query string
// Unconfigured: no search source is set up.
Unconfigured bool
// Error is a search that failed outright.
Error string
// Narrowed names sources that did not answer on a search that otherwise
// did.
Narrowed []Narrowing
Considered int
Folded int
// Looked is how many .torrent files were read for their file lists.
Looked int
Candidates []Candidate
// KeepError: the candidates could not be stored.
KeepError string
}
// Grabbable is what may be taken, best first.
func (r Result) Grabbable() []Candidate {
var out []Candidate
for _, c := range r.Candidates {
if c.Verdict == Grab {
out = append(out, c)
}
}
return out
}
// ForAPerson says whether anything found is worth a person's look: a
// release that may be the album, whatever its quality.
func (r Result) ForAPerson() bool {
for _, c := range r.Candidates {
if c.Verdict == Grab || c.Verdict == Review || c.Verdict == Refused {
return true
}
}
return false
}
// Service searches and grabs.
type Service struct {
// Search is nil when no source is configured.
Search Searcher
// Grabber is nil when no download client is configured.
Grabber *Downloader
Profile release.Profile
// Memory stores candidates, grabs and blocks. Nil keeps nothing.
Memory *Memory
}
// Query is the text sent to the indexers: the artist and the title, without
// a trailing bracketed part the release names usually leave off.
func Query(w release.Want) string {
title := w.Title
if i := strings.IndexAny(title, "(["); i > 0 {
title = title[:i]
}
return strings.Join(strings.Fields(strings.TrimSpace(w.Artist)+" "+strings.TrimSpace(title)), " ")
}
// For searches for one want and judges everything that comes back. It never
// returns an error: a search that could not be made is a Result saying so.
func (s *Service) For(ctx context.Context, want Want) Result {
r := Result{Query: Query(want.Want)}
if s == nil || s.Search == nil {
r.Unconfigured = true
return r
}
found, err := s.Search.Search(ctx, indexer.Query{Text: r.Query, Categories: indexer.MusicCategories})
// A partial search arrives as results and an error together.
if narrowed, answered := narrowings(err); narrowed != nil && answered {
r.Narrowed, err = narrowed, nil
}
switch {
case errors.Is(err, indexer.ErrUnconfigured):
r.Unconfigured = true
return r
case err != nil:
r.Error = indexer.FirstLine(err.Error())
return r
}
r.Considered = len(found)
cands, folded := fold(found)
r.Folded = folded
blocked := s.Memory.blocked(ctx)
for i := range cands {
decide(&cands[i], want.Want, s.Profile, blocked)
}
sortCandidates(cands)
for _, i := range s.peek(ctx, cands) {
decide(&cands[i], want.Want, s.Profile, blocked)
r.Looked++
}
sortCandidates(cands)
r.Candidates = cands
if s.Memory != nil && want.ID.Valid {
if err := s.Memory.Keep(ctx, want.ID, cands); err != nil {
r.KeepError = indexer.FirstLine(err.Error())
}
}
return r
}
// decide works out one candidate's verdict, in order: given up on, then
// whether it is the album, then whether the profile takes its quality, then
// whether anything leaves it to a person. A quality refusal said about the
// wrong album would be noise, so identity comes first.
func decide(c *Candidate, w release.Want, profile release.Profile, blocked map[string]string) {
c.Verdict, c.Reason, c.Reasons = "", "", nil
if why, ok := blocked[c.InfoHash]; ok && c.InfoHash != "" {
set(c, Blocked, Hard, CodeBlocked, why)
return
}
m := release.Judge(w, c.Release)
c.YearAgrees = m.YearAgrees
if m.Veto != "" {
set(c, Rejected, Hard, m.Veto, m.Reason)
return
}
doubt := m.Doubt
if len(c.Files) > 0 {
if in := release.Inspect(c.Files); in.Format != "" {
c.Format = in.Format
}
veto, reason, fileDoubt := release.JudgeFiles(w, c.Files)
if veto != "" {
set(c, Rejected, Hard, veto, reason)
return
}
if doubt == "" {
doubt = fileDoubt
}
}
c.Tier = release.TierOf(c.Release, c.Category)
if !profile.Allows(c.Tier) {
set(c, Refused, Soft, CodeProfile, "Your quality settings don’t take "+strings.ToLower(release.Label(c.Tier))+".")
return
}
if doubt != "" {
set(c, Review, Soft, CodeDoubt, doubt)
return
}
if c.Seeders == 0 {
set(c, Review, Soft, CodeNoSeeders, "Nobody is sharing it right now.")
return
}
c.Verdict = Grab
c.Reason = "The album, " + strings.ToLower(release.Label(c.Tier)) + "."
if d := release.Describe(c.Release); d != "" {
c.Reason = "The album, " + d + "."
}
}
func set(c *Candidate, v Verdict, kind, code, text string) {
c.Verdict, c.Reason = v, text
c.Reasons = append(c.Reasons, Reason{Kind: kind, Code: code, Text: text})
}
// sortCandidates puts the best first, stably so the same search proposes the
// same grab twice.
func sortCandidates(cands []Candidate) {
sort.SliceStable(cands, func(a, b int) bool { return better(cands[a], cands[b]) })
}
// better says whether x ranks above y: verdict, then quality, then how many
// share it, then whether its year agrees.
func better(x, y Candidate) bool {
if order[x.Verdict] != order[y.Verdict] {
return order[x.Verdict] < order[y.Verdict]
}
if rx, ry := release.Rank(x.Tier), release.Rank(y.Tier); rx != ry {
return rx > ry
}
if x.Seeders != y.Seeders {
return x.Seeders > y.Seeders
}
return x.YearAgrees && !y.YearAgrees
}
// fold merges results that are the same torrent, by infohash. Prowlarr
// aggregates, so one release arrives once per tracker that lists it. A
// result without a hash is never folded: an absent hash is not a shared one.
func fold(found []indexer.Result) ([]Candidate, int) {
out := make([]Candidate, 0, len(found))
at := make(map[string]int, len(found))
folded := 0
for _, f := range found {
if strings.TrimSpace(f.Title) == "" {
continue
}
hash := indexer.NormalizeHash(f.InfoHash)
if hash == "" && f.Magnet != "" {
hash = indexer.MagnetHash(f.Magnet)
}
if hash != "" {
if i, seen := at[hash]; seen {
folded++
if f.Indexer != "" {
out[i].FoldedFrom = append(out[i].FoldedFrom, f.Indexer)
}
// A measured swarm beats a reported one; among like, the
// highest count.
switch {
case f.SwarmKnown && !out[i].SwarmKnown:
out[i].Seeders, out[i].SwarmKnown = f.Seeders, true
case f.SwarmKnown == out[i].SwarmKnown && f.Seeders > out[i].Seeders:
out[i].Seeders = f.Seeders
}
if f.FilesCount > 0 && out[i].FilesCount == 0 {
out[i].FilesCount = f.FilesCount
}
continue
}
at[hash] = len(out)
}
out = append(out, Candidate{
Release: release.Parse(f.Title), Source: f, InfoHash: hash, Indexer: f.Indexer,
SizeBytes: f.SizeBytes, Seeders: f.Seeders, SwarmKnown: f.SwarmKnown,
Category: f.Category, FilesCount: f.FilesCount,
})
}
return out, folded
}
// maxPeeks is how many .torrent files one search reads. Few: each goes
// through Prowlarr to a tracker, and Prowlarr answers 429 to a burst.
const maxPeeks = 3
// peek reads the .torrent of candidates that are the album but whose name
// states no format, so their files can say what they are; it returns which
// now carry a file list. A failed fetch leaves the candidate as it was.
func (s *Service) peek(ctx context.Context, cands []Candidate) []int {
f, ok := s.Search.(Fetcher)
if !ok {
return nil
}
var looked []int
for i := range cands {
if len(looked) >= maxPeeks || ctx.Err() != nil {
break
}
c := &cands[i]
if c.Tier != release.TierUnknown || (c.Verdict != Grab && c.Verdict != Refused) ||
c.Files != nil || c.Source.DownloadURL == "" || c.Source.Magnet != "" {
continue
}
body, err := f.Download(ctx, c.Source)
if err != nil {
continue
}
meta, err := release.ParseTorrent(body)
if err != nil {
continue
}
c.Files = meta.Files
if c.InfoHash == "" {
c.InfoHash = meta.InfoHash
}
if c.FilesCount == 0 {
c.FilesCount = len(meta.Files)
}
looked = append(looked, i)
}
return looked
}
// pick finds a candidate by key: the infohash first, the exact name for a
// result that has none.
func pick(cands []Candidate, key string) *Candidate {
key = strings.TrimSpace(key)
if key == "" {
return nil
}
hash := indexer.NormalizeHash(key)
for i := range cands {
if cands[i].InfoHash != "" && cands[i].InfoHash == hash {
return &cands[i]
}
}
for i := range cands {
if cands[i].InfoHash == "" && cands[i].Name == key {
return &cands[i]
}
}
return nil
}
+241
View File
@@ -0,0 +1,241 @@
package acquire
import (
"context"
"errors"
"strings"
"testing"
"time"
"git.fabledsword.com/bvandeusen/minstrel/internal/downloader"
"git.fabledsword.com/bvandeusen/minstrel/internal/indexer"
"git.fabledsword.com/bvandeusen/minstrel/internal/release"
)
type fakeSearch struct {
results []indexer.Result
err error
asked []indexer.Query
}
func (f *fakeSearch) Search(_ context.Context, q indexer.Query) ([]indexer.Result, error) {
f.asked = append(f.asked, q)
return f.results, f.err
}
var ram = Want{Want: release.Want{
Artist: "Daft Punk", Title: "Random Access Memories", Year: 2013, PrimaryType: "Album",
MinTracks: 13, MaxTracks: 15,
}}
var anyQuality = release.NewProfile([]string{
"lossless_hires", "lossless", "lossy_high", "lossy_standard", "lossy_low", "lossy_poor", "lossy_trash",
})
func result(title, hash string, seeders int) indexer.Result {
return indexer.Result{Title: title, InfoHash: hash, Seeders: seeders, Indexer: "Knaben", Source: "prowlarr",
Magnet: "magnet:?xt=urn:btih:" + hash}
}
func TestFor_JudgesSortsAndFolds(t *testing.T) {
search := &fakeSearch{results: []indexer.Result{
result("Daft Punk - Random Access Memories (2013) [MP3 320]", "aaaa", 40),
result("Daft Punk Random Access Memories (Drumless Edition 2023) [Flac 24-88]", "bbbb", 90),
result("Daft Punk - Random Access Memories (2013) [FLAC]", "cccc", 12),
result("Daft Punk - Random Access Memories (2013) [FLAC]", "CCCC", 30), // the same torrent from another tracker
result("Daft Punk - Discovery (2001) [FLAC]", "dddd", 50),
}}
s := &Service{Search: search, Profile: anyQuality}
r := s.For(context.Background(), ram)
if len(search.asked) != 1 || search.asked[0].Text != "Daft Punk Random Access Memories" {
t.Fatalf("asked %+v", search.asked)
}
if r.Considered != 5 || r.Folded != 1 || len(r.Candidates) != 4 {
t.Fatalf("considered %d folded %d candidates %d", r.Considered, r.Folded, len(r.Candidates))
}
// Lossless before high-quality lossy among what can be taken; the
// folded copy keeps the higher seeder count.
first, second := r.Candidates[0], r.Candidates[1]
if first.InfoHash != "cccc" || first.Verdict != Grab || first.Seeders != 30 || first.Tier != release.TierLossless {
t.Errorf("first = %s %s %d %s", first.Name, first.Verdict, first.Seeders, first.Tier)
}
if second.InfoHash != "aaaa" || second.Verdict != Grab {
t.Errorf("second = %s %s", second.Name, second.Verdict)
}
for _, c := range r.Candidates[2:] {
if c.Verdict != Rejected || len(c.Reasons) != 1 || c.Reasons[0].Kind != Hard {
t.Errorf("%s: %s %+v", c.Name, c.Verdict, c.Reasons)
}
}
if len(r.Grabbable()) != 2 || !r.ForAPerson() {
t.Errorf("grabbable %d", len(r.Grabbable()))
}
}
func TestFor_ProfileSeedersAndDoubt(t *testing.T) {
search := &fakeSearch{results: []indexer.Result{
result("Daft Punk - Random Access Memories [MP3 128]", "aaaa", 5),
result("Daft Punk - Random Access Memories [FLAC]", "bbbb", 0),
result("Daft Punk - Random Access Memories", "cccc", 9),
}}
s := &Service{Search: search, Profile: release.NewProfile([]string{"lossless", "lossy_high"})}
r := s.For(context.Background(), ram)
verdicts := map[string]Verdict{}
for _, c := range r.Candidates {
verdicts[c.InfoHash] = c.Verdict
}
if verdicts["aaaa"] != Refused || verdicts["bbbb"] != Review || verdicts["cccc"] != Refused {
t.Errorf("verdicts = %v", verdicts)
}
if len(r.Grabbable()) != 0 || !r.ForAPerson() {
t.Error("nothing grabbable, but something for a person")
}
}
func TestFor_BlockedReleasesAreSetAside(t *testing.T) {
c := Candidate{Release: release.Parse("Daft Punk - Random Access Memories [FLAC]"), InfoHash: "aaaa", Seeders: 3}
decide(&c, ram.Want, anyQuality, map[string]string{"aaaa": "You set this release aside."})
if c.Verdict != Blocked || c.Reason != "You set this release aside." {
t.Errorf("verdict %s %q", c.Verdict, c.Reason)
}
}
func TestFor_UnconfiguredAndFailedSearches(t *testing.T) {
if r := (&Service{}).For(context.Background(), ram); !r.Unconfigured {
t.Error("no source is unconfigured")
}
r := (&Service{Search: &fakeSearch{err: indexer.ErrUnconfigured}}).For(context.Background(), ram)
if !r.Unconfigured {
t.Error("ErrUnconfigured is unconfigured")
}
r = (&Service{Search: &fakeSearch{err: errors.New("dial tcp: refused")}}).For(context.Background(), ram)
if r.Error == "" {
t.Error("a failed search says so")
}
}
func TestFanout_ANarrowedSearchKeepsWhatAnswered(t *testing.T) {
up := &fakeSearch{results: []indexer.Result{result("Daft Punk - Random Access Memories [FLAC]", "aaaa", 3)}}
down := &fakeSearch{err: indexer.ErrUnreachable}
f := &Fanout{Sources: []Source{{Code: "prowlarr", Label: "Prowlarr", Search: up}, {Code: "torznab", Label: "Feed", Search: down}}}
r := (&Service{Search: f, Profile: anyQuality}).For(context.Background(), ram)
if r.Error != "" || len(r.Candidates) != 1 || len(r.Narrowed) != 1 || r.Narrowed[0].Source != "torznab" {
t.Errorf("narrowed search: error %q, %d candidates, narrowed %+v", r.Error, len(r.Candidates), r.Narrowed)
}
f.Sources = f.Sources[1:]
if r := (&Service{Search: f}).For(context.Background(), ram); r.Error == "" {
t.Error("every source down is a failed search")
}
}
func TestPacer_SpacesSearchesAndHoldsAfterA429(t *testing.T) {
now := time.Unix(0, 0)
var slept time.Duration
p := &Pacer{Now: func() time.Time { return now }, Sleep: func(_ context.Context, d time.Duration) error { slept += d; return nil }}
src := []Source{{Code: "prowlarr", PerHour: 120, Search: &fakeSearch{}}}
p.spend(src[0])
if !p.Wait(context.Background(), src, now.Add(time.Hour)) || slept != 30*time.Second {
t.Errorf("slept %s, want 30s", slept)
}
if p.Wait(context.Background(), src, now.Add(time.Second)) {
t.Error("waited past the deadline")
}
p.hold("prowlarr", &indexer.RateLimited{RetryAfter: 5 * time.Minute})
if at := p.readyAt(src); at.Sub(now) != 5*time.Minute {
t.Errorf("held until %s", at.Sub(now))
}
}
type fakeClient struct {
downloader.Driver
added []downloader.Add
addErr error
deleted []string
progress float64
}
func (c *fakeClient) AddTorrent(_ context.Context, a downloader.Add) error {
c.added = append(c.added, a)
return c.addErr
}
func (c *fakeClient) Torrents(_ context.Context, hashes ...string) ([]downloader.Torrent, error) {
return []downloader.Torrent{{Hash: hashes[0], Progress: c.progress}}, nil
}
func (c *fakeClient) Delete(_ context.Context, _ bool, hashes ...string) error {
c.deleted = append(c.deleted, hashes...)
return nil
}
func TestGrab_AddsAMagnetToItsOwnFolder(t *testing.T) {
client := &fakeClient{}
s := &Service{Search: &fakeSearch{results: []indexer.Result{
result("Daft Punk - Random Access Memories (2013) [FLAC]", "ABCD", 8),
}}, Profile: anyQuality, Grabber: &Downloader{Client: client, Code: "qbittorrent", Landing: "/downloads"}}
r := s.For(context.Background(), ram)
took := s.Grab(context.Background(), ram, &r.Candidates[0], r.Candidates, false, Origin{By: OriginSchedule})
if !took.Added || took.SavePath != "/downloads/abcd" {
t.Fatalf("took %+v", took)
}
if len(client.added) != 1 || !strings.Contains(strings.ToLower(client.added[0].Magnet), "abcd") || client.added[0].Tags[0] != Tag {
t.Errorf("added %+v", client.added)
}
}
func TestGrab_RefusesWhatTheVerdictWouldNotTake(t *testing.T) {
client := &fakeClient{}
s := &Service{Grabber: &Downloader{Client: client, Landing: "/downloads"}}
review := Candidate{Verdict: Review, InfoHash: "aaaa", Source: indexer.Result{Magnet: "magnet:?xt=urn:btih:aaaa"}}
if took := s.Grab(context.Background(), ram, &review, nil, false, Origin{By: OriginSchedule}); !took.Refused {
t.Error("a review was taken unforced")
}
if took := s.Grab(context.Background(), ram, &review, nil, true, Origin{By: OriginOperator}); !took.Added {
t.Errorf("a forced review was not taken: %+v", took)
}
blocked := Candidate{Verdict: Blocked, InfoHash: "bbbb"}
if took := s.Grab(context.Background(), ram, &blocked, nil, true, Origin{By: OriginOperator}); !took.Refused {
t.Error("forcing took a release set aside")
}
if took := (&Service{}).Grab(context.Background(), ram, &review, nil, true, Origin{}); !took.Unconfigured || took.Missing != MissingClient {
t.Errorf("no client: %+v", took)
}
}
func TestGrab_AClientThatAlreadyHasItCounts(t *testing.T) {
client := &fakeClient{addErr: downloader.ErrDuplicate}
s := &Service{Grabber: &Downloader{Client: client, Landing: "/downloads"}}
c := Candidate{Verdict: Grab, InfoHash: "aaaa", Source: indexer.Result{Magnet: "magnet:?xt=urn:btih:aaaa"}}
if took := s.Grab(context.Background(), ram, &c, nil, false, Origin{By: OriginSchedule}); !took.Added {
t.Errorf("a duplicate: %+v", took)
}
}
func TestBetter_QualityThenSeedersThenYear(t *testing.T) {
flac := Candidate{Verdict: Grab, Tier: release.TierLossless, Seeders: 2}
mp3 := Candidate{Verdict: Grab, Tier: release.TierLossyHigh, Seeders: 90}
if !better(flac, mp3) || better(mp3, flac) {
t.Error("quality first among grabs")
}
busy := Candidate{Verdict: Grab, Tier: release.TierLossless, Seeders: 20}
if !better(busy, flac) {
t.Error("then seeders")
}
dated := Candidate{Verdict: Grab, Tier: release.TierLossless, Seeders: 2, YearAgrees: true}
if !better(dated, flac) {
t.Error("then the year")
}
if better(Candidate{Verdict: Review}, flac) || !better(flac, Candidate{Verdict: Review}) {
t.Error("a grab ranks above a review")
}
}
func TestLandingFor_OneFolderPerTorrent(t *testing.T) {
if got := landingFor("/downloads", "abcd", "x"); got != "/downloads/abcd" {
t.Error(got)
}
if got := landingFor("/downloads", "", "Daft Punk: R.A.M. [FLAC]"); got != "/downloads/daft-punk-r-a-m--flac" {
t.Error(got)
}
}
+299
View File
@@ -0,0 +1,299 @@
package api
import (
"context"
"encoding/json"
"errors"
"net/http"
"time"
"github.com/go-chi/chi/v5"
"github.com/jackc/pgx/v5"
"git.fabledsword.com/bvandeusen/minstrel/internal/acquire"
"git.fabledsword.com/bvandeusen/minstrel/internal/apierror"
"git.fabledsword.com/bvandeusen/minstrel/internal/db/dbq"
"git.fabledsword.com/bvandeusen/minstrel/internal/release"
"git.fabledsword.com/bvandeusen/minstrel/internal/wanted"
)
// Searching for wants and grabbing releases (M502 #5521): the schedule's
// settings, each want's candidates, and a person's Search, Grab and "never
// this one".
// searchTimeout bounds a search a person waits on: a Prowlarr fanning out to
// many trackers can take most of a minute.
const searchTimeout = 2 * time.Minute
type searchSettingsView struct {
AutoGrab bool `json:"auto_grab"`
SearchIntervalHours int32 `json:"search_interval_hours"`
AllowedQualities []string `json:"allowed_qualities"`
Tiers []release.TierInfo `json:"tiers"`
LastPassAt *string `json:"last_pass_at"`
LastPass json.RawMessage `json:"last_pass"`
}
func searchSettings(s dbq.AcquisitionSetting) searchSettingsView {
v := searchSettingsView{
AutoGrab: s.AutoGrab, SearchIntervalHours: s.SearchIntervalHours,
AllowedQualities: s.AllowedQualities, Tiers: release.Tiers, LastPass: s.LastPass,
}
if v.AllowedQualities == nil {
v.AllowedQualities = []string{}
}
if len(v.LastPass) == 0 {
v.LastPass = json.RawMessage("{}")
}
if s.LastPassAt.Valid {
at := formatTimestamp(s.LastPassAt)
v.LastPassAt = &at
}
return v
}
// handleGetSearchSettings implements GET /api/admin/acquisition/settings.
func (h *handlers) handleGetSearchSettings(w http.ResponseWriter, r *http.Request) {
s, err := dbq.New(h.pool).GetAcquisitionSettings(r.Context())
if err != nil {
writeErrWithLog(w, h.logger, "admin search settings: get", apierror.Internal(err))
return
}
writeJSON(w, http.StatusOK, searchSettings(s))
}
type putSearchSettingsBody struct {
AutoGrab bool `json:"auto_grab"`
SearchIntervalHours int32 `json:"search_interval_hours"`
AllowedQualities []string `json:"allowed_qualities"`
}
// handlePutSearchSettings implements PUT /api/admin/acquisition/settings.
func (h *handlers) handlePutSearchSettings(w http.ResponseWriter, r *http.Request) {
var body putSearchSettingsBody
if !decodeBody(w, r, &body) {
return
}
if body.SearchIntervalHours < 0 || body.SearchIntervalHours > 720 {
writeErr(w, apierror.BadRequest("bad_interval", "search_interval_hours must be between 0 and 720"))
return
}
if len(body.AllowedQualities) == 0 {
writeErr(w, apierror.BadRequest("bad_quality", "allow at least one quality"))
return
}
for _, t := range body.AllowedQualities {
if !release.KnownTier(t) {
writeErr(w, apierror.BadRequest("bad_quality", "unknown quality "+t))
return
}
}
s, err := dbq.New(h.pool).UpdateAcquisitionSettings(r.Context(), dbq.UpdateAcquisitionSettingsParams{
AutoGrab: body.AutoGrab, SearchIntervalHours: body.SearchIntervalHours, AllowedQualities: body.AllowedQualities,
})
if err != nil {
writeErrWithLog(w, h.logger, "admin search settings: put", apierror.Internal(err))
return
}
writeJSON(w, http.StatusOK, searchSettings(s))
}
// candidateView is one release a want's last search found.
type candidateView struct {
Key string `json:"key"`
Name string `json:"name"`
InfoHash string `json:"info_hash"`
Indexer string `json:"indexer"`
SizeBytes int64 `json:"size_bytes"`
Seeders int32 `json:"seeders"`
FilesCount int32 `json:"files_count"`
Format string `json:"format"`
Quality string `json:"quality"`
Year int32 `json:"year"`
Verdict string `json:"verdict"`
Reason string `json:"reason"`
Reasons json.RawMessage `json:"reasons"`
FoldedFrom []string `json:"folded_from"`
SearchedAt string `json:"searched_at"`
Downloading bool `json:"downloading"`
}
// handleListCandidates implements GET /api/admin/wanted/{id}/candidates.
func (h *handlers) handleListCandidates(w http.ResponseWriter, r *http.Request) {
id, ok := requireURLUUID(w, r, "id")
if !ok {
return
}
q := dbq.New(h.pool)
rows, err := q.ListWantCandidates(r.Context(), id)
if err != nil {
writeErrWithLog(w, h.logger, "admin wanted: candidates", apierror.Internal(err))
return
}
live, err := q.GetLiveGrab(r.Context(), id)
if err != nil && !errors.Is(err, pgx.ErrNoRows) {
writeErrWithLog(w, h.logger, "admin wanted: live grab", apierror.Internal(err))
return
}
out := make([]candidateView, 0, len(rows))
for _, c := range rows {
v := candidateView{
Key: c.Key, Name: c.Name, InfoHash: c.InfoHash, Indexer: c.Indexer, SizeBytes: c.SizeBytes,
Seeders: c.Seeders, FilesCount: c.FilesCount, Format: c.Format, Quality: c.Quality, Year: c.Year,
Verdict: c.Verdict, Reason: c.Reason, Reasons: c.Reasons, FoldedFrom: c.FoldedFrom,
SearchedAt: formatTimestamp(c.SearchedAt),
Downloading: live.ID.Valid && live.Key == c.Key,
}
if v.FoldedFrom == nil {
v.FoldedFrom = []string{}
}
if len(v.Reasons) == 0 {
v.Reasons = json.RawMessage("[]")
}
out = append(out, v)
}
writeJSON(w, http.StatusOK, map[string]any{"items": out})
}
// handleSearchWant implements POST /api/admin/wanted/{id}/search: search
// now, as the schedule would, taking the best release when auto-grab is on.
func (h *handlers) handleSearchWant(w http.ResponseWriter, r *http.Request) {
id, ok := requireURLUUID(w, r, "id")
if !ok {
return
}
ctx, cancel := context.WithTimeout(r.Context(), searchTimeout)
defer cancel()
res, err := h.searcher.SearchNow(ctx, id)
if h.writeSearchErr(w, err) {
return
}
writeJSON(w, http.StatusOK, res)
}
type grabBody struct {
Key string `json:"key"`
Force bool `json:"force"`
}
// grabView is what came of a person's grab.
type grabView struct {
Added bool `json:"added"`
Refused bool `json:"refused"`
Gone bool `json:"gone"`
Unconfigured bool `json:"unconfigured"`
Missing string `json:"missing,omitempty"`
Kept bool `json:"kept"`
Replaced bool `json:"replaced"`
Verdict string `json:"verdict,omitempty"`
Reason string `json:"reason,omitempty"`
Error string `json:"error,omitempty"`
}
// handleGrabCandidate implements POST /api/admin/wanted/{id}/grab: take the
// release named by key. The search runs again first, so the indexer's link
// never leaves the server; force takes what the verdict would not.
func (h *handlers) handleGrabCandidate(w http.ResponseWriter, r *http.Request) {
admin, ok := requireUser(w, r)
if !ok {
return
}
id, ok := requireURLUUID(w, r, "id")
if !ok {
return
}
var body grabBody
if !decodeBody(w, r, &body) {
return
}
ctx, cancel := context.WithTimeout(r.Context(), searchTimeout)
defer cancel()
took, err := h.searcher.Take(ctx, id, body.Key, body.Force, admin.ID)
if h.writeSearchErr(w, err) {
return
}
v := grabView{
Added: took.Added, Refused: took.Refused, Gone: took.Gone, Unconfigured: took.Unconfigured,
Missing: took.Missing, Error: took.Error,
}
if took.Chosen != nil {
v.Verdict, v.Reason = string(took.Chosen.Verdict), took.Chosen.Reason
}
if took.Contest != nil {
v.Kept, v.Replaced = took.Contest.Kept, took.Contest.Replaced
if took.Contest.Kept {
v.Reason = took.Contest.Reason
}
}
writeJSON(w, http.StatusOK, v)
}
type blockBody struct {
Key string `json:"key"`
}
// handleBlockCandidate implements POST /api/admin/wanted/{id}/block: never
// take this release again, for any want.
func (h *handlers) handleBlockCandidate(w http.ResponseWriter, r *http.Request) {
admin, ok := requireUser(w, r)
if !ok {
return
}
id, ok := requireURLUUID(w, r, "id")
if !ok {
return
}
var body blockBody
if !decodeBody(w, r, &body) {
return
}
c, err := dbq.New(h.pool).GetWantCandidate(r.Context(), dbq.GetWantCandidateParams{WantID: id, Key: body.Key})
if errors.Is(err, pgx.ErrNoRows) {
writeErr(w, apierror.NotFound("candidate"))
return
}
if err != nil {
writeErrWithLog(w, h.logger, "admin wanted: block", apierror.Internal(err))
return
}
if c.InfoHash == "" {
writeErr(w, apierror.BadRequest("no_infohash", "this release has no infohash to set aside by"))
return
}
if err := acquire.NewMemory(h.pool).Block(r.Context(), dbq.BlockReleaseParams{
InfoHash: c.InfoHash, Name: c.Name, Reason: acquire.BlockOperator, WantID: id, BlockedBy: admin.ID,
}); err != nil {
writeErrWithLog(w, h.logger, "admin wanted: block", apierror.Internal(err))
return
}
w.WriteHeader(http.StatusNoContent)
}
// handleUnblockRelease implements DELETE /api/admin/release-blocks/{hash}.
func (h *handlers) handleUnblockRelease(w http.ResponseWriter, r *http.Request) {
removed, err := acquire.NewMemory(h.pool).Unblock(r.Context(), chi.URLParam(r, "hash"))
if err != nil {
writeErrWithLog(w, h.logger, "admin wanted: unblock", apierror.Internal(err))
return
}
if !removed {
writeErr(w, apierror.NotFound("block"))
return
}
w.WriteHeader(http.StatusNoContent)
}
// writeSearchErr answers a Searcher error, and says whether there was one.
func (h *handlers) writeSearchErr(w http.ResponseWriter, err error) bool {
switch {
case err == nil:
return false
case errors.Is(err, pgx.ErrNoRows):
writeErr(w, apierror.NotFound("want"))
case errors.Is(err, wanted.ErrWantHeld):
writeErr(w, apierror.Conflict("want_held", "the library already holds this album"))
default:
writeErrWithLog(w, h.logger, "admin wanted: search", apierror.Internal(err))
}
return true
}
+35
View File
@@ -35,6 +35,25 @@ type wantView struct {
WantedAt string `json:"wanted_at"`
HeldAt *string `json:"held_at"`
ImageURL string `json:"image_url"`
// Search is the schedule's last look; Grab the download running (#5521).
Search *wantSearchView `json:"search"`
Grab *wantGrabView `json:"grab"`
}
type wantSearchView struct {
SearchedAt string `json:"searched_at"`
Outcome string `json:"outcome"`
Considered int32 `json:"considered"`
BestRelease string `json:"best_release"`
BestVerdict string `json:"best_verdict"`
Error string `json:"error"`
}
type wantGrabView struct {
Name string `json:"name"`
GrabbedAt string `json:"grabbed_at"`
Origin string `json:"origin"`
Quality string `json:"quality"`
}
// wantCounts sizes each tab of the Wanted page.
@@ -118,6 +137,22 @@ func (h *handlers) handleListWants(w http.ResponseWriter, r *http.Request) {
if v.SecondaryTypes == nil {
v.SecondaryTypes = []string{}
}
if row.SearchedAt.Valid {
v.Search = &wantSearchView{
SearchedAt: formatTimestamp(row.SearchedAt), Outcome: row.SearchOutcome,
Considered: row.SearchConsidered, BestRelease: row.BestRelease,
BestVerdict: row.BestVerdict, Error: row.SearchError,
}
}
if row.GrabName != nil {
v.Grab = &wantGrabView{Name: *row.GrabName, GrabbedAt: formatTimestamp(row.GrabbedAt)}
if row.GrabOrigin != nil {
v.Grab.Origin = *row.GrabOrigin
}
if row.GrabQuality != nil {
v.Grab.Quality = *row.GrabQuality
}
}
out.Items = append(out.Items, v)
}
writeJSON(w, http.StatusOK, out)
+11
View File
@@ -77,6 +77,7 @@ func Mount(r chi.Router, pool *pgxpool.Pool, logger *slog.Logger, events *playev
acquisition: acquire.NewStore(pool),
catalogue: cat,
wanted: wanted.New(pool, cat, logger),
searcher: wanted.NewSearcher(pool, cat, logger),
catalogueSearches: newSearchCache(),
catalogueArt: catalogue.NewArt(dataDir, artSources(coverSettings)),
leveled: newLeveledRenderer(dataDir, loudSettings, logger),
@@ -248,6 +249,14 @@ func Mount(r chi.Router, pool *pgxpool.Pool, logger *slog.Logger, events *playev
// profiles that decide which of their releases count.
admin.Get("/wanted", h.handleListWants)
admin.Delete("/wanted/{id}", h.handleRemoveWant)
// Searching and grabbing (M502 #5521).
admin.Get("/acquisition/settings", h.handleGetSearchSettings)
admin.Put("/acquisition/settings", h.handlePutSearchSettings)
admin.Get("/wanted/{id}/candidates", h.handleListCandidates)
admin.Post("/wanted/{id}/search", h.handleSearchWant)
admin.Post("/wanted/{id}/grab", h.handleGrabCandidate)
admin.Post("/wanted/{id}/block", h.handleBlockCandidate)
admin.Delete("/release-blocks/{hash}", h.handleUnblockRelease)
admin.Get("/follows", h.handleListFollows)
admin.Get("/follows/{mbid}", h.handleGetFollow)
admin.Put("/follows/{mbid}", h.handlePutFollow)
@@ -416,6 +425,8 @@ type handlers struct {
// wanted is the wanted list: follows, release-type profiles and wants
// (M502 #5520). Its worker runs in main; the API reads and edits.
wanted *wanted.Service
// searcher searches for wants and grabs releases for them (#5521).
searcher *wanted.Searcher
// catalogueSearches reuses recent MusicBrainz answers; nil asks every
// time, which is what handler tests get.
catalogueSearches *searchCache
+67
View File
@@ -217,6 +217,16 @@ type AcquisitionService struct {
UpdatedAt pgtype.Timestamptz
}
type AcquisitionSetting struct {
ID bool
AutoGrab bool
SearchIntervalHours int32
AllowedQualities []string
LastPassAt pgtype.Timestamptz
LastPass []byte
UpdatedAt pgtype.Timestamptz
}
type Album struct {
ID pgtype.UUID
Title string
@@ -420,6 +430,26 @@ type GeneralLikesArtist struct {
LikedAt pgtype.Timestamptz
}
type Grab struct {
ID pgtype.UUID
WantID pgtype.UUID
InfoHash string
Name string
Key string
Indexer string
Source string
Client string
SavePath string
Quality string
Origin string
Forced bool
GrabbedBy pgtype.UUID
GrabbedAt pgtype.Timestamptz
EndedAt pgtype.Timestamptz
EndReason string
EndDetail string
}
type LibraryChange struct {
ID int64
EntityType string
@@ -652,6 +682,16 @@ type RegistrationSetting struct {
Mode string
}
type ReleaseBlock struct {
InfoHash string
Name string
Reason string
Detail string
WantID pgtype.UUID
BlockedBy pgtype.UUID
BlockedAt pgtype.Timestamptz
}
type ReleaseTypeProfile struct {
ID pgtype.UUID
Name string
@@ -976,6 +1016,33 @@ type Want struct {
FulfilledAt pgtype.Timestamptz
CreatedAt pgtype.Timestamptz
UpdatedAt pgtype.Timestamptz
SearchedAt pgtype.Timestamptz
SearchOutcome string
SearchConsidered int32
BestRelease string
BestVerdict string
SearchError string
}
type WantCandidate struct {
WantID pgtype.UUID
Key string
Name string
InfoHash string
Indexer string
Source string
SizeBytes int64
Seeders int32
FilesCount int32
Format string
Quality string
Year int32
Verdict string
Reason string
Reasons []byte
FoldedFrom []string
Rank int32
SearchedAt pgtype.Timestamptz
}
type YouMightLikeAlbum struct {
+7 -1
View File
@@ -212,7 +212,7 @@ ON CONFLICT (release_group_mbid) DO UPDATE
fulfilled_at = NULL,
updated_at = now()
WHERE wants.fulfilled_at IS NOT NULL
RETURNING id, release_group_mbid, artist_mbid, artist_name, title, primary_type, first_release_date, reason, album_id, created_by, wanted_at, fulfilled_at, created_at, updated_at
RETURNING id, release_group_mbid, artist_mbid, artist_name, title, primary_type, first_release_date, reason, album_id, created_by, wanted_at, fulfilled_at, created_at, updated_at, searched_at, search_outcome, search_considered, best_release, best_verdict, search_error
`
type WantMissingAlbumParams struct {
@@ -249,6 +249,12 @@ func (q *Queries) WantMissingAlbum(ctx context.Context, arg WantMissingAlbumPara
&i.FulfilledAt,
&i.CreatedAt,
&i.UpdatedAt,
&i.SearchedAt,
&i.SearchOutcome,
&i.SearchConsidered,
&i.BestRelease,
&i.BestVerdict,
&i.SearchError,
)
return i, err
}
+585
View File
@@ -0,0 +1,585 @@
// Code generated by sqlc. DO NOT EDIT.
// versions:
// sqlc v1.31.1
// source: search.sql
package dbq
import (
"context"
"github.com/jackc/pgx/v5/pgtype"
)
const blockRelease = `-- name: BlockRelease :exec
INSERT INTO release_blocks (info_hash, name, reason, detail, want_id, blocked_by)
VALUES ($1, $2, $3, $4,
$5, $6)
ON CONFLICT (info_hash) DO UPDATE
SET reason = EXCLUDED.reason, detail = EXCLUDED.detail, blocked_at = now()
`
type BlockReleaseParams struct {
InfoHash string
Name string
Reason string
Detail string
WantID pgtype.UUID
BlockedBy pgtype.UUID
}
func (q *Queries) BlockRelease(ctx context.Context, arg BlockReleaseParams) error {
_, err := q.db.Exec(ctx, blockRelease,
arg.InfoHash,
arg.Name,
arg.Reason,
arg.Detail,
arg.WantID,
arg.BlockedBy,
)
return err
}
const deleteWantCandidates = `-- name: DeleteWantCandidates :exec
DELETE FROM want_candidates WHERE want_id = $1
`
func (q *Queries) DeleteWantCandidates(ctx context.Context, wantID pgtype.UUID) error {
_, err := q.db.Exec(ctx, deleteWantCandidates, wantID)
return err
}
const editionTrackSpan = `-- name: EditionTrackSpan :one
SELECT COALESCE(MIN(track_count) FILTER (WHERE status = 'Official'), MIN(track_count), 0)::int AS min_tracks,
COALESCE(MAX(track_count), 0)::int AS max_tracks
FROM mb_releases
WHERE release_group_mbid = $1 AND track_count > 0
`
type EditionTrackSpanRow struct {
MinTracks int32
MaxTracks int32
}
// The fewest tracks an official edition of the group has (any edition when
// none is official), and the most any has. Zero when none is catalogued.
func (q *Queries) EditionTrackSpan(ctx context.Context, releaseGroupMbid string) (EditionTrackSpanRow, error) {
row := q.db.QueryRow(ctx, editionTrackSpan, releaseGroupMbid)
var i EditionTrackSpanRow
err := row.Scan(&i.MinTracks, &i.MaxTracks)
return i, err
}
const endGrab = `-- name: EndGrab :exec
UPDATE grabs
SET ended_at = now(), end_reason = $1, end_detail = $2
WHERE id = $3 AND ended_at IS NULL
`
type EndGrabParams struct {
EndReason string
EndDetail string
ID pgtype.UUID
}
func (q *Queries) EndGrab(ctx context.Context, arg EndGrabParams) error {
_, err := q.db.Exec(ctx, endGrab, arg.EndReason, arg.EndDetail, arg.ID)
return err
}
const getAcquisitionSettings = `-- name: GetAcquisitionSettings :one
SELECT id, auto_grab, search_interval_hours, allowed_qualities, last_pass_at, last_pass, updated_at FROM acquisition_settings WHERE id = true
`
// Searching for wants and grabbing releases (M502 #5521).
func (q *Queries) GetAcquisitionSettings(ctx context.Context) (AcquisitionSetting, error) {
row := q.db.QueryRow(ctx, getAcquisitionSettings)
var i AcquisitionSetting
err := row.Scan(
&i.ID,
&i.AutoGrab,
&i.SearchIntervalHours,
&i.AllowedQualities,
&i.LastPassAt,
&i.LastPass,
&i.UpdatedAt,
)
return i, err
}
const getLiveGrab = `-- name: GetLiveGrab :one
SELECT id, want_id, info_hash, name, key, indexer, source, client, save_path, quality, origin, forced, grabbed_by, grabbed_at, ended_at, end_reason, end_detail FROM grabs WHERE want_id = $1 AND ended_at IS NULL
`
func (q *Queries) GetLiveGrab(ctx context.Context, wantID pgtype.UUID) (Grab, error) {
row := q.db.QueryRow(ctx, getLiveGrab, wantID)
var i Grab
err := row.Scan(
&i.ID,
&i.WantID,
&i.InfoHash,
&i.Name,
&i.Key,
&i.Indexer,
&i.Source,
&i.Client,
&i.SavePath,
&i.Quality,
&i.Origin,
&i.Forced,
&i.GrabbedBy,
&i.GrabbedAt,
&i.EndedAt,
&i.EndReason,
&i.EndDetail,
)
return i, err
}
const getWantCandidate = `-- name: GetWantCandidate :one
SELECT want_id, key, name, info_hash, indexer, source, size_bytes, seeders, files_count, format, quality, year, verdict, reason, reasons, folded_from, rank, searched_at FROM want_candidates WHERE want_id = $1 AND key = $2
`
type GetWantCandidateParams struct {
WantID pgtype.UUID
Key string
}
func (q *Queries) GetWantCandidate(ctx context.Context, arg GetWantCandidateParams) (WantCandidate, error) {
row := q.db.QueryRow(ctx, getWantCandidate, arg.WantID, arg.Key)
var i WantCandidate
err := row.Scan(
&i.WantID,
&i.Key,
&i.Name,
&i.InfoHash,
&i.Indexer,
&i.Source,
&i.SizeBytes,
&i.Seeders,
&i.FilesCount,
&i.Format,
&i.Quality,
&i.Year,
&i.Verdict,
&i.Reason,
&i.Reasons,
&i.FoldedFrom,
&i.Rank,
&i.SearchedAt,
)
return i, err
}
const getWantForSearch = `-- name: GetWantForSearch :one
SELECT w.id, w.release_group_mbid, w.artist_name, w.title, w.reason, w.fulfilled_at,
COALESCE(NULLIF(w.primary_type, ''), g.primary_type, '')::text AS primary_type,
COALESCE(g.secondary_types, '{}')::text[] AS secondary_types,
COALESCE(NULLIF(w.first_release_date, ''), g.first_release_date, '')::text AS first_release_date
FROM wants w
LEFT JOIN mb_release_groups g ON g.mbid = w.release_group_mbid
WHERE w.id = $1
`
type GetWantForSearchRow struct {
ID pgtype.UUID
ReleaseGroupMbid string
ArtistName string
Title string
Reason string
FulfilledAt pgtype.Timestamptz
PrimaryType string
SecondaryTypes []string
FirstReleaseDate string
}
func (q *Queries) GetWantForSearch(ctx context.Context, id pgtype.UUID) (GetWantForSearchRow, error) {
row := q.db.QueryRow(ctx, getWantForSearch, id)
var i GetWantForSearchRow
err := row.Scan(
&i.ID,
&i.ReleaseGroupMbid,
&i.ArtistName,
&i.Title,
&i.Reason,
&i.FulfilledAt,
&i.PrimaryType,
&i.SecondaryTypes,
&i.FirstReleaseDate,
)
return i, err
}
const insertGrab = `-- name: InsertGrab :one
INSERT INTO grabs (want_id, info_hash, name, key, indexer, source, client, save_path, quality,
origin, forced, grabbed_by)
VALUES ($1, $2, $3, $4, $5,
$6, $7, $8, $9,
$10, $11, $12)
RETURNING id, want_id, info_hash, name, key, indexer, source, client, save_path, quality, origin, forced, grabbed_by, grabbed_at, ended_at, end_reason, end_detail
`
type InsertGrabParams struct {
WantID pgtype.UUID
InfoHash string
Name string
Key string
Indexer string
Source string
Client string
SavePath string
Quality string
Origin string
Forced bool
GrabbedBy pgtype.UUID
}
func (q *Queries) InsertGrab(ctx context.Context, arg InsertGrabParams) (Grab, error) {
row := q.db.QueryRow(ctx, insertGrab,
arg.WantID,
arg.InfoHash,
arg.Name,
arg.Key,
arg.Indexer,
arg.Source,
arg.Client,
arg.SavePath,
arg.Quality,
arg.Origin,
arg.Forced,
arg.GrabbedBy,
)
var i Grab
err := row.Scan(
&i.ID,
&i.WantID,
&i.InfoHash,
&i.Name,
&i.Key,
&i.Indexer,
&i.Source,
&i.Client,
&i.SavePath,
&i.Quality,
&i.Origin,
&i.Forced,
&i.GrabbedBy,
&i.GrabbedAt,
&i.EndedAt,
&i.EndReason,
&i.EndDetail,
)
return i, err
}
const insertWantCandidate = `-- name: InsertWantCandidate :exec
INSERT INTO want_candidates (want_id, key, name, info_hash, indexer, source, size_bytes, seeders,
files_count, format, quality, year, verdict, reason, reasons, folded_from, rank)
VALUES ($1, $2, $3, $4, $5,
$6, $7, $8, $9, $10,
$11, $12, $13, $14, $15,
$16, $17)
ON CONFLICT (want_id, key) DO NOTHING
`
type InsertWantCandidateParams struct {
WantID pgtype.UUID
Key string
Name string
InfoHash string
Indexer string
Source string
SizeBytes int64
Seeders int32
FilesCount int32
Format string
Quality string
Year int32
Verdict string
Reason string
Reasons []byte
FoldedFrom []string
Rank int32
}
func (q *Queries) InsertWantCandidate(ctx context.Context, arg InsertWantCandidateParams) error {
_, err := q.db.Exec(ctx, insertWantCandidate,
arg.WantID,
arg.Key,
arg.Name,
arg.InfoHash,
arg.Indexer,
arg.Source,
arg.SizeBytes,
arg.Seeders,
arg.FilesCount,
arg.Format,
arg.Quality,
arg.Year,
arg.Verdict,
arg.Reason,
arg.Reasons,
arg.FoldedFrom,
arg.Rank,
)
return err
}
const listReleaseBlocks = `-- name: ListReleaseBlocks :many
SELECT info_hash, name, reason, detail, want_id, blocked_by, blocked_at FROM release_blocks ORDER BY blocked_at DESC
`
func (q *Queries) ListReleaseBlocks(ctx context.Context) ([]ReleaseBlock, error) {
rows, err := q.db.Query(ctx, listReleaseBlocks)
if err != nil {
return nil, err
}
defer rows.Close()
var items []ReleaseBlock
for rows.Next() {
var i ReleaseBlock
if err := rows.Scan(
&i.InfoHash,
&i.Name,
&i.Reason,
&i.Detail,
&i.WantID,
&i.BlockedBy,
&i.BlockedAt,
); err != nil {
return nil, err
}
items = append(items, i)
}
if err := rows.Err(); err != nil {
return nil, err
}
return items, nil
}
const listWantCandidates = `-- name: ListWantCandidates :many
SELECT want_id, key, name, info_hash, indexer, source, size_bytes, seeders, files_count, format, quality, year, verdict, reason, reasons, folded_from, rank, searched_at FROM want_candidates WHERE want_id = $1 ORDER BY rank, key
`
func (q *Queries) ListWantCandidates(ctx context.Context, wantID pgtype.UUID) ([]WantCandidate, error) {
rows, err := q.db.Query(ctx, listWantCandidates, wantID)
if err != nil {
return nil, err
}
defer rows.Close()
var items []WantCandidate
for rows.Next() {
var i WantCandidate
if err := rows.Scan(
&i.WantID,
&i.Key,
&i.Name,
&i.InfoHash,
&i.Indexer,
&i.Source,
&i.SizeBytes,
&i.Seeders,
&i.FilesCount,
&i.Format,
&i.Quality,
&i.Year,
&i.Verdict,
&i.Reason,
&i.Reasons,
&i.FoldedFrom,
&i.Rank,
&i.SearchedAt,
); err != nil {
return nil, err
}
items = append(items, i)
}
if err := rows.Err(); err != nil {
return nil, err
}
return items, nil
}
const listWantsDueForSearch = `-- name: ListWantsDueForSearch :many
SELECT w.id, w.release_group_mbid, w.artist_name, w.title, w.reason,
COALESCE(NULLIF(w.primary_type, ''), g.primary_type, '')::text AS primary_type,
COALESCE(g.secondary_types, '{}')::text[] AS secondary_types,
COALESCE(NULLIF(w.first_release_date, ''), g.first_release_date, '')::text AS first_release_date
FROM wants w
LEFT JOIN mb_release_groups g ON g.mbid = w.release_group_mbid
WHERE w.fulfilled_at IS NULL
AND NOT EXISTS (SELECT 1 FROM grabs gr WHERE gr.want_id = w.id AND gr.ended_at IS NULL)
AND (w.searched_at IS NULL OR w.searched_at < now() - make_interval(hours => $1::int))
ORDER BY w.searched_at IS NOT NULL,
CASE w.reason WHEN 'request' THEN 0 WHEN 'missing' THEN 1 ELSE 2 END,
w.searched_at NULLS FIRST, w.wanted_at, w.id
LIMIT $2
`
type ListWantsDueForSearchParams struct {
Hours int32
MaxRows int32
}
type ListWantsDueForSearchRow struct {
ID pgtype.UUID
ReleaseGroupMbid string
ArtistName string
Title string
Reason string
PrimaryType string
SecondaryTypes []string
FirstReleaseDate string
}
// Open wants with no download running that were never searched or not since
// @hours ago: never-searched first, then requests, then missing files, then
// followed artists' releases.
func (q *Queries) ListWantsDueForSearch(ctx context.Context, arg ListWantsDueForSearchParams) ([]ListWantsDueForSearchRow, error) {
rows, err := q.db.Query(ctx, listWantsDueForSearch, arg.Hours, arg.MaxRows)
if err != nil {
return nil, err
}
defer rows.Close()
var items []ListWantsDueForSearchRow
for rows.Next() {
var i ListWantsDueForSearchRow
if err := rows.Scan(
&i.ID,
&i.ReleaseGroupMbid,
&i.ArtistName,
&i.Title,
&i.Reason,
&i.PrimaryType,
&i.SecondaryTypes,
&i.FirstReleaseDate,
); err != nil {
return nil, err
}
items = append(items, i)
}
if err := rows.Err(); err != nil {
return nil, err
}
return items, nil
}
const markCandidatesBlocked = `-- name: MarkCandidatesBlocked :exec
UPDATE want_candidates
SET verdict = 'blocked', reason = $1,
reasons = jsonb_build_array(jsonb_build_object('kind', 'hard', 'code', 'blocked', 'text', $1::text))
WHERE info_hash = $2
`
type MarkCandidatesBlockedParams struct {
Reason string
InfoHash string
}
func (q *Queries) MarkCandidatesBlocked(ctx context.Context, arg MarkCandidatesBlockedParams) error {
_, err := q.db.Exec(ctx, markCandidatesBlocked, arg.Reason, arg.InfoHash)
return err
}
const markCandidatesUnblocked = `-- name: MarkCandidatesUnblocked :exec
UPDATE want_candidates
SET verdict = 'review', reason = $1,
reasons = jsonb_build_array(jsonb_build_object('kind', 'soft', 'code', 'unblocked', 'text', $1::text))
WHERE info_hash = $2 AND verdict = 'blocked'
`
type MarkCandidatesUnblockedParams struct {
Reason string
InfoHash string
}
func (q *Queries) MarkCandidatesUnblocked(ctx context.Context, arg MarkCandidatesUnblockedParams) error {
_, err := q.db.Exec(ctx, markCandidatesUnblocked, arg.Reason, arg.InfoHash)
return err
}
const recordSearchPass = `-- name: RecordSearchPass :exec
UPDATE acquisition_settings SET last_pass_at = now(), last_pass = $1 WHERE id = true
`
func (q *Queries) RecordSearchPass(ctx context.Context, report []byte) error {
_, err := q.db.Exec(ctx, recordSearchPass, report)
return err
}
const recordWantSearch = `-- name: RecordWantSearch :exec
UPDATE wants
SET searched_at = now(),
search_outcome = $1,
search_considered = $2,
best_release = $3,
best_verdict = $4,
search_error = $5,
updated_at = now()
WHERE id = $6
`
type RecordWantSearchParams struct {
Outcome string
Considered int32
BestRelease string
BestVerdict string
SearchError string
ID pgtype.UUID
}
func (q *Queries) RecordWantSearch(ctx context.Context, arg RecordWantSearchParams) error {
_, err := q.db.Exec(ctx, recordWantSearch,
arg.Outcome,
arg.Considered,
arg.BestRelease,
arg.BestVerdict,
arg.SearchError,
arg.ID,
)
return err
}
const unblockRelease = `-- name: UnblockRelease :execrows
DELETE FROM release_blocks WHERE info_hash = $1
`
func (q *Queries) UnblockRelease(ctx context.Context, infoHash string) (int64, error) {
result, err := q.db.Exec(ctx, unblockRelease, infoHash)
if err != nil {
return 0, err
}
return result.RowsAffected(), nil
}
const updateAcquisitionSettings = `-- name: UpdateAcquisitionSettings :one
UPDATE acquisition_settings
SET auto_grab = $1,
search_interval_hours = $2,
allowed_qualities = $3,
updated_at = now()
WHERE id = true
RETURNING id, auto_grab, search_interval_hours, allowed_qualities, last_pass_at, last_pass, updated_at
`
type UpdateAcquisitionSettingsParams struct {
AutoGrab bool
SearchIntervalHours int32
AllowedQualities []string
}
func (q *Queries) UpdateAcquisitionSettings(ctx context.Context, arg UpdateAcquisitionSettingsParams) (AcquisitionSetting, error) {
row := q.db.QueryRow(ctx, updateAcquisitionSettings, arg.AutoGrab, arg.SearchIntervalHours, arg.AllowedQualities)
var i AcquisitionSetting
err := row.Scan(
&i.ID,
&i.AutoGrab,
&i.SearchIntervalHours,
&i.AllowedQualities,
&i.LastPassAt,
&i.LastPass,
&i.UpdatedAt,
)
return i, err
}
+46 -4
View File
@@ -206,7 +206,7 @@ func (q *Queries) GetReleaseTypeProfile(ctx context.Context, id pgtype.UUID) (Re
}
const getWantByReleaseGroup = `-- name: GetWantByReleaseGroup :one
SELECT id, release_group_mbid, artist_mbid, artist_name, title, primary_type, first_release_date, reason, album_id, created_by, wanted_at, fulfilled_at, created_at, updated_at FROM wants WHERE release_group_mbid = $1
SELECT id, release_group_mbid, artist_mbid, artist_name, title, primary_type, first_release_date, reason, album_id, created_by, wanted_at, fulfilled_at, created_at, updated_at, searched_at, search_outcome, search_considered, best_release, best_verdict, search_error FROM wants WHERE release_group_mbid = $1
`
func (q *Queries) GetWantByReleaseGroup(ctx context.Context, releaseGroupMbid string) (Want, error) {
@@ -227,6 +227,12 @@ func (q *Queries) GetWantByReleaseGroup(ctx context.Context, releaseGroupMbid st
&i.FulfilledAt,
&i.CreatedAt,
&i.UpdatedAt,
&i.SearchedAt,
&i.SearchOutcome,
&i.SearchConsidered,
&i.BestRelease,
&i.BestVerdict,
&i.SearchError,
)
return i, err
}
@@ -406,7 +412,7 @@ func (q *Queries) ListFollowsDue(ctx context.Context, arg ListFollowsDueParams)
}
const listOpenWants = `-- name: ListOpenWants :many
SELECT id, release_group_mbid, artist_mbid, artist_name, title, primary_type, first_release_date, reason, album_id, created_by, wanted_at, fulfilled_at, created_at, updated_at FROM wants
SELECT id, release_group_mbid, artist_mbid, artist_name, title, primary_type, first_release_date, reason, album_id, created_by, wanted_at, fulfilled_at, created_at, updated_at, searched_at, search_outcome, search_considered, best_release, best_verdict, search_error FROM wants
WHERE fulfilled_at IS NULL
ORDER BY wanted_at, id
LIMIT $1
@@ -436,6 +442,12 @@ func (q *Queries) ListOpenWants(ctx context.Context, limit int32) ([]Want, error
&i.FulfilledAt,
&i.CreatedAt,
&i.UpdatedAt,
&i.SearchedAt,
&i.SearchOutcome,
&i.SearchConsidered,
&i.BestRelease,
&i.BestVerdict,
&i.SearchError,
); err != nil {
return nil, err
}
@@ -505,10 +517,14 @@ SELECT w.id, w.release_group_mbid, w.artist_mbid, w.artist_name, w.title, w.reas
COALESCE(g.secondary_types, '{}')::text[] AS secondary_types,
COALESCE(NULLIF(w.first_release_date, ''), g.first_release_date, '')::text AS first_release_date,
u.username AS requested_by,
(SELECT ar.id FROM artists ar WHERE ar.mbid = w.artist_mbid) AS library_artist_id
(SELECT ar.id FROM artists ar WHERE ar.mbid = w.artist_mbid) AS library_artist_id,
-- The search schedule's last look (#5521), and the download running.
w.searched_at, w.search_outcome, w.search_considered, w.best_release, w.best_verdict, w.search_error,
gr.name AS grab_name, gr.grabbed_at, gr.origin AS grab_origin, gr.quality AS grab_quality
FROM wants w
LEFT JOIN users u ON u.id = w.created_by
LEFT JOIN mb_release_groups g ON g.mbid = w.release_group_mbid
LEFT JOIN grabs gr ON gr.want_id = w.id AND gr.ended_at IS NULL
WHERE CASE $1::text
WHEN 'held' THEN w.fulfilled_at IS NOT NULL
WHEN 'missing' THEN w.fulfilled_at IS NULL AND w.reason = 'missing'
@@ -539,6 +555,16 @@ type ListWantsRow struct {
FirstReleaseDate string
RequestedBy *string
LibraryArtistID pgtype.UUID
SearchedAt pgtype.Timestamptz
SearchOutcome string
SearchConsidered int32
BestRelease string
BestVerdict string
SearchError string
GrabName *string
GrabbedAt pgtype.Timestamptz
GrabOrigin *string
GrabQuality *string
}
// The Wanted page: open wants (all but missing ones, or only missing ones),
@@ -567,6 +593,16 @@ func (q *Queries) ListWants(ctx context.Context, arg ListWantsParams) ([]ListWan
&i.FirstReleaseDate,
&i.RequestedBy,
&i.LibraryArtistID,
&i.SearchedAt,
&i.SearchOutcome,
&i.SearchConsidered,
&i.BestRelease,
&i.BestVerdict,
&i.SearchError,
&i.GrabName,
&i.GrabbedAt,
&i.GrabOrigin,
&i.GrabQuality,
); err != nil {
return nil, err
}
@@ -791,7 +827,7 @@ INSERT INTO wants (release_group_mbid, artist_mbid, artist_name, title, primary_
VALUES ($1, $2, $3, $4,
$5, $6, $7, $8)
ON CONFLICT (release_group_mbid) DO UPDATE SET updated_at = wants.updated_at
RETURNING id, release_group_mbid, artist_mbid, artist_name, title, primary_type, first_release_date, reason, album_id, created_by, wanted_at, fulfilled_at, created_at, updated_at
RETURNING id, release_group_mbid, artist_mbid, artist_name, title, primary_type, first_release_date, reason, album_id, created_by, wanted_at, fulfilled_at, created_at, updated_at, searched_at, search_outcome, search_considered, best_release, best_verdict, search_error
`
type WantReleaseGroupParams struct {
@@ -834,6 +870,12 @@ func (q *Queries) WantReleaseGroup(ctx context.Context, arg WantReleaseGroupPara
&i.FulfilledAt,
&i.CreatedAt,
&i.UpdatedAt,
&i.SearchedAt,
&i.SearchOutcome,
&i.SearchConsidered,
&i.BestRelease,
&i.BestVerdict,
&i.SearchError,
)
return i, err
}
@@ -0,0 +1,12 @@
DROP TABLE release_blocks;
DROP TABLE grabs;
DROP TABLE want_candidates;
DROP INDEX wants_search_due_idx;
ALTER TABLE wants
DROP COLUMN searched_at,
DROP COLUMN search_outcome,
DROP COLUMN search_considered,
DROP COLUMN best_release,
DROP COLUMN best_verdict,
DROP COLUMN search_error;
DROP TABLE acquisition_settings;
@@ -0,0 +1,102 @@
-- 0082_search_and_grab.up.sql — searching for wants and grabbing releases
-- (Scribe milestone #502, #5521).
--
-- acquisition_settings one row: whether the schedule takes the best release
-- by itself, how often a want is searched again, which
-- quality tiers are taken, and the last pass's report.
-- wants (columns) when each want was last searched and what came of it.
-- want_candidates every release the last search found for a want, with
-- its verdict and the reasons behind it.
-- grabs downloads started for wants; at most one live per want.
-- release_blocks releases never to take again, by infohash.
CREATE TABLE acquisition_settings (
id boolean PRIMARY KEY DEFAULT true CONSTRAINT acquisition_settings_one_row CHECK (id),
auto_grab boolean NOT NULL DEFAULT true,
-- 0 switches the schedule off.
search_interval_hours integer NOT NULL DEFAULT 24
CONSTRAINT acquisition_settings_interval_check CHECK (search_interval_hours BETWEEN 0 AND 720),
-- Lidarr's "Any" profile (note #5527): trash lossy to lossless; unknown off.
allowed_qualities text[] NOT NULL DEFAULT ARRAY['lossless_hires', 'lossless', 'lossy_high',
'lossy_standard', 'lossy_low', 'lossy_poor', 'lossy_trash'],
last_pass_at timestamptz,
last_pass jsonb NOT NULL DEFAULT '{}',
updated_at timestamptz NOT NULL DEFAULT now()
);
INSERT INTO acquisition_settings DEFAULT VALUES;
ALTER TABLE wants
ADD COLUMN searched_at timestamptz,
ADD COLUMN search_outcome text NOT NULL DEFAULT ''
CONSTRAINT wants_search_outcome_check CHECK (search_outcome IN ('', 'grabbed', 'waiting', 'nothing', 'error')),
ADD COLUMN search_considered integer NOT NULL DEFAULT 0,
ADD COLUMN best_release text NOT NULL DEFAULT '',
ADD COLUMN best_verdict text NOT NULL DEFAULT '',
ADD COLUMN search_error text NOT NULL DEFAULT '';
CREATE INDEX wants_search_due_idx ON wants (searched_at NULLS FIRST) WHERE fulfilled_at IS NULL;
CREATE TABLE want_candidates (
want_id uuid NOT NULL REFERENCES wants (id) ON DELETE CASCADE,
-- The infohash, or the name when the source gave none.
key text NOT NULL,
name text NOT NULL,
info_hash text NOT NULL DEFAULT '',
indexer text NOT NULL DEFAULT '',
source text NOT NULL DEFAULT '',
size_bytes bigint NOT NULL DEFAULT 0,
seeders integer NOT NULL DEFAULT 0,
files_count integer NOT NULL DEFAULT 0,
format text NOT NULL DEFAULT '',
quality text NOT NULL DEFAULT '',
year integer NOT NULL DEFAULT 0,
verdict text NOT NULL
CONSTRAINT want_candidates_verdict_check CHECK (verdict IN ('grab', 'review', 'refused', 'rejected', 'blocked')),
reason text NOT NULL DEFAULT '',
reasons jsonb NOT NULL DEFAULT '[]',
folded_from text[] NOT NULL DEFAULT '{}',
-- Position in the search's order, best first.
rank integer NOT NULL DEFAULT 0,
searched_at timestamptz NOT NULL DEFAULT now(),
PRIMARY KEY (want_id, key)
);
CREATE INDEX want_candidates_info_hash_idx ON want_candidates (info_hash) WHERE info_hash <> '';
CREATE TABLE grabs (
id uuid PRIMARY KEY DEFAULT gen_random_uuid(),
want_id uuid REFERENCES wants (id) ON DELETE SET NULL,
info_hash text NOT NULL DEFAULT '',
name text NOT NULL,
key text NOT NULL,
indexer text NOT NULL DEFAULT '',
source text NOT NULL DEFAULT '',
-- The acquisition_services row of the download client it went to.
client text NOT NULL,
save_path text NOT NULL,
quality text NOT NULL DEFAULT '',
origin text NOT NULL CONSTRAINT grabs_origin_check CHECK (origin IN ('schedule', 'operator')),
forced boolean NOT NULL DEFAULT false,
grabbed_by uuid REFERENCES users (id) ON DELETE SET NULL,
grabbed_at timestamptz NOT NULL DEFAULT now(),
ended_at timestamptz,
end_reason text NOT NULL DEFAULT ''
CONSTRAINT grabs_end_reason_check CHECK (end_reason IN ('', 'superseded')),
end_detail text NOT NULL DEFAULT ''
);
-- One live grab per want; the service decides which, this is the backstop.
CREATE UNIQUE INDEX grabs_one_live_per_want ON grabs (want_id) WHERE ended_at IS NULL AND want_id IS NOT NULL;
CREATE INDEX grabs_info_hash_idx ON grabs (info_hash) WHERE info_hash <> '';
CREATE TABLE release_blocks (
info_hash text PRIMARY KEY,
name text NOT NULL DEFAULT '',
-- operator: "never this one"; files: its .torrent held the wrong thing.
reason text NOT NULL CONSTRAINT release_blocks_reason_check CHECK (reason IN ('operator', 'files')),
detail text NOT NULL DEFAULT '',
want_id uuid REFERENCES wants (id) ON DELETE SET NULL,
blocked_by uuid REFERENCES users (id) ON DELETE SET NULL,
blocked_at timestamptz NOT NULL DEFAULT now()
);
+121
View File
@@ -0,0 +1,121 @@
-- Searching for wants and grabbing releases (M502 #5521).
-- name: GetAcquisitionSettings :one
SELECT * FROM acquisition_settings WHERE id = true;
-- name: UpdateAcquisitionSettings :one
UPDATE acquisition_settings
SET auto_grab = sqlc.arg(auto_grab),
search_interval_hours = sqlc.arg(search_interval_hours),
allowed_qualities = sqlc.arg(allowed_qualities),
updated_at = now()
WHERE id = true
RETURNING *;
-- name: RecordSearchPass :exec
UPDATE acquisition_settings SET last_pass_at = now(), last_pass = sqlc.arg(report) WHERE id = true;
-- name: ListWantsDueForSearch :many
-- Open wants with no download running that were never searched or not since
-- @hours ago: never-searched first, then requests, then missing files, then
-- followed artists' releases.
SELECT w.id, w.release_group_mbid, w.artist_name, w.title, w.reason,
COALESCE(NULLIF(w.primary_type, ''), g.primary_type, '')::text AS primary_type,
COALESCE(g.secondary_types, '{}')::text[] AS secondary_types,
COALESCE(NULLIF(w.first_release_date, ''), g.first_release_date, '')::text AS first_release_date
FROM wants w
LEFT JOIN mb_release_groups g ON g.mbid = w.release_group_mbid
WHERE w.fulfilled_at IS NULL
AND NOT EXISTS (SELECT 1 FROM grabs gr WHERE gr.want_id = w.id AND gr.ended_at IS NULL)
AND (w.searched_at IS NULL OR w.searched_at < now() - make_interval(hours => sqlc.arg(hours)::int))
ORDER BY w.searched_at IS NOT NULL,
CASE w.reason WHEN 'request' THEN 0 WHEN 'missing' THEN 1 ELSE 2 END,
w.searched_at NULLS FIRST, w.wanted_at, w.id
LIMIT sqlc.arg(max_rows);
-- name: GetWantForSearch :one
SELECT w.id, w.release_group_mbid, w.artist_name, w.title, w.reason, w.fulfilled_at,
COALESCE(NULLIF(w.primary_type, ''), g.primary_type, '')::text AS primary_type,
COALESCE(g.secondary_types, '{}')::text[] AS secondary_types,
COALESCE(NULLIF(w.first_release_date, ''), g.first_release_date, '')::text AS first_release_date
FROM wants w
LEFT JOIN mb_release_groups g ON g.mbid = w.release_group_mbid
WHERE w.id = $1;
-- name: EditionTrackSpan :one
-- The fewest tracks an official edition of the group has (any edition when
-- none is official), and the most any has. Zero when none is catalogued.
SELECT COALESCE(MIN(track_count) FILTER (WHERE status = 'Official'), MIN(track_count), 0)::int AS min_tracks,
COALESCE(MAX(track_count), 0)::int AS max_tracks
FROM mb_releases
WHERE release_group_mbid = $1 AND track_count > 0;
-- name: RecordWantSearch :exec
UPDATE wants
SET searched_at = now(),
search_outcome = sqlc.arg(outcome),
search_considered = sqlc.arg(considered),
best_release = sqlc.arg(best_release),
best_verdict = sqlc.arg(best_verdict),
search_error = sqlc.arg(search_error),
updated_at = now()
WHERE id = sqlc.arg(id);
-- name: DeleteWantCandidates :exec
DELETE FROM want_candidates WHERE want_id = $1;
-- name: InsertWantCandidate :exec
INSERT INTO want_candidates (want_id, key, name, info_hash, indexer, source, size_bytes, seeders,
files_count, format, quality, year, verdict, reason, reasons, folded_from, rank)
VALUES (sqlc.arg(want_id), sqlc.arg(key), sqlc.arg(name), sqlc.arg(info_hash), sqlc.arg(indexer),
sqlc.arg(source), sqlc.arg(size_bytes), sqlc.arg(seeders), sqlc.arg(files_count), sqlc.arg(format),
sqlc.arg(quality), sqlc.arg(year), sqlc.arg(verdict), sqlc.arg(reason), sqlc.arg(reasons),
sqlc.arg(folded_from), sqlc.arg(rank))
ON CONFLICT (want_id, key) DO NOTHING;
-- name: ListWantCandidates :many
SELECT * FROM want_candidates WHERE want_id = $1 ORDER BY rank, key;
-- name: GetWantCandidate :one
SELECT * FROM want_candidates WHERE want_id = $1 AND key = $2;
-- name: MarkCandidatesBlocked :exec
UPDATE want_candidates
SET verdict = 'blocked', reason = sqlc.arg(reason),
reasons = jsonb_build_array(jsonb_build_object('kind', 'hard', 'code', 'blocked', 'text', sqlc.arg(reason)::text))
WHERE info_hash = sqlc.arg(info_hash);
-- name: MarkCandidatesUnblocked :exec
UPDATE want_candidates
SET verdict = 'review', reason = sqlc.arg(reason),
reasons = jsonb_build_array(jsonb_build_object('kind', 'soft', 'code', 'unblocked', 'text', sqlc.arg(reason)::text))
WHERE info_hash = sqlc.arg(info_hash) AND verdict = 'blocked';
-- name: GetLiveGrab :one
SELECT * FROM grabs WHERE want_id = $1 AND ended_at IS NULL;
-- name: InsertGrab :one
INSERT INTO grabs (want_id, info_hash, name, key, indexer, source, client, save_path, quality,
origin, forced, grabbed_by)
VALUES (sqlc.narg(want_id), sqlc.arg(info_hash), sqlc.arg(name), sqlc.arg(key), sqlc.arg(indexer),
sqlc.arg(source), sqlc.arg(client), sqlc.arg(save_path), sqlc.arg(quality),
sqlc.arg(origin), sqlc.arg(forced), sqlc.narg(grabbed_by))
RETURNING *;
-- name: EndGrab :exec
UPDATE grabs
SET ended_at = now(), end_reason = sqlc.arg(end_reason), end_detail = sqlc.arg(end_detail)
WHERE id = sqlc.arg(id) AND ended_at IS NULL;
-- name: ListReleaseBlocks :many
SELECT * FROM release_blocks ORDER BY blocked_at DESC;
-- name: BlockRelease :exec
INSERT INTO release_blocks (info_hash, name, reason, detail, want_id, blocked_by)
VALUES (sqlc.arg(info_hash), sqlc.arg(name), sqlc.arg(reason), sqlc.arg(detail),
sqlc.narg(want_id), sqlc.narg(blocked_by))
ON CONFLICT (info_hash) DO UPDATE
SET reason = EXCLUDED.reason, detail = EXCLUDED.detail, blocked_at = now();
-- name: UnblockRelease :execrows
DELETE FROM release_blocks WHERE info_hash = $1;
+5 -1
View File
@@ -186,10 +186,14 @@ SELECT w.id, w.release_group_mbid, w.artist_mbid, w.artist_name, w.title, w.reas
COALESCE(g.secondary_types, '{}')::text[] AS secondary_types,
COALESCE(NULLIF(w.first_release_date, ''), g.first_release_date, '')::text AS first_release_date,
u.username AS requested_by,
(SELECT ar.id FROM artists ar WHERE ar.mbid = w.artist_mbid) AS library_artist_id
(SELECT ar.id FROM artists ar WHERE ar.mbid = w.artist_mbid) AS library_artist_id,
-- The search schedule's last look (#5521), and the download running.
w.searched_at, w.search_outcome, w.search_considered, w.best_release, w.best_verdict, w.search_error,
gr.name AS grab_name, gr.grabbed_at, gr.origin AS grab_origin, gr.quality AS grab_quality
FROM wants w
LEFT JOIN users u ON u.id = w.created_by
LEFT JOIN mb_release_groups g ON g.mbid = w.release_group_mbid
LEFT JOIN grabs gr ON gr.want_id = w.id AND gr.ended_at IS NULL
WHERE CASE sqlc.arg(state)::text
WHEN 'held' THEN w.fulfilled_at IS NOT NULL
WHEN 'missing' THEN w.fulfilled_at IS NULL AND w.reason = 'missing'
+11
View File
@@ -109,6 +109,9 @@ var dataTables = []string{
"mb_release_groups", // M502 #5519
"mb_artists", // M502 #5519
"requests", // M502 #5520 (was lidarr_requests)
"release_blocks", // M502 #5521
"grabs", // M502 #5521
"want_candidates", // M502 #5521
"wants", // M502 #5520
"follow_seen_groups", // M502 #5520
"follows", // M502 #5520
@@ -180,6 +183,14 @@ func ResetDB(t *testing.T, pool *pgxpool.Pool) {
); err != nil {
t.Fatalf("dbtest.ResetDB reset release-type profiles: %v", err)
}
// Search and grab settings (M502 #5521), a singleton reset the same way.
if _, err := pool.Exec(ctx, `
UPDATE acquisition_settings
SET auto_grab = DEFAULT, search_interval_hours = DEFAULT, allowed_qualities = DEFAULT,
last_pass_at = NULL, last_pass = DEFAULT, updated_at = DEFAULT`,
); err != nil {
t.Fatalf("dbtest.ResetDB reset acquisition settings: %v", err)
}
// Notification email settings (M489 #5346), reset the same way.
if _, err := pool.Exec(ctx, `
UPDATE notification_email_settings
+76
View File
@@ -0,0 +1,76 @@
package release
import (
"path"
"strconv"
"strings"
)
// File is one file inside a torrent.
type File struct {
Index int
Path string
Size int64
}
// Contents is what a torrent's file list holds.
type Contents struct {
Audio int
Video int
// Format is the format most of the audio files are in, "" when none.
Format string
}
var videoExt = map[string]bool{
"mkv": true, "mp4": true, "avi": true, "m2ts": true, "vob": true,
"mov": true, "webm": true, "wmv": true, "m4v": true, "ts": true,
}
// Inspect counts a file list's audio and video files.
func Inspect(files []File) Contents {
var c Contents
formats := map[string]int{}
for _, f := range files {
ext := strings.TrimPrefix(strings.ToLower(path.Ext(f.Path)), ".")
if fm := formatOf(ext); fm != "" {
c.Audio++
formats[fm]++
} else if videoExt[ext] {
c.Video++
}
}
best := 0
for fm, n := range formats {
if n > best || (n == best && formatRank[fm] > formatRank[c.Format]) {
c.Format, best = fm, n
}
}
return c
}
// JudgeFiles says what a torrent's files say about whether it is the album:
// a veto (code and sentence), or a doubt for a person, or nothing.
func JudgeFiles(w Want, files []File) (veto, reason, doubt string) {
if len(files) == 0 {
return "", "", ""
}
c := Inspect(files)
switch {
case c.Audio == 0 && c.Video > 0:
return VetoVideo, "It holds video files and no audio.", ""
case c.Audio == 0:
return VetoNoAudio, "It holds no audio files Minstrel can read.", ""
case w.MinTracks > 0 && c.Audio < w.MinTracks:
return VetoIncomplete, "It holds " + tracks(c.Audio) + "; the shortest edition has " + tracks(w.MinTracks) + ".", ""
case w.MaxTracks > 0 && c.Audio > 2*w.MaxTracks+4:
return "", "", "It holds " + tracks(c.Audio) + ", more than any edition of the album has: probably more than one album."
}
return "", "", ""
}
func tracks(n int) string {
if n == 1 {
return "1 track"
}
return strconv.Itoa(n) + " tracks"
}
+210
View File
@@ -0,0 +1,210 @@
package release
import "strings"
// Want is the album a search is for, as the catalogue describes it.
type Want struct {
Artist string
Title string
Year int
PrimaryType string // Album, EP, Single, Broadcast, Other
SecondaryTypes []string
// MinTracks and MaxTracks span the group's editions' track counts; zero
// when the catalogue has not read them.
MinTracks int
MaxTracks int
}
// Veto codes: why a name is not the album. Kept on a candidate as its
// reason's code.
const (
VetoArtist = "wrong_artist"
VetoTitle = "wrong_title"
VetoVideo = "video"
VetoDiscography = "discography"
VetoVersion = "other_version"
VetoType = "wrong_type"
VetoNoAudio = "no_audio"
VetoIncomplete = "incomplete"
)
// Match is what a name says about whether it is the album wanted.
type Match struct {
// Veto is a code from above when the name is not the album, with Reason
// saying why in a sentence.
Veto string
Reason string
// Doubt is set when the name may be the album and cannot say for sure: a
// person decides. It is a sentence.
Doubt string
// YearAgrees: the name states the album's year, give or take one.
YearAgrees bool
}
// versions are words that make a release a different recording of the
// album, with what to call it. A word the wanted title or artist holds does
// not count ("Live Through This", "Instrumental Core").
var versions = []struct {
words []string
what string
// typ is the MusicBrainz secondary type that wants this version.
typ string
}{
{[]string{"instrumental", "instrumentals"}, "an instrumental version", ""},
{[]string{"drumless"}, "a drumless edition", ""},
{[]string{"karaoke"}, "a karaoke version", ""},
{[]string{"acapella", "acappella", "accapella", "acapellas"}, "an a cappella version", ""},
{[]string{"remix", "remixes", "remixed"}, "a remix release", "Remix"},
{[]string{"live", "bootleg"}, "a live recording", "Live"},
{[]string{"demo", "demos"}, "demo recordings", "Demo"},
// Not "covers": in a release name that is usually the artwork scans.
{[]string{"tribute", "lullaby", "lullabies"}, "a tribute release", ""},
{[]string{"nightcore", "slowed", "sped"}, "a slowed or sped-up edit", ""},
{[]string{"commentary", "interview", "interviews"}, "commentary or interviews", "Interview"},
}
// collections are words naming more than one album.
var collections = []string{"collection", "anthology", "albums", "boxset", "discography"}
// Judge says whether a release name is the album wanted. It looks only at
// the name; JudgeFiles looks at what a torrent holds.
func Judge(w Want, r Release) Match {
var m Match
m.YearAgrees = r.Year != 0 && w.Year != 0 && abs(r.Year-w.Year) <= 1
if r.Video != "" {
m.Veto, m.Reason = VetoVideo, "A video, not audio (“"+r.Video+"”)."
return m
}
artist := Tokens(w.Artist)
title := Tokens(w.Title)
if len(artist) > 0 && !contains(r.Tokens, artist) && !contains(r.Tokens, withoutThe(artist)) {
m.Veto, m.Reason = VetoArtist, "The name doesn’t mention "+w.Artist+"."
return m
}
if len(title) > 0 && !contains(r.Tokens, title) && !contains(r.Tokens, bare(w.Title)) {
m.Veto, m.Reason = VetoTitle, "The name doesn’t contain the title “"+w.Title+"”."
return m
}
own := map[string]bool{}
for _, t := range append(append([]string{}, title...), artist...) {
own[t] = true
}
if r.Discography != "" && !own[r.Discography] {
m.Veto, m.Reason = VetoDiscography, "A discography or collection ("+r.Discography+"), and one album is wanted."
return m
}
for _, c := range collections {
if !own[c] && has(r.Tokens, c) {
m.Veto, m.Reason = VetoDiscography, "A collection of albums (“"+c+"”), and one album is wanted."
return m
}
}
for _, v := range versions {
if v.typ != "" && hasType(w.SecondaryTypes, v.typ) {
continue
}
for _, word := range v.words {
if !own[word] && has(r.Tokens, word) {
m.Veto, m.Reason = VetoVersion, "This is "+v.what+" (“"+word+"”)."
return m
}
}
}
if w.PrimaryType == "Album" || w.PrimaryType == "EP" {
if !own["single"] && has(r.Tokens, "single") {
m.Veto, m.Reason = VetoType, "A single, and the "+strings.ToLower(w.PrimaryType)+" is wanted."
return m
}
}
if w.PrimaryType == "Album" && !own["ep"] && has(r.Tokens, "ep") {
m.Veto, m.Reason = VetoType, "An EP, and the album is wanted."
return m
}
// A self-titled album: every release of the artist names it once. Two
// mentions, or its year, say it is this one.
if len(title) > 0 && equal(title, artist) && count(r.Tokens, title) < 2 && !m.YearAgrees {
m.Doubt = "The album shares the artist’s name, and nothing else in the name says it is this album."
}
return m
}
func abs(n int) int {
if n < 0 {
return -n
}
return n
}
func withoutThe(t []string) []string {
if len(t) > 1 && t[0] == "the" {
return t[1:]
}
return t
}
// bare is a title without a trailing bracketed part ("Album (Remastered)"),
// for names that leave it off.
func bare(title string) []string {
if i := strings.IndexAny(title, "(["); i > 0 {
return Tokens(title[:i])
}
return nil
}
// contains says whether want's words appear in name in order and together,
// or, for two or more words, run together ("ACDC" for AC/DC). A single word
// is never looked for run together: "muse" is inside "museum".
func contains(name, want []string) bool {
if len(want) == 0 {
return false
}
if count(name, want) > 0 {
return true
}
joined := strings.Join(want, "")
if len(want) < 2 || len(joined) < 4 {
return false
}
return strings.Contains(strings.Join(name, ""), joined)
}
// count is how often want's words appear in name in order and together.
func count(name, want []string) int {
n := 0
for i := 0; i+len(want) <= len(name); i++ {
if equal(name[i:i+len(want)], want) {
n++
}
}
return n
}
func equal(a, b []string) bool {
if len(a) != len(b) {
return false
}
for i := range a {
if a[i] != b[i] {
return false
}
}
return true
}
func has(tokens []string, word string) bool {
for _, t := range tokens {
if t == word {
return true
}
}
return false
}
func hasType(types []string, t string) bool {
for _, x := range types {
if strings.EqualFold(x, t) {
return true
}
}
return false
}
+289
View File
@@ -0,0 +1,289 @@
// Package release reads music release names and judges them against a
// wanted album (M502 #5521): what a name says it is (format, bitrate, bit
// depth, source, year, edition), whether it is the album wanted, and which
// quality tier it sits in. Nothing here touches the network or the database,
// so every rule is testable on a string.
package release
import (
"regexp"
"strconv"
"strings"
)
// Formats a name can state.
const (
FormatFLAC = "FLAC"
FormatALAC = "ALAC"
FormatAPE = "APE"
FormatWV = "WavPack"
FormatWAV = "WAV"
FormatAIFF = "AIFF"
FormatDSD = "DSD"
FormatMP3 = "MP3"
FormatAAC = "AAC"
FormatVorbis = "Vorbis"
FormatOpus = "Opus"
)
// Sources a name can state.
const (
SourceWEB = "WEB"
SourceCD = "CD"
SourceVinyl = "Vinyl"
SourceSACD = "SACD"
SourceCassette = "Cassette"
)
// Release is what a release name says about itself. Zero values mean the
// name does not say.
type Release struct {
Name string
Year int
Format string
// Bitrate is a constant bitrate in kbps; VBR is a LAME preset (V0, V1, V2).
Bitrate int
VBR string
// BitDepth (16, 24) and SampleRate (kHz) for lossless.
BitDepth int
SampleRate float64
Source string
// Video is the marker that says this is a video ("x264", "1080p"), or "".
Video string
// Discography: a year range or a word naming many albums.
Discography string
// Tokens are the name's words, normalised for matching.
Tokens []string
}
var (
formatRe = regexp.MustCompile(`\b(flac|alac|ape|wavpack|wv|wav|aiff|dsd|dsf|dff|mp3|aac|m4a|ogg|vorbis|opus)\b`)
// "24-96", "24/192", "16-44 1" (44.1 after dots become spaces).
depthRateRe = regexp.MustCompile(`\b(16|24|32) ?[-/ ] ?(44 1|44|48|88 2|88|96|176 4|176|192|352 8|384)\b`)
depthRe = regexp.MustCompile(`\b(16|24|32) ?-?bits?\b`)
rateRe = regexp.MustCompile(`\b(44 1|48|88 2|96|176 4|192) ?khz\b`)
hiResRe = regexp.MustCompile(`\bhi ?-?res\b`)
codecRateRe = regexp.MustCompile(`\b(?:mp3|aac|m4a|cbr|ogg|opus)[ \-@]*(320|256|224|192|160|128|112|96|64)\b`)
kbpsRe = regexp.MustCompile(`\b(320|256|224|192|160|128|112|96|64) ?(?:kbps|kbit|kb/s|k)\b`)
vbrRe = regexp.MustCompile(`\bv([0-2])\b`)
webRe = regexp.MustCompile(`\bweb(?: ?-?dl)?\b`)
cdRe = regexp.MustCompile(`\bcd ?(?:rip)?\b`)
vinylRe = regexp.MustCompile(`\b(?:vinyl|vinylrip|lp ?rip|needle ?drop)\b`)
sacdRe = regexp.MustCompile(`\bsacd\b`)
cassetteRe = regexp.MustCompile(`\b(?:cassette|tape ?rip)\b`)
yearRe = regexp.MustCompile(`\b(19[3-9]\d|20[0-4]\d)\b`)
bracketYear = regexp.MustCompile(`[(\[{] ?(19[3-9]\d|20[0-4]\d) ?[)\]}]`)
yearRangeRe = regexp.MustCompile(`\b(19[3-9]\d|20[0-4]\d) ?(?:-|–|to) ?(19[3-9]\d|20[0-4]\d)\b`)
videoRe = regexp.MustCompile(`\b(x264|x265|h ?264|h ?265|hevc|xvid|divx|1080p|1080i|720p|2160p|bluray|blu ?ray|bdrip|brrip|bdremux|dvdrip|dvd9|dvd5|webrip|hdtv|mkv|avi|m2ts)\b`)
discogRe = regexp.MustCompile(`\b(discography|discografia|discographie|diskografie|discografie)\b`)
)
// Parse reads what a release name says.
func Parse(name string) Release {
r := Release{Name: name}
s := spaced(name)
r.Format = bestFormat(formatRe.FindAllStringSubmatch(s, -1))
if m := depthRateRe.FindStringSubmatch(s); m != nil {
r.BitDepth, _ = strconv.Atoi(m[1])
r.SampleRate = rate(m[2])
}
if r.BitDepth == 0 {
if m := depthRe.FindStringSubmatch(s); m != nil {
r.BitDepth, _ = strconv.Atoi(m[1])
}
}
if r.SampleRate == 0 {
if m := rateRe.FindStringSubmatch(s); m != nil {
r.SampleRate = rate(m[1])
}
}
if r.BitDepth == 0 && hiResRe.MatchString(s) {
r.BitDepth = 24
}
if m := codecRateRe.FindStringSubmatch(s); m != nil {
r.Bitrate, _ = strconv.Atoi(m[1])
} else if m := kbpsRe.FindStringSubmatch(s); m != nil {
r.Bitrate, _ = strconv.Atoi(m[1])
}
// A bare "V0" is a LAME preset only beside MP3 or VBR; elsewhere it can
// be part of a title.
if m := vbrRe.FindStringSubmatch(s); m != nil && (r.Format == FormatMP3 || strings.Contains(s, "vbr")) {
r.VBR = "V" + m[1]
}
if r.Format == "" && (r.Bitrate > 0 || r.VBR != "") {
r.Format = FormatMP3
}
switch {
case sacdRe.MatchString(s):
r.Source = SourceSACD
case vinylRe.MatchString(s):
r.Source = SourceVinyl
case webRe.MatchString(s):
r.Source = SourceWEB
case cassetteRe.MatchString(s):
r.Source = SourceCassette
case cdRe.MatchString(s):
r.Source = SourceCD
}
if m := videoRe.FindStringSubmatch(s); m != nil {
r.Video = m[1]
}
if m := discogRe.FindStringSubmatch(s); m != nil {
r.Discography = m[1]
} else if m := yearRangeRe.FindStringSubmatch(s); m != nil && m[2] > m[1] {
r.Discography = m[1] + "–" + m[2]
}
r.Year = year(s)
r.Tokens = Tokens(name)
return r
}
// spaced lowercases a name and turns its separators into spaces, keeping
// the slash in "24/96" and "kb/s".
func spaced(name string) string {
var b strings.Builder
for _, c := range strings.ToLower(name) {
switch c {
case '_', '.', ',':
b.WriteByte(' ')
default:
b.WriteRune(c)
}
}
return b.String()
}
// rate reads a sample rate in kHz; "44", "88" and "176" are the short
// spellings of 44.1, 88.2 and 176.4.
func rate(s string) float64 {
switch s {
case "44":
return 44.1
case "88":
return 88.2
case "176":
return 176.4
}
f, _ := strconv.ParseFloat(strings.ReplaceAll(s, " ", "."), 64)
return f
}
// formatRank orders formats a name might state together ("FLAC + MP3"):
// the best is the one it is described by.
var formatRank = map[string]int{
FormatDSD: 11, FormatFLAC: 10, FormatALAC: 9, FormatWV: 8, FormatAPE: 7, FormatWAV: 6, FormatAIFF: 5,
FormatOpus: 4, FormatAAC: 3, FormatVorbis: 2, FormatMP3: 1,
}
func bestFormat(found [][]string) string {
best := ""
for _, m := range found {
f := formatOf(m[1])
if formatRank[f] > formatRank[best] {
best = f
}
}
return best
}
// formatOf names the format a word or file extension stands for.
func formatOf(word string) string {
switch word {
case "flac":
return FormatFLAC
case "alac":
return FormatALAC
case "ape":
return FormatAPE
case "wavpack", "wv":
return FormatWV
case "wav":
return FormatWAV
case "aiff", "aif":
return FormatAIFF
case "dsd", "dsf", "dff":
return FormatDSD
case "mp3":
return FormatMP3
case "aac", "m4a":
return FormatAAC
case "ogg", "vorbis":
return FormatVorbis
case "opus":
return FormatOpus
}
return ""
}
// year is the release year a name states: one in brackets when there is
// one (so the title "1989" is not read as its year), else the last.
func year(s string) int {
if m := bracketYear.FindStringSubmatch(s); m != nil {
y, _ := strconv.Atoi(m[1])
return y
}
all := yearRe.FindAllString(s, -1)
if len(all) == 0 {
return 0
}
y, _ := strconv.Atoi(all[len(all)-1])
return y
}
// fold maps the accented letters music names carry to plain ones.
var fold = map[rune]string{
'à': "a", 'á': "a", 'â': "a", 'ã': "a", 'ä': "a", 'å': "a", 'ā': "a", 'ą': "a",
'æ': "ae", 'ç': "c", 'ć': "c", 'č': "c",
'è': "e", 'é': "e", 'ê': "e", 'ë': "e", 'ē': "e", 'ė': "e", 'ę': "e", 'ě': "e",
'ì': "i", 'í': "i", 'î': "i", 'ï': "i", 'ī': "i", 'ı': "i",
'ñ': "n", 'ń': "n", 'ň': "n",
'ò': "o", 'ó': "o", 'ô': "o", 'õ': "o", 'ö': "o", 'ø': "o", 'ō': "o", 'ő': "o",
'œ': "oe", 'ù': "u", 'ú': "u", 'û': "u", 'ü': "u", 'ū': "u", 'ů': "u", 'ű': "u",
'ý': "y", 'ÿ': "y", 'ß': "ss", 'š': "s", 'ś': "s", 'ş': "s",
'ž': "z", 'ź': "z", 'ż': "z", 'ł': "l", 'đ': "d", 'ð': "d", 'þ': "th", 'ř': "r",
}
// abbreviations are spelled out on both sides before comparing.
var abbreviations = map[string]string{"vol": "volume", "pt": "part", "n": "and"}
// Tokens are a name's words for matching: lowercase, accents folded, "&"
// and "+" read as "and", apostrophes dropped (so "Don't" is "dont"), every
// other mark a word break.
func Tokens(s string) []string {
var b strings.Builder
for _, c := range strings.ToLower(s) {
switch {
case c >= 'a' && c <= 'z', c >= '0' && c <= '9':
b.WriteRune(c)
case c == '\'' || c == '’' || c == '`' || c == '´':
case c == '&' || c == '+':
b.WriteString(" and ")
default:
if f, ok := fold[c]; ok {
b.WriteString(f)
} else if c > 127 && isLetter(c) {
b.WriteRune(c)
} else {
b.WriteByte(' ')
}
}
}
words := strings.Fields(b.String())
for i, w := range words {
if full, ok := abbreviations[w]; ok {
words[i] = full
}
}
return words
}
// isLetter keeps letters of scripts the fold table does not cover (Cyrillic,
// CJK) as letters rather than breaks.
func isLetter(c rune) bool {
return (c >= 0x0400 && c <= 0x04FF) || (c >= 0x0370 && c <= 0x03FF) || c >= 0x3040
}
+169
View File
@@ -0,0 +1,169 @@
package release
import (
"fmt"
"strconv"
"strings"
)
// Tier is a rung of the quality ladder, Lidarr's names for them.
type Tier string
// The ladder, best first.
const (
TierLosslessHiRes Tier = "lossless_hires"
TierLossless Tier = "lossless"
TierLossyHigh Tier = "lossy_high"
TierLossyStandard Tier = "lossy_standard"
TierLossyLow Tier = "lossy_low"
TierLossyPoor Tier = "lossy_poor"
TierLossyTrash Tier = "lossy_trash"
TierUnknown Tier = "unknown"
)
// losslessCategory is Newznab's Audio/Lossless.
const losslessCategory = 3040
// TierInfo is a rung as the settings page shows it.
type TierInfo struct {
Tier Tier `json:"tier"`
Label string `json:"label"`
What string `json:"what"`
}
// Tiers is the ladder, best first.
var Tiers = []TierInfo{
{TierLosslessHiRes, "Lossless, hi-res", "24-bit or above 48 kHz: FLAC, ALAC, DSD"},
{TierLossless, "Lossless", "FLAC, ALAC, APE, WavPack, WAV at CD quality"},
{TierLossyHigh, "High quality lossy", "MP3 320 or V0, AAC 256 and up"},
{TierLossyStandard, "Standard quality lossy", "MP3 224–256 or V1/V2, AAC 192, Vorbis, Opus, or a lossy format with no bitrate stated"},
{TierLossyLow, "Low quality lossy", "160–192 kbps"},
{TierLossyPoor, "Poor quality lossy", "128 kbps"},
{TierLossyTrash, "Trash quality lossy", "Under 128 kbps"},
{TierUnknown, "Unknown", "Nothing says what format it is"},
}
// DefaultTiers is what a new install allows: Lidarr's "Any" profile, every
// format from trash lossy to lossless (note #5527). Unknown is left out; a
// release that names no format has its .torrent read to find out.
var DefaultTiers = []Tier{
TierLosslessHiRes, TierLossless, TierLossyHigh, TierLossyStandard,
TierLossyLow, TierLossyPoor, TierLossyTrash,
}
// KnownTier says whether t is a rung of the ladder.
func KnownTier(t string) bool {
for _, info := range Tiers {
if string(info.Tier) == t {
return true
}
}
return false
}
// Rank orders tiers: higher is better, Unknown lowest.
func Rank(t Tier) int {
for i, info := range Tiers {
if info.Tier == t {
return len(Tiers) - i
}
}
return 0
}
// Label is a tier's name for a person.
func Label(t Tier) string {
for _, info := range Tiers {
if info.Tier == t {
return info.Label
}
}
return string(t)
}
// TierOf places a release on the ladder. category is the indexer's; 3040
// (Lossless) stands in for a name that states no format.
func TierOf(r Release, category int) Tier {
switch r.Format {
case FormatDSD:
return TierLosslessHiRes
case FormatFLAC, FormatALAC, FormatAPE, FormatWV, FormatWAV, FormatAIFF:
if r.BitDepth >= 24 || r.SampleRate > 48 {
return TierLosslessHiRes
}
return TierLossless
case FormatMP3:
switch r.VBR {
case "V0":
return TierLossyHigh
case "V1", "V2":
return TierLossyStandard
}
return byBitrate(r.Bitrate, 320)
case FormatAAC, FormatVorbis, FormatOpus:
return byBitrate(r.Bitrate, 256)
}
if category == losslessCategory {
return TierLossless
}
return TierUnknown
}
// byBitrate places a lossy bitrate; high is the bitrate from which the codec
// counts as high quality. An unstated bitrate is standard.
func byBitrate(kbps, high int) Tier {
switch {
case kbps == 0:
return TierLossyStandard
case kbps >= high:
return TierLossyHigh
case kbps >= 224 || (high == 256 && kbps >= 192):
return TierLossyStandard
case kbps >= 160:
return TierLossyLow
case kbps >= 128:
return TierLossyPoor
}
return TierLossyTrash
}
// Describe is a release's format in a few words: "FLAC 24-bit/96 kHz",
// "MP3 V0", "AAC 256". Empty when the name states none.
func Describe(r Release) string {
if r.Format == "" {
return ""
}
parts := []string{r.Format}
switch {
case r.BitDepth > 0 && r.SampleRate > 0:
parts = append(parts, fmt.Sprintf("%d-bit/%s kHz", r.BitDepth, khz(r.SampleRate)))
case r.BitDepth > 0:
parts = append(parts, fmt.Sprintf("%d-bit", r.BitDepth))
case r.VBR != "":
parts = append(parts, r.VBR)
case r.Bitrate > 0:
parts = append(parts, strconv.Itoa(r.Bitrate))
}
return strings.Join(parts, " ")
}
func khz(f float64) string {
return strings.TrimSuffix(strconv.FormatFloat(f, 'f', 1, 64), ".0")
}
// Profile is which tiers the operator takes.
type Profile struct {
Allowed map[Tier]bool
}
// NewProfile builds a profile from the stored tier list.
func NewProfile(tiers []string) Profile {
p := Profile{Allowed: map[Tier]bool{}}
for _, t := range tiers {
p.Allowed[Tier(t)] = true
}
return p
}
// Allows says whether the profile takes a tier.
func (p Profile) Allows(t Tier) bool { return p.Allowed[t] }
+251
View File
@@ -0,0 +1,251 @@
package release
import (
"crypto/sha1"
"encoding/hex"
"fmt"
"strings"
"testing"
)
func TestParse_ReadsWhatTheNameSays(t *testing.T) {
for _, c := range []struct {
name string
format, vbr, src string
bitrate, depth, yr int
rate float64
tier Tier
}{
// The wrong edition Lidarr grabbed on 2026-10-09 (note #5527).
{"Daft Punk Random Access Memories (Drumless Edition 2023) [Flac 24-88]", FormatFLAC, "", "", 0, 24, 2023, 88.2, TierLosslessHiRes},
{"Gorillaz - Humanz - (Japanese Edition) - [CD-FLAC] 2017", FormatFLAC, "", SourceCD, 0, 0, 2017, 0, TierLossless},
{"Radiohead - OK Computer (1997) [MP3 320]", FormatMP3, "", "", 320, 0, 1997, 0, TierLossyHigh},
{"Artist - Album 2019 MP3 V0", FormatMP3, "V0", "", 0, 0, 2019, 0, TierLossyHigh},
{"Artist - Album (2020) [WEB] [AAC 256]", FormatAAC, "", SourceWEB, 256, 0, 2020, 0, TierLossyHigh},
{"Artist_-_Album-(2004)-192kbps", FormatMP3, "", "", 192, 0, 2004, 0, TierLossyLow},
{"Artist - Album [24bit-96kHz] FLAC Vinyl", FormatFLAC, "", SourceVinyl, 0, 24, 0, 96, TierLosslessHiRes},
{"Taylor Swift - 1989 (2014) [FLAC]", FormatFLAC, "", "", 0, 0, 2014, 0, TierLossless},
{"Artist - Album", "", "", "", 0, 0, 0, 0, TierUnknown},
} {
r := Parse(c.name)
got := fmt.Sprint(r.Format, r.VBR, r.Source, r.Bitrate, r.BitDepth, r.Year, r.SampleRate, TierOf(r, 0))
want := fmt.Sprint(c.format, c.vbr, c.src, c.bitrate, c.depth, c.yr, c.rate, c.tier)
if got != want {
t.Errorf("Parse(%q)\n got %s\nwant %s", c.name, got, want)
}
}
}
func TestParse_VideoAndDiscographies(t *testing.T) {
if v := Parse("Artist.Album.2018.1080p.BluRay.x264").Video; v != "1080p" {
t.Errorf("video = %q", v)
}
if d := Parse("Artist Discography 1994-2010 FLAC").Discography; d != "discography" {
t.Errorf("discography = %q", d)
}
if d := Parse("Artist - Studio Albums 1994 - 2010 [MP3]").Discography; d != "1994–2010" {
t.Errorf("year range = %q", d)
}
if d := Parse("Artist - Album (2010) [FLAC]").Discography; d != "" {
t.Errorf("one year read as a range: %q", d)
}
}
func judge(w Want, name string) Match { return Judge(w, Parse(name)) }
func TestJudge_TheAlbumAndWhatIsNot(t *testing.T) {
ram := Want{Artist: "Daft Punk", Title: "Random Access Memories", Year: 2013, PrimaryType: "Album"}
for name, veto := range map[string]string{
"Daft Punk - Random Access Memories (2013) [FLAC]": "",
"Daft_Punk-Random_Access_Memories-WEB-2013": "",
"Daft Punk Random Access Memories (Drumless Edition 2023) [Flac 24-88]": VetoVersion,
"Daft Punk - Random Access Memories (Instrumentals) [MP3]": VetoVersion,
"Daft Punk - Random Access Memories Remixes": VetoVersion,
"Daft Punk - Random Access Memories (Live) FLAC": VetoVersion,
"Daft Punk - Discovery (2001) [FLAC]": VetoTitle,
"Random Access Memories (2013) [FLAC]": VetoArtist,
"Daft Punk - Discography 1997-2013 FLAC": VetoTitle,
"Daft Punk - Random Access Memories + Discovery + Homework Collection": VetoDiscography,
"Daft Punk - Random Access Memories (2013) 1080p BluRay": VetoVideo,
"Daft Punk - Random Access Memories (Single) 2013": VetoType,
"Daft Punk - Random Access Memories (Deluxe 10th Anniversary Edition) 2023": "",
} {
if got := judge(ram, name).Veto; got != veto {
t.Errorf("%q: veto %q, want %q", name, got, veto)
}
}
if !judge(ram, "Daft Punk - Random Access Memories (2013) [FLAC]").YearAgrees {
t.Error("the year did not agree")
}
}
func TestJudge_WordsTheWantedTitleHoldsDoNotVeto(t *testing.T) {
hole := Want{Artist: "Hole", Title: "Live Through This", PrimaryType: "Album"}
if m := judge(hole, "Hole - Live Through This (1994) [FLAC]"); m.Veto != "" {
t.Errorf("Live Through This: %s", m.Reason)
}
alive := Want{Artist: "Daft Punk", Title: "Alive 2007", PrimaryType: "Album", SecondaryTypes: []string{"Live"}}
if m := judge(alive, "Daft Punk - Alive 2007 (Live) [FLAC]"); m.Veto != "" {
t.Errorf("a wanted live album: %s", m.Reason)
}
alive.SecondaryTypes = nil
if m := judge(alive, "Daft Punk - Alive 2007 (Live) [FLAC]"); m.Veto != VetoVersion {
t.Errorf("live without the Live type: veto %q", m.Veto)
}
}
func TestJudge_SpellingsOfNames(t *testing.T) {
for _, c := range []struct {
w Want
name string
}{
{Want{Artist: "AC/DC", Title: "Back in Black"}, "ACDC - Back In Black (1980) FLAC"},
{Want{Artist: "AC/DC", Title: "Back in Black"}, "AC-DC - Back in Black [MP3 320]"},
{Want{Artist: "Sigur Rós", Title: "Ágætis byrjun"}, "Sigur Ros - Agaetis Byrjun (1999) [FLAC]"},
{Want{Artist: "Simon & Garfunkel", Title: "Bookends"}, "Simon and Garfunkel - Bookends"},
{Want{Artist: "The Beatles", Title: "Abbey Road"}, "Beatles - Abbey Road (Remastered 2009)"},
{Want{Artist: "Florence + the Machine", Title: "Lungs"}, "Florence and The Machine - Lungs (2009)"},
{Want{Artist: "Guns N' Roses", Title: "Appetite for Destruction"}, "Guns N Roses - Appetite For Destruction"},
} {
if m := judge(c.w, c.name); m.Veto != "" {
t.Errorf("%q for %s / %s: %s", c.name, c.w.Artist, c.w.Title, m.Reason)
}
}
// One word is never matched run together: Muse is not inside Museum.
if m := judge(Want{Artist: "Muse", Title: "Absolution"}, "Museum - Absolution"); m.Veto != VetoArtist {
t.Errorf("Museum read as Muse: %q", m.Veto)
}
}
func TestJudge_ASelfTitledAlbumNeedsMoreThanTheName(t *testing.T) {
weezer := Want{Artist: "Weezer", Title: "Weezer", Year: 1994, PrimaryType: "Album"}
if m := judge(weezer, "Weezer - Weezer (Blue Album) [FLAC]"); m.Doubt != "" || m.Veto != "" {
t.Errorf("named twice: %+v", m)
}
if m := judge(weezer, "Weezer (1994) [FLAC]"); m.Doubt != "" {
t.Errorf("the year agrees: %+v", m)
}
if m := judge(weezer, "Weezer (Green Album) FLAC"); m.Doubt == "" {
t.Error("a self-titled name with nothing else was taken as sure")
}
}
func TestTierOf_TheLadder(t *testing.T) {
for _, c := range []struct {
r Release
cat int
want Tier
}{
{Release{Format: FormatMP3, Bitrate: 256}, 0, TierLossyStandard},
{Release{Format: FormatMP3, Bitrate: 128}, 0, TierLossyPoor},
{Release{Format: FormatMP3, Bitrate: 96}, 0, TierLossyTrash},
{Release{Format: FormatMP3, VBR: "V2"}, 0, TierLossyStandard},
{Release{Format: FormatMP3}, 0, TierLossyStandard},
{Release{Format: FormatAAC, Bitrate: 192}, 0, TierLossyStandard},
{Release{Format: FormatALAC, SampleRate: 96}, 0, TierLosslessHiRes},
{Release{Format: FormatDSD}, 0, TierLosslessHiRes},
{Release{}, 3040, TierLossless},
{Release{}, 3010, TierUnknown},
} {
if got := TierOf(c.r, c.cat); got != c.want {
t.Errorf("TierOf(%+v, %d) = %s, want %s", c.r, c.cat, got, c.want)
}
}
if Rank(TierLossless) <= Rank(TierLossyHigh) || Rank(TierUnknown) >= Rank(TierLossyTrash) {
t.Error("the ladder is out of order")
}
if !NewProfile([]string{"lossless"}).Allows(TierLossless) || NewProfile([]string{"lossless"}).Allows(TierUnknown) {
t.Error("profile")
}
}
func files(paths ...string) []File {
out := make([]File, len(paths))
for i, p := range paths {
out[i] = File{Index: i, Path: p, Size: 1 << 20}
}
return out
}
func numbered(n int, ext string) []string {
var out []string
for i := 1; i <= n; i++ {
out = append(out, fmt.Sprintf("Album/%02d - Track.%s", i, ext))
}
return out
}
func TestJudgeFiles_WhatTheTorrentHolds(t *testing.T) {
w := Want{MinTracks: 10, MaxTracks: 13}
if veto, _, _ := JudgeFiles(w, files(numbered(13, "flac")...)); veto != "" {
t.Errorf("a whole album: %s", veto)
}
if veto, reason, _ := JudgeFiles(w, files(numbered(3, "flac")...)); veto != VetoIncomplete || !strings.Contains(reason, "3 tracks") {
t.Errorf("three tracks: %s %s", veto, reason)
}
if veto, _, _ := JudgeFiles(w, files("Concert/Concert.mkv", "Concert/cover.jpg")); veto != VetoVideo {
t.Errorf("a video: %s", veto)
}
if veto, _, _ := JudgeFiles(w, files("Album.rar", "Album.r00")); veto != VetoNoAudio {
t.Errorf("an archive: %s", veto)
}
if _, _, doubt := JudgeFiles(w, files(numbered(60, "mp3")...)); doubt == "" {
t.Error("sixty tracks for a thirteen-track album raised no doubt")
}
if c := Inspect(files(append(numbered(12, "flac"), "Album/cover.jpg", "Album/album.cue")...)); c.Format != FormatFLAC || c.Audio != 12 {
t.Errorf("inspect = %+v", c)
}
}
// bencode writes the few shapes a test torrent needs.
func bencode(v any) string {
switch x := v.(type) {
case string:
return fmt.Sprintf("%d:%s", len(x), x)
case int:
return fmt.Sprintf("i%de", x)
case []any:
var b strings.Builder
b.WriteString("l")
for _, e := range x {
b.WriteString(bencode(e))
}
return b.String() + "e"
case [][2]any: // an ordered dictionary
var b strings.Builder
b.WriteString("d")
for _, kv := range x {
b.WriteString(bencode(kv[0]) + bencode(kv[1]))
}
return b.String() + "e"
}
panic(fmt.Sprintf("bencode %T", v))
}
func TestParseTorrent_FilesAndInfohash(t *testing.T) {
info := bencode([][2]any{
{"files", []any{
[][2]any{{"length", 300}, {"path", []any{"01 - One.flac"}}},
[][2]any{{"attr", "p"}, {"length", 12}, {"path", []any{".pad", "12"}}},
[][2]any{{"length", 200}, {"path", []any{"Scans", "front.jpg"}}},
}},
{"name", "Artist - Album"},
{"piece length", 16384},
{"pieces", strings.Repeat("x", 20)},
})
body := "d" + bencode("announce") + bencode("http://tracker/") + bencode("info") + info + "e"
meta, err := ParseTorrent([]byte(body))
if err != nil {
t.Fatal(err)
}
sum := sha1.Sum([]byte(info))
if meta.InfoHash != hex.EncodeToString(sum[:]) {
t.Errorf("infohash = %s", meta.InfoHash)
}
if len(meta.Files) != 2 || meta.Files[0].Path != "Artist - Album/01 - One.flac" || meta.Files[1].Path != "Artist - Album/Scans/front.jpg" {
t.Errorf("files = %+v", meta.Files)
}
if _, err := ParseTorrent([]byte("<html>not a torrent</html>")); err == nil {
t.Error("HTML parsed as a torrent")
}
}
+178
View File
@@ -0,0 +1,178 @@
package release
import (
"bytes"
"crypto/sha1"
"encoding/hex"
"errors"
"fmt"
"strconv"
"strings"
)
// Reading a .torrent file, ported from Fabled Librarian (#4463). A .torrent
// is the metadata: its info dictionary names every file, and its SHA-1 is the
// infohash. So a release offered as a .torrent link can be judged on what it
// holds before any of it is downloaded. Bencode is four shapes; walking them
// here costs less than a dependency.
// TorrentMeta is what a .torrent says about itself.
type TorrentMeta struct {
// InfoHash is the v1 infohash: the SHA-1 of the info dictionary's own
// bytes, lowercase hex.
InfoHash string
Name string
Files []File
}
// ErrNotTorrent is bytes that are not a .torrent.
var ErrNotTorrent = errors.New("not a .torrent file")
// maxTorrentBytes bounds what is read, as Exchange bounds what it accepts.
const maxTorrentBytes = 10 << 20
// ParseTorrent reads a .torrent's info dictionary.
func ParseTorrent(b []byte) (TorrentMeta, error) {
if len(b) == 0 || len(b) > maxTorrentBytes || b[0] != 'd' {
return TorrentMeta{}, ErrNotTorrent
}
d := &bdecoder{b: b}
root, err := d.value(0)
if err != nil {
return TorrentMeta{}, fmt.Errorf("%w: %v", ErrNotTorrent, err)
}
top, ok := root.(map[string]any)
if !ok || d.infoEnd == 0 {
return TorrentMeta{}, fmt.Errorf("%w: no info dictionary", ErrNotTorrent)
}
info, ok := top["info"].(map[string]any)
if !ok {
return TorrentMeta{}, fmt.Errorf("%w: no info dictionary", ErrNotTorrent)
}
sum := sha1.Sum(b[d.infoStart:d.infoEnd])
out := TorrentMeta{InfoHash: hex.EncodeToString(sum[:])}
out.Name, _ = info["name"].(string)
files, multi := info["files"].([]any)
if !multi {
// A single-file torrent: the name IS the file.
length, _ := info["length"].(int64)
out.Files = []File{{Path: out.Name, Size: length}}
return out, nil
}
for index, f := range files {
entry, ok := f.(map[string]any)
if !ok {
continue
}
parts, _ := entry["path"].([]any)
segs := []string{out.Name}
for _, p := range parts {
if s, ok := p.(string); ok {
segs = append(segs, s)
}
}
length, _ := entry["length"].(int64)
// BEP 47 padding files carry no content.
if attr, _ := entry["attr"].(string); strings.Contains(attr, "p") {
continue
}
out.Files = append(out.Files, File{Index: index, Path: strings.Join(segs, "/"), Size: length})
}
return out, nil
}
// bdecoder walks bencode, and remembers where the top-level info dictionary
// sits so its raw bytes can be hashed exactly as published.
type bdecoder struct {
b []byte
i int
infoStart, infoEnd int
}
// maxDepth: a real torrent nests four deep. Anything past this is hostile.
const maxDepth = 32
func (d *bdecoder) value(depth int) (any, error) {
if depth > maxDepth {
return nil, errors.New("nested too deep")
}
if d.i >= len(d.b) {
return nil, errors.New("truncated")
}
switch c := d.b[d.i]; {
case c == 'i':
end := bytes.IndexByte(d.b[d.i:], 'e')
if end < 0 {
return nil, errors.New("unterminated integer")
}
n, err := strconv.ParseInt(string(d.b[d.i+1:d.i+end]), 10, 64)
if err != nil {
return nil, err
}
d.i += end + 1
return n, nil
case c == 'l':
d.i++
var out []any
for d.i < len(d.b) && d.b[d.i] != 'e' {
v, err := d.value(depth + 1)
if err != nil {
return nil, err
}
out = append(out, v)
}
if d.i >= len(d.b) {
return nil, errors.New("unterminated list")
}
d.i++
return out, nil
case c == 'd':
d.i++
out := map[string]any{}
for d.i < len(d.b) && d.b[d.i] != 'e' {
k, err := d.str()
if err != nil {
return nil, err
}
start := d.i
v, err := d.value(depth + 1)
if err != nil {
return nil, err
}
if depth == 0 && k == "info" {
d.infoStart, d.infoEnd = start, d.i
}
// Pieces are 20 bytes a piece and never read: not kept.
if k != "pieces" {
out[k] = v
}
}
if d.i >= len(d.b) {
return nil, errors.New("unterminated dictionary")
}
d.i++
return out, nil
case c >= '0' && c <= '9':
return d.str()
default:
return nil, fmt.Errorf("unexpected %q", c)
}
}
func (d *bdecoder) str() (string, error) {
colon := bytes.IndexByte(d.b[d.i:], ':')
if colon < 0 {
return "", errors.New("unterminated string length")
}
n, err := strconv.Atoi(string(d.b[d.i : d.i+colon]))
if err != nil || n < 0 {
return "", errors.New("bad string length")
}
start := d.i + colon + 1
if start+n > len(d.b) {
return "", errors.New("string runs past the end")
}
d.i = start + n
return string(d.b[start : start+n]), nil
}
+322
View File
@@ -0,0 +1,322 @@
package wanted
import (
"context"
"encoding/json"
"errors"
"log/slog"
"strconv"
"time"
"github.com/jackc/pgx/v5/pgtype"
"github.com/jackc/pgx/v5/pgxpool"
"git.fabledsword.com/bvandeusen/minstrel/internal/acquire"
"git.fabledsword.com/bvandeusen/minstrel/internal/catalogue"
"git.fabledsword.com/bvandeusen/minstrel/internal/db/dbq"
"git.fabledsword.com/bvandeusen/minstrel/internal/indexer"
"git.fabledsword.com/bvandeusen/minstrel/internal/release"
)
// Searching for wants on a schedule (M502 #5521), ported from Fabled
// Librarian's internal/wanted. The schedule decides which wants and when;
// internal/acquire judges what a search finds and grabs.
// Search outcomes, matching wants_search_outcome_check.
const (
OutcomeGrabbed = "grabbed"
OutcomeWaiting = "waiting"
OutcomeNothing = "nothing"
OutcomeError = "error"
)
const (
// searchTick is how often the schedule looks for wants due; the
// operator's setting is how long a want waits between searches.
searchTick = 10 * time.Minute
// searchWindow is how long one pass searches: most of a tick, so a pass
// ends before the next begins.
searchWindow = 9 * time.Minute
// duePage bounds what one pass reads. The sources' pace is the real
// limit; this caps a pass whose sources are all unpaced.
duePage = 200
// grabTries is how many grabbable releases one search tries in turn
// when the first is refused for its files.
grabTries = 3
// passTimeout backs up the window for a call none of the clients'
// own timeouts cover, such as a wedged database read.
passTimeout = 15 * time.Minute
)
// ErrWantHeld: the want is already answered by the library.
var ErrWantHeld = errors.New("wanted: the library already holds this album")
// Searcher searches for wants and grabs what they need.
type Searcher struct {
pool *pgxpool.Pool
cat *catalogue.Service
logger *slog.Logger
}
// NewSearcher returns a Searcher reading editions through cat (nil reads
// only what the catalogue already holds).
func NewSearcher(pool *pgxpool.Pool, cat *catalogue.Service, logger *slog.Logger) *Searcher {
if logger == nil {
logger = slog.Default()
}
return &Searcher{pool: pool, cat: cat, logger: logger}
}
// Report is one pass.
type Report struct {
Looked int `json:"looked"`
Grabbed int `json:"grabbed"`
Waiting int `json:"waiting"`
Nothing int `json:"nothing"`
Failed int `json:"failed"`
// Paced: the pass stopped because the next search would wait past its
// window for a source's pace.
Paced bool `json:"paced"`
// Off: the schedule is switched off. Unconfigured: nothing to search
// with. Both are states, not faults.
Off bool `json:"off"`
Unconfigured bool `json:"unconfigured"`
AutoGrab bool `json:"auto_grab"`
Error string `json:"error,omitempty"`
}
// Run searches every searchTick until ctx ends.
func (s *Searcher) Run(ctx context.Context) {
ticker := time.NewTicker(searchTick)
defer ticker.Stop()
for {
select {
case <-ctx.Done():
return
case <-ticker.C:
}
rep := s.Pass(ctx)
if rep.Looked > 0 || rep.Error != "" {
s.logger.Info("wanted: search pass", "looked", rep.Looked, "grabbed", rep.Grabbed,
"waiting", rep.Waiting, "nothing", rep.Nothing, "failed", rep.Failed,
"paced", rep.Paced, "error", rep.Error)
}
}
}
// Pass searches for the wants due, paced by the sources, until the window
// closes, and records what it did.
func (s *Searcher) Pass(ctx context.Context) Report {
rep := s.pass(ctx)
if body, err := json.Marshal(rep); err == nil {
if err := dbq.New(s.pool).RecordSearchPass(ctx, body); err != nil && ctx.Err() == nil {
s.logger.Warn("wanted: record search pass", "err", err)
}
}
return rep
}
func (s *Searcher) pass(ctx context.Context) Report {
var rep Report
ctx, cancel := context.WithTimeout(ctx, passTimeout)
defer cancel()
svc, settings, err := acquire.Build(ctx, s.pool)
if err != nil {
rep.Error = indexer.FirstLine(err.Error())
return rep
}
rep.AutoGrab = settings.AutoGrab
switch {
case settings.SearchIntervalHours <= 0:
rep.Off = true
return rep
case svc.Search == nil:
rep.Unconfigured = true
return rep
}
deadline := time.Now().Add(searchWindow)
due, err := dbq.New(s.pool).ListWantsDueForSearch(ctx, dbq.ListWantsDueForSearchParams{
Hours: settings.SearchIntervalHours, MaxRows: duePage,
})
if err != nil {
rep.Error = indexer.FirstLine(err.Error())
return rep
}
for _, row := range due {
if !svc.Ready(ctx, deadline) {
rep.Paced = ctx.Err() == nil
return rep
}
want := s.wantFor(ctx, row.ID, row.ArtistName, row.Title, row.PrimaryType, row.SecondaryTypes,
row.FirstReleaseDate, row.ReleaseGroupMbid)
out := s.one(ctx, svc, want, settings.AutoGrab)
rep.Looked++
switch out.Outcome {
case OutcomeGrabbed:
rep.Grabbed++
case OutcomeWaiting:
rep.Waiting++
case OutcomeNothing:
rep.Nothing++
default:
rep.Failed++
}
}
return rep
}
// wantFor describes a want for the matcher, with its editions' track counts
// from the catalogue (read from MusicBrainz the first time when it can be).
func (s *Searcher) wantFor(ctx context.Context, id pgtype.UUID, artist, title, primary string, secondary []string, date, group string) acquire.Want {
w := acquire.Want{ID: id, Want: release.Want{
Artist: artist, Title: title, Year: yearOf(date), PrimaryType: primary, SecondaryTypes: secondary,
}}
q := dbq.New(s.pool)
span, err := q.EditionTrackSpan(ctx, group)
if err == nil && span.MinTracks == 0 && s.cat != nil {
if _, rerr := s.cat.Releases(ctx, group); rerr == nil {
span, err = q.EditionTrackSpan(ctx, group)
}
}
if err == nil {
w.MinTracks, w.MaxTracks = int(span.MinTracks), int(span.MaxTracks)
}
return w
}
func yearOf(date string) int {
if len(date) < 4 {
return 0
}
y, _ := strconv.Atoi(date[:4])
return y
}
// Searched is what one want's search came to.
type Searched struct {
Outcome string `json:"outcome"`
Considered int `json:"considered"`
Best string `json:"best_release"`
Verdict string `json:"best_verdict"`
Grabbed string `json:"grabbed,omitempty"`
Narrowed []acquire.Narrowing `json:"narrowed"`
Error string `json:"error,omitempty"`
}
// one searches for a want, takes the best release when auto-grab is on, and
// records what came of it on the want.
func (s *Searcher) one(ctx context.Context, svc *acquire.Service, want acquire.Want, autoGrab bool) Searched {
found := svc.For(ctx, want)
if found.KeepError != "" {
s.logger.Warn("wanted: searched, but could not keep the candidates", "want", want.Title, "err", found.KeepError)
}
out := Searched{Considered: found.Considered, Narrowed: found.Narrowed}
if out.Narrowed == nil {
out.Narrowed = []acquire.Narrowing{}
}
if len(found.Candidates) > 0 {
out.Best, out.Verdict = found.Candidates[0].Name, string(found.Candidates[0].Verdict)
}
switch {
case found.Unconfigured:
out.Outcome, out.Error = OutcomeError, "No indexer is set up, so there is nothing to search with."
case found.Error != "":
out.Outcome, out.Error = OutcomeError, found.Error
case !found.ForAPerson():
out.Outcome = OutcomeNothing
case len(found.Grabbable()) == 0 || !autoGrab:
out.Outcome = OutcomeWaiting
default:
out.Outcome = OutcomeWaiting
s.grabBest(ctx, svc, want, found, &out)
}
s.record(ctx, want.ID, out)
return out
}
// grabBest takes the best grabbable release, trying the next when one is
// refused for what its files hold.
func (s *Searcher) grabBest(ctx context.Context, svc *acquire.Service, want acquire.Want, found acquire.Result, out *Searched) {
for i, c := range found.Grabbable() {
if i == grabTries {
return
}
took := svc.Grab(ctx, want, &c, found.Candidates, false, acquire.Origin{By: acquire.OriginSchedule})
switch {
case took.Added:
out.Outcome, out.Grabbed, out.Best, out.Verdict = OutcomeGrabbed, c.Name, c.Name, string(acquire.Grab)
if took.Error != "" {
s.logger.Warn("wanted: took a release but could not record it", "want", want.Title, "err", took.Error)
}
s.logger.Info("wanted: took a release", "want", want.Title, "artist", want.Artist,
"release", c.Name, "quality", c.Tier, "save_path", took.SavePath)
return
case took.Refused:
continue
case took.Unconfigured:
out.Outcome, out.Error = OutcomeError, "No download client is set up, so nothing was taken."
return
case took.Contest != nil:
return
case took.Error != "":
out.Outcome, out.Error = OutcomeError, took.Error
return
}
}
}
func (s *Searcher) record(ctx context.Context, id pgtype.UUID, out Searched) {
if err := dbq.New(s.pool).RecordWantSearch(ctx, dbq.RecordWantSearchParams{
ID: id, Outcome: out.Outcome, Considered: int32(out.Considered),
BestRelease: out.Best, BestVerdict: out.Verdict, SearchError: out.Error,
}); err != nil && ctx.Err() == nil {
s.logger.Warn("wanted: record search", "err", err)
}
}
// SearchNow searches for one want at once, as the schedule would, for a
// person pressing Search: it spends the sources' pace but never waits on it.
func (s *Searcher) SearchNow(ctx context.Context, id pgtype.UUID) (Searched, error) {
want, err := s.load(ctx, id)
if err != nil {
return Searched{}, err
}
svc, settings, err := acquire.Build(ctx, s.pool)
if err != nil {
return Searched{}, err
}
return s.one(ctx, svc, want, settings.AutoGrab), nil
}
// Take grabs the release named by key for a person, searching again first.
func (s *Searcher) Take(ctx context.Context, id pgtype.UUID, key string, force bool, by pgtype.UUID) (acquire.Taken, error) {
want, err := s.load(ctx, id)
if err != nil {
return acquire.Taken{}, err
}
svc, _, err := acquire.Build(ctx, s.pool)
if err != nil {
return acquire.Taken{}, err
}
took := svc.Take(ctx, want, key, force, acquire.Origin{By: acquire.OriginOperator, User: by})
if took.Added && took.Chosen != nil {
s.record(ctx, id, Searched{
Outcome: OutcomeGrabbed, Best: took.Chosen.Name, Verdict: string(took.Chosen.Verdict),
Considered: 0, Grabbed: took.Chosen.Name,
})
}
return took, nil
}
// load reads an open want for a search.
func (s *Searcher) load(ctx context.Context, id pgtype.UUID) (acquire.Want, error) {
row, err := dbq.New(s.pool).GetWantForSearch(ctx, id)
if err != nil {
return acquire.Want{}, err
}
if row.FulfilledAt.Valid {
return acquire.Want{}, ErrWantHeld
}
return s.wantFor(ctx, row.ID, row.ArtistName, row.Title, row.PrimaryType, row.SecondaryTypes,
row.FirstReleaseDate, row.ReleaseGroupMbid), nil
}
+195
View File
@@ -0,0 +1,195 @@
package wanted
import (
"context"
"encoding/json"
"io"
"log/slog"
"net/http"
"net/http/httptest"
"strings"
"sync/atomic"
"testing"
"git.fabledsword.com/bvandeusen/minstrel/internal/acquire"
"git.fabledsword.com/bvandeusen/minstrel/internal/db/dbq"
)
const ramHash = "0123456789abcdef0123456789abcdef01234567"
const ramFeed = `<?xml version="1.0"?>
<rss xmlns:torznab="http://torznab.com/schemas/2015/feed"><channel>
<item><title>Daft Punk - Random Access Memories (2013) [FLAC]</title>
<link>magnet:?xt=urn:btih:` + ramHash + `</link><size>400000000</size>
<torznab:attr name="seeders" value="12"/><torznab:attr name="indexer" value="Knaben"/></item>
<item><title>Daft Punk Random Access Memories (Drumless Edition 2023) [Flac 24-88]</title>
<link>magnet:?xt=urn:btih:fedcba9876543210fedcba9876543210fedcba98</link><size>900000000</size>
<torznab:attr name="seeders" value="40"/></item>
</channel></rss>`
// searchStack is a fake indexer feed and a fake FabledExchange download
// client, configured the way the operator would in Integrations.
func searchStack(t *testing.T, ctx context.Context, store *acquire.Store) *atomic.Int32 {
t.Helper()
var added atomic.Int32
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
switch {
case r.URL.Path == "/feed":
if q := r.URL.Query().Get("q"); q != "Daft Punk Random Access Memories" {
t.Errorf("searched for %q", q)
}
_, _ = io.WriteString(w, ramFeed)
case r.URL.Path == "/api/downloads" && r.Method == http.MethodPost:
var body struct {
Magnet string `json:"magnet"`
SavePath string `json:"save_path"`
}
_ = json.NewDecoder(r.Body).Decode(&body)
if !strings.Contains(body.Magnet, ramHash) || body.SavePath != "/downloads/"+ramHash {
t.Errorf("added %+v", body)
}
added.Add(1)
_, _ = io.WriteString(w, "{}")
default:
http.NotFound(w, r)
}
}))
t.Cleanup(srv.Close)
feed, err := store.Create(ctx, acquire.ProtocolTorznab)
if err != nil {
t.Fatal(err)
}
if _, err := store.Save(ctx, feed.Code, acquire.Update{
Label: "Feed", Enabled: true, BaseURL: srv.URL + "/feed", TimeoutMs: 5000, LocalPath: "/downloads",
}); err != nil {
t.Fatal(err)
}
ex, err := store.Create(ctx, acquire.ProtocolExchange)
if err != nil {
t.Fatal(err)
}
if _, err := store.Save(ctx, ex.Code, acquire.Update{
Label: "Exchange", Enabled: true, BaseURL: srv.URL, TimeoutMs: 5000, ServesDownloads: true, LocalPath: "/downloads",
}); err != nil {
t.Fatal(err)
}
return &added
}
func TestSearcher_FindsAndTakesAWant_Integration(t *testing.T) {
ctx := context.Background()
pool := newPool(t)
q := dbq.New(pool)
added := searchStack(t, ctx, acquire.NewStore(pool))
w, err := WantWith(ctx, q, WantParams{
GroupMBID: "rg-ram", ArtistMBID: "mb-daft-punk", ArtistName: "Daft Punk", Title: "Random Access Memories",
PrimaryType: "Album", FirstReleaseDate: "2013-05-17", Reason: ReasonRequest,
})
if err != nil {
t.Fatal(err)
}
s := NewSearcher(pool, nil, slog.New(slog.NewTextHandler(io.Discard, nil)))
rep := s.Pass(ctx)
if rep.Looked != 1 || rep.Grabbed != 1 || !rep.AutoGrab || added.Load() != 1 {
t.Fatalf("pass = %+v, added %d", rep, added.Load())
}
got, err := q.GetWantByReleaseGroup(ctx, "rg-ram")
if err != nil {
t.Fatal(err)
}
if got.SearchOutcome != OutcomeGrabbed || got.SearchConsidered != 2 || !strings.Contains(got.BestRelease, "(2013) [FLAC]") {
t.Errorf("want after the pass: %s %d %q", got.SearchOutcome, got.SearchConsidered, got.BestRelease)
}
cands, err := q.ListWantCandidates(ctx, w.ID)
if err != nil {
t.Fatal(err)
}
if len(cands) != 2 || cands[0].Verdict != "grab" || cands[1].Verdict != "rejected" || !strings.Contains(cands[1].Reason, "drumless") {
t.Errorf("candidates = %+v", cands)
}
g, err := q.GetLiveGrab(ctx, w.ID)
if err != nil {
t.Fatal(err)
}
if g.InfoHash != ramHash || g.Client != "exchange" || g.Origin != acquire.OriginSchedule || g.Quality != "lossless" {
t.Errorf("grab = %+v", g)
}
settings, _ := q.GetAcquisitionSettings(ctx)
if !settings.LastPassAt.Valid || !strings.Contains(string(settings.LastPass), `"grabbed":1`) {
t.Errorf("last pass = %s", settings.LastPass)
}
// A want with a download running is not searched again.
if rep := s.Pass(ctx); rep.Looked != 0 {
t.Errorf("searched a want already downloading: %+v", rep)
}
// Setting the drumless edition aside marks it on every kept candidate;
// allowing it again leaves it for a person until the next search.
mem := acquire.NewMemory(pool)
if err := mem.Block(ctx, dbq.BlockReleaseParams{
InfoHash: cands[1].InfoHash, Name: cands[1].Name, Reason: acquire.BlockOperator, WantID: w.ID,
}); err != nil {
t.Fatal(err)
}
c, _ := q.GetWantCandidate(ctx, dbq.GetWantCandidateParams{WantID: w.ID, Key: cands[1].Key})
if c.Verdict != "blocked" {
t.Errorf("blocked candidate = %s", c.Verdict)
}
if ok, err := mem.Unblock(ctx, cands[1].InfoHash); err != nil || !ok {
t.Fatalf("unblock: %v %v", ok, err)
}
c, _ = q.GetWantCandidate(ctx, dbq.GetWantCandidateParams{WantID: w.ID, Key: cands[1].Key})
if c.Verdict != "review" {
t.Errorf("unblocked candidate = %s", c.Verdict)
}
}
func TestSearcher_OffAndUnconfigured_Integration(t *testing.T) {
ctx := context.Background()
pool := newPool(t)
s := NewSearcher(pool, nil, slog.New(slog.NewTextHandler(io.Discard, nil)))
if rep := s.Pass(ctx); !rep.Unconfigured {
t.Errorf("no indexer: %+v", rep)
}
if _, err := dbq.New(pool).UpdateAcquisitionSettings(ctx, dbq.UpdateAcquisitionSettingsParams{
AutoGrab: true, SearchIntervalHours: 0, AllowedQualities: []string{"lossless"},
}); err != nil {
t.Fatal(err)
}
if rep := s.Pass(ctx); !rep.Off {
t.Errorf("interval 0: %+v", rep)
}
}
func TestGrabs_OneLivePerWant_Integration(t *testing.T) {
ctx := context.Background()
pool := newPool(t)
q := dbq.New(pool)
w, err := WantWith(ctx, q, WantParams{GroupMBID: "rg-x", ArtistMBID: "mb-x", ArtistName: "X", Title: "Y", Reason: ReasonFollow})
if err != nil {
t.Fatal(err)
}
mem := acquire.NewMemory(pool)
grab := func(hash string) dbq.InsertGrabParams {
return dbq.InsertGrabParams{WantID: w.ID, InfoHash: hash, Name: hash, Key: hash, Client: "qbittorrent",
SavePath: "/downloads/" + hash, Origin: acquire.OriginSchedule}
}
first, err := mem.Grabbed(ctx, grab("aaaa"), dbq.Grab{}.ID, "")
if err != nil {
t.Fatal(err)
}
if _, err := mem.Grabbed(ctx, grab("bbbb"), dbq.Grab{}.ID, ""); err == nil {
t.Error("a second live grab for one want was recorded")
}
second, err := mem.Grabbed(ctx, grab("bbbb"), first.ID, "Replaced by bbbb.")
if err != nil {
t.Fatalf("replacing: %v", err)
}
live, ok, err := mem.LiveGrab(ctx, w.ID)
if err != nil || !ok || live.ID != second.ID {
t.Errorf("live = %+v %v %v", live, ok, err)
}
}
+3
View File
@@ -45,6 +45,9 @@ export const qk = {
adminFollows: () => ['adminFollows'] as const,
adminFollow: (artistMbid: string) => ['adminFollow', artistMbid] as const,
releaseProfiles: () => ['releaseProfiles'] as const,
// Search and grab (M502 #5521).
wantCandidates: (wantId: string) => ['wantCandidates', wantId] as const,
searchSettings: () => ['searchSettings'] as const,
myQuarantine: () => ['myQuarantine'] as const,
adminQuarantine: () => ['adminQuarantine'] as const,
adminQuarantineActions: (limit?: number) =>
+95
View File
@@ -590,6 +590,101 @@ export type Want = {
wanted_at: string;
held_at: string | null;
image_url: string;
// The search schedule's last look, and the download running (#5521).
search?: WantSearch | null;
grab?: WantGrab | null;
};
export type SearchOutcome = 'grabbed' | 'waiting' | 'nothing' | 'error';
export type WantSearch = {
searched_at: string;
outcome: SearchOutcome;
considered: number;
best_release: string;
best_verdict: string;
error: string;
};
export type WantGrab = {
name: string;
grabbed_at: string;
origin: 'schedule' | 'operator';
quality: string;
};
// Search and grab (M502 #5521) ----------------------------------------------
export type CandidateVerdict = 'grab' | 'review' | 'refused' | 'rejected' | 'blocked';
// One release a want's last search found, with what was decided about it.
export type Candidate = {
key: string;
name: string;
info_hash: string;
indexer: string;
size_bytes: number;
seeders: number;
files_count: number;
format: string;
quality: string;
year: number;
verdict: CandidateVerdict;
reason: string;
reasons: { kind: 'hard' | 'soft'; code: string; text: string }[];
folded_from: string[];
searched_at: string;
downloading: boolean;
};
export type Narrowing = { source: string; label: string; reason: string };
export type SearchResult = {
outcome: SearchOutcome;
considered: number;
best_release: string;
best_verdict: string;
grabbed?: string;
narrowed: Narrowing[];
error?: string;
};
export type GrabResult = {
added: boolean;
refused: boolean;
gone: boolean;
unconfigured: boolean;
missing?: string;
kept: boolean;
replaced: boolean;
verdict?: CandidateVerdict;
reason?: string;
error?: string;
};
export type QualityTier = { tier: string; label: string; what: string };
// The last scheduled pass; empty before the first.
export type SearchPass = {
looked?: number;
grabbed?: number;
waiting?: number;
nothing?: number;
failed?: number;
paced?: boolean;
off?: boolean;
unconfigured?: boolean;
auto_grab?: boolean;
error?: string;
};
export type SearchSettings = {
auto_grab: boolean;
search_interval_hours: number;
allowed_qualities: string[];
tiers: QualityTier[];
last_pass_at: string | null;
last_pass: SearchPass;
};
export type WantCounts = { wanted: number; missing: number; held: number; following: number };
+60
View File
@@ -2,11 +2,15 @@ import { createInfiniteQuery, createQuery } from '@tanstack/svelte-query';
import { api } from './client';
import { qk } from './queries';
import type {
Candidate,
Follow,
FollowState,
ReleaseProfile,
ReleaseProfileInput,
ReleaseProfilesResponse,
GrabResult,
SearchResult,
SearchSettings,
WantState,
WantsPage
} from './types';
@@ -100,3 +104,59 @@ export async function updateReleaseProfile(id: string, body: ReleaseProfileInput
export async function deleteReleaseProfile(id: string): Promise<void> {
await api.del(`/api/admin/release-profiles/${id}`);
}
// Search and grab (M502 #5521): each want's last search, a person's Search,
// Grab and "never this one", and the schedule's settings.
export async function listCandidates(wantId: string): Promise<Candidate[]> {
const res = await api.get<{ items: Candidate[] }>(`/api/admin/wanted/${wantId}/candidates`);
return res.items;
}
export function createCandidatesQuery(wantId: string) {
return createQuery({
queryKey: qk.wantCandidates(wantId),
queryFn: () => listCandidates(wantId),
staleTime: 30_000
});
}
// Searches now, as the schedule would: takes the best release when
// auto-grab is on. Can take most of a minute.
export async function searchWant(wantId: string): Promise<SearchResult> {
return api.post<SearchResult>(`/api/admin/wanted/${wantId}/search`, {});
}
// Takes the release named by key; force takes what the verdict would not.
export async function grabCandidate(wantId: string, key: string, force: boolean): Promise<GrabResult> {
return api.post<GrabResult>(`/api/admin/wanted/${wantId}/grab`, { key, force });
}
// Never takes this release again, for any want.
export async function blockCandidate(wantId: string, key: string): Promise<void> {
await api.post(`/api/admin/wanted/${wantId}/block`, { key });
}
export async function unblockRelease(infoHash: string): Promise<void> {
await api.del(`/api/admin/release-blocks/${infoHash}`);
}
export async function getSearchSettings(): Promise<SearchSettings> {
return api.get<SearchSettings>('/api/admin/acquisition/settings');
}
export function createSearchSettingsQuery() {
return createQuery({
queryKey: qk.searchSettings(),
queryFn: getSearchSettings,
staleTime: 30_000
});
}
export async function putSearchSettings(body: {
auto_grab: boolean;
search_interval_hours: number;
allowed_qualities: string[];
}): Promise<SearchSettings> {
return api.put<SearchSettings>('/api/admin/acquisition/settings', body);
}
@@ -0,0 +1,162 @@
<script lang="ts">
import { useQueryClient } from '@tanstack/svelte-query';
import { createSearchSettingsQuery, putSearchSettings } from '#lib/api/wanted.js';
import { qk } from '#lib/api/queries.js';
import { errMessage } from '#lib/api/errors.js';
import { pushToast } from '#lib/stores/toast.svelte.js';
import { relativeTime } from '#lib/utils/relativeTime.js';
import type { SearchSettings } from '#lib/api/types.js';
// Searching and grabbing (M502 #5521): how often each want is searched
// for, whether the best release is taken without asking, and which
// qualities are taken at all. The schedule wakes every ten minutes and
// keeps to each indexer's searches-per-hour.
const client = useQueryClient();
const store = createSearchSettingsQuery();
const query = $derived($store);
type Form = { auto_grab: boolean; search_interval_hours: number; allowed: string[] };
// The form is the server's settings until edited; saved holds what a save
// returned until the query catches up.
let saved = $state<SearchSettings | null>(null);
let edited = $state<Form | null>(null);
let saving = $state(false);
const base = $derived(saved ?? query.data ?? null);
const form = $derived<Form | null>(edited ?? (base ? fromSettings(base) : null));
function fromSettings(s: SearchSettings): Form {
return { auto_grab: s.auto_grab, search_interval_hours: s.search_interval_hours, allowed: [...s.allowed_qualities] };
}
const dirty = $derived(!!edited && !!base && JSON.stringify(edited) !== JSON.stringify(fromSettings(base)));
function edit(change: Partial<Form>) {
if (form) edited = { ...form, ...change };
}
function toggle(tier: string) {
if (!form) return;
edit({ allowed: form.allowed.includes(tier) ? form.allowed.filter((t) => t !== tier) : [...form.allowed, tier] });
}
async function save() {
if (!form) return;
saving = true;
try {
saved = await putSearchSettings({
auto_grab: form.auto_grab,
search_interval_hours: Number(form.search_interval_hours),
allowed_qualities: form.allowed
});
edited = null;
await client.invalidateQueries({ queryKey: qk.searchSettings() });
pushToast('Search settings saved. The next pass uses them.');
} catch (e) {
pushToast(errMessage(e), 'error');
} finally {
saving = false;
}
}
function lastPass(s: SearchSettings): string {
const p = s.last_pass;
if (!s.last_pass_at) return 'The schedule hasn’t run yet.';
const when = `Last pass ${relativeTime(s.last_pass_at)}`;
if (p.off) return `${when}: the schedule is off.`;
if (p.unconfigured) return `${when}: no indexer is set up, so nothing was searched.`;
if (p.error) return `${when} failed: ${p.error}`;
if (!p.looked) return `${when}: nothing was due.`;
let said = `${when}: searched ${p.looked}, took ${p.grabbed ?? 0}, ${p.waiting ?? 0} waiting for you, ${p.nothing ?? 0} with nothing found`;
if (p.failed) said += `, ${p.failed} failed`;
said += '.';
if (p.paced) said += ' It stopped to keep to your indexers’ searches per hour; the rest wait for the next pass.';
return said;
}
</script>
<section class="space-y-4 rounded-xl border border-border bg-surface p-5" data-testid="search-settings-card">
<div>
<h3 class="font-display text-lg font-medium text-text-primary">Searching and grabbing</h3>
<p class="mt-1 text-sm text-text-secondary">
Minstrel searches your indexers for each album on the
<a href="/admin/wanted" class="text-accent-fg hover:underline">wanted list</a>, judges every
release it finds, and hands the best one to your download client. Each album's releases and
the reason for every verdict are on the Wanted page.
</p>
</div>
{#if query.isPending}
<p class="text-sm text-text-secondary">Loading…</p>
{:else if query.isError || !form || !query.data}
<p class="text-sm text-action-destructive">Couldn't load the search settings.</p>
{:else}
<label class="flex items-start gap-3">
<input
type="checkbox"
checked={form.auto_grab}
onchange={(e) => edit({ auto_grab: e.currentTarget.checked })}
class="mt-1"
/>
<span>
<span class="text-sm text-text-primary">Take the best release without asking</span>
<span class="block text-xs text-text-secondary">
Off, a search only lists what it found, and you choose from the Wanted page.
</span>
</span>
</label>
<label class="block">
<span class="text-sm text-text-primary">Search each album again after</span>
<span class="block text-xs text-text-secondary">
Hours between searches for an album still wanted. 0 switches the schedule off; Search on the
Wanted page still works.
</span>
<span class="mt-1 flex items-center gap-2">
<input
type="number"
min="0"
max="720"
value={form.search_interval_hours}
oninput={(e) => edit({ search_interval_hours: Number(e.currentTarget.value) })}
class="w-24 rounded border border-border bg-background px-2 py-1 text-sm text-text-primary
focus-visible:outline-solid focus-visible:outline-2 focus-visible:outline-accent"
/>
<span class="text-sm text-text-secondary">hours</span>
</span>
</label>
<fieldset>
<legend class="text-sm text-text-primary">Qualities to take</legend>
<span class="block text-xs text-text-secondary">
The best of these is taken first. A release that names no format has its file list read to
find out.
</span>
<div class="mt-2 space-y-1.5">
{#each query.data.tiers as t (t.tier)}
<label class="flex items-start gap-2">
<input type="checkbox" checked={form.allowed.includes(t.tier)} onchange={() => toggle(t.tier)} class="mt-0.5" />
<span class="text-sm text-text-primary">
{t.label}
<span class="block text-xs text-text-secondary">{t.what}</span>
</span>
</label>
{/each}
</div>
</fieldset>
<p class="text-xs text-text-secondary" data-testid="last-pass">{lastPass(query.data)}</p>
<div class="flex justify-end">
<button
type="button"
onclick={save}
disabled={!dirty || saving || form.allowed.length === 0}
class="rounded-md bg-action-secondary px-4 py-2 text-sm text-action-fg hover:opacity-90 disabled:cursor-not-allowed disabled:opacity-50"
>
{saving ? 'Saving…' : 'Save'}
</button>
</div>
{/if}
</section>
@@ -0,0 +1,69 @@
import { afterEach, describe, expect, test, vi } from 'vitest';
import { fireEvent, render, screen, waitFor } from '@testing-library/svelte';
import { mockQuery } from '../../test-utils/query';
import type { SearchSettings } from '#lib/api/types.js';
vi.mock('#lib/api/wanted.js', () => ({
createSearchSettingsQuery: vi.fn(),
putSearchSettings: vi.fn()
}));
vi.mock('#lib/stores/toast.svelte.js', () => ({ pushToast: vi.fn() }));
import SearchSettingsCard from './SearchSettingsCard.svelte';
import { createSearchSettingsQuery, putSearchSettings } from '#lib/api/wanted.js';
const settings: SearchSettings = {
auto_grab: true,
search_interval_hours: 24,
allowed_qualities: ['lossless', 'lossy_high'],
tiers: [
{ tier: 'lossless', label: 'Lossless', what: 'FLAC, ALAC' },
{ tier: 'lossy_high', label: 'High quality lossy', what: 'MP3 320 or V0' },
{ tier: 'unknown', label: 'Unknown', what: 'Nothing says what format it is' }
],
last_pass_at: new Date(Date.now() - 5 * 60_000).toISOString(),
last_pass: { looked: 12, grabbed: 3, waiting: 2, nothing: 7, failed: 0, paced: true }
};
afterEach(() => vi.clearAllMocks());
function setup(s: SearchSettings = settings) {
vi.mocked(createSearchSettingsQuery).mockReturnValue(
mockQuery({ data: s }) as unknown as ReturnType<typeof createSearchSettingsQuery>
);
vi.mocked(putSearchSettings).mockImplementation(async (body) => ({ ...s, ...body }));
return render(SearchSettingsCard);
}
describe('SearchSettingsCard', () => {
test('shows the settings and what the last pass did', () => {
setup();
expect((screen.getByLabelText(/take the best release without asking/i) as HTMLInputElement).checked).toBe(true);
expect((screen.getByLabelText(/hours between searches/i) as HTMLInputElement).value).toBe('24');
expect((screen.getByLabelText(/^unknown/i) as HTMLInputElement).checked).toBe(false);
const last = screen.getByTestId('last-pass').textContent ?? '';
expect(last).toMatch(/searched 12, took 3, 2 waiting for you, 7 with nothing found\./);
expect(last).toMatch(/searches per hour/);
});
test('saves what changed', async () => {
setup();
const save = screen.getByRole('button', { name: /^save$/i }) as HTMLButtonElement;
expect(save.disabled).toBe(true);
await fireEvent.click(screen.getByLabelText(/take the best release without asking/i));
await fireEvent.click(screen.getByLabelText(/^unknown/i));
await fireEvent.click(save);
await waitFor(() =>
expect(putSearchSettings).toHaveBeenCalledWith({
auto_grab: false,
search_interval_hours: 24,
allowed_qualities: ['lossless', 'lossy_high', 'unknown']
})
);
});
test('before the first pass it says so', () => {
setup({ ...settings, last_pass_at: null, last_pass: {} });
expect(screen.getByTestId('last-pass').textContent).toMatch(/hasn’t run yet/);
});
});
@@ -0,0 +1,184 @@
<script lang="ts">
import { useQueryClient } from '@tanstack/svelte-query';
import {
createCandidatesQuery,
grabCandidate,
blockCandidate,
unblockRelease
} from '#lib/api/wanted.js';
import { qk } from '#lib/api/queries.js';
import { errMessage } from '#lib/api/errors.js';
import { pushToast } from '#lib/stores/toast.svelte.js';
import { relativeTime } from '#lib/utils/relativeTime.js';
import type { Candidate, CandidateVerdict, GrabResult } from '#lib/api/types.js';
// Every release a want's last search found (M502 #5521), with the verdict
// and the reasons behind it, so a person can see why nothing was taken and
// take something anyway, or set a release aside for good.
let { wantId, title }: { wantId: string; title: string } = $props();
const client = useQueryClient();
const store = $derived(createCandidatesQuery(wantId));
const query = $derived($store);
const items = $derived(query.data ?? []);
let busy = $state<string | null>(null);
const verdictLabel: Record<CandidateVerdict, string> = {
grab: 'Would take',
review: 'Needs you',
refused: 'Quality refused',
rejected: 'Not the album',
blocked: 'Set aside'
};
const verdictClass: Record<CandidateVerdict, string> = {
grab: 'bg-accent-tint text-accent-fg',
review: 'bg-surface-hover text-text-primary',
refused: 'bg-surface-hover text-text-secondary',
rejected: 'bg-surface-hover text-text-muted',
blocked: 'bg-surface-hover text-text-muted'
};
function size(bytes: number): string {
if (bytes <= 0) return '';
if (bytes >= 1e9) return `${(bytes / 1e9).toFixed(1)} GB`;
return `${Math.max(1, Math.round(bytes / 1e6))} MB`;
}
function meta(c: Candidate): string {
const parts = [c.format || 'format not stated', size(c.size_bytes)];
parts.push(c.seeders === 1 ? '1 seeder' : `${c.seeders} seeders`);
if (c.indexer) parts.push(c.folded_from.length ? `${c.indexer} +${c.folded_from.length}` : c.indexer);
return parts.filter(Boolean).join(' · ');
}
async function refresh() {
await Promise.all([
client.invalidateQueries({ queryKey: qk.wantCandidates(wantId) }),
client.invalidateQueries({ queryKey: ['adminWants'] })
]);
}
function grabbed(r: GrabResult): string {
if (r.added) return r.replaced ? 'Taken, replacing the download that was running.' : `Taken for ${title}.`;
if (r.kept) return r.reason ?? 'A download for this album is already running.';
if (r.gone) return 'The indexers no longer offer that release.';
if (r.unconfigured) {
return r.missing === 'download_client'
? 'No download client is set up. Add one under Integrations.'
: 'No indexer is set up. Add one under Integrations.';
}
if (r.refused) return r.reason ? `Not taken: ${r.reason}` : 'Not taken.';
return r.error ?? 'Not taken.';
}
async function onGrab(c: Candidate) {
busy = c.key;
try {
const r = await grabCandidate(wantId, c.key, c.verdict !== 'grab');
pushToast(grabbed(r), r.added ? 'info' : 'error');
await refresh();
} catch (e) {
pushToast(errMessage(e), 'error');
} finally {
busy = null;
}
}
async function onBlock(c: Candidate) {
busy = c.key;
try {
await blockCandidate(wantId, c.key);
pushToast('Set aside. No search will offer it again.');
await refresh();
} catch (e) {
pushToast(errMessage(e), 'error');
} finally {
busy = null;
}
}
async function onUnblock(c: Candidate) {
busy = c.key;
try {
await unblockRelease(c.info_hash);
pushToast('Allowed again. The next search judges it afresh.');
await refresh();
} catch (e) {
pushToast(errMessage(e), 'error');
} finally {
busy = null;
}
}
</script>
<div class="space-y-2" data-testid="want-candidates">
{#if query.isPending}
<p class="text-xs text-text-secondary">Loading what the last search found…</p>
{:else if query.isError}
<p class="text-xs text-error-fg">Couldn't load the releases.</p>
{:else if items.length === 0}
<p class="text-xs text-text-secondary">
Nothing kept from a search yet. Search now, or wait for the schedule.
</p>
{:else}
<p class="text-xs text-text-muted">
{items.length === 1 ? '1 release' : `${items.length} releases`} found {relativeTime(items[0].searched_at)}, best first.
</p>
<ul class="divide-y divide-border rounded-md border border-border">
{#each items as c (c.key)}
<li class="flex flex-wrap items-start gap-2 p-2" data-testid="candidate-row">
<div class="min-w-0 flex-1 space-y-0.5">
<div class="break-words text-xs text-text-primary">{c.name}</div>
<div class="text-xs text-text-muted">{meta(c)}</div>
<div class="flex flex-wrap items-center gap-1.5 text-xs">
<span class="rounded-full px-2 py-0.5 {verdictClass[c.verdict]}">
{c.downloading ? 'Downloading' : verdictLabel[c.verdict]}
</span>
<span class="text-text-secondary" data-testid="candidate-reason">{c.reason}</span>
</div>
</div>
<div class="flex shrink-0 gap-1.5">
{#if c.verdict === 'blocked'}
<button
type="button"
onclick={() => onUnblock(c)}
disabled={busy !== null}
aria-label="Allow {c.name} again"
class="rounded-md border border-border px-2 py-1 text-xs text-text-secondary hover:bg-surface-hover hover:text-text-primary disabled:opacity-50"
>
Allow again
</button>
{:else if !c.downloading}
<button
type="button"
onclick={() => onGrab(c)}
disabled={busy !== null}
aria-label="{c.verdict === 'grab' ? 'Grab' : 'Grab anyway'} {c.name}"
class="rounded-md px-2 py-1 text-xs disabled:opacity-50
{c.verdict === 'grab'
? 'bg-action-primary text-action-fg hover:opacity-90'
: 'border border-border text-text-secondary hover:bg-surface-hover hover:text-text-primary'}"
>
{busy === c.key ? 'Taking…' : c.verdict === 'grab' ? 'Grab' : 'Grab anyway'}
</button>
{#if c.info_hash}
<button
type="button"
onclick={() => onBlock(c)}
disabled={busy !== null}
aria-label="Never take {c.name}"
class="rounded-md px-2 py-1 text-xs text-text-muted hover:bg-surface-hover hover:text-text-primary disabled:opacity-50"
>
Never this one
</button>
{/if}
{/if}
</div>
</li>
{/each}
</ul>
{/if}
</div>
@@ -0,0 +1,90 @@
import { afterEach, describe, expect, test, vi } from 'vitest';
import { fireEvent, render, screen, waitFor } from '@testing-library/svelte';
import { mockQuery } from '../../test-utils/query';
import type { Candidate } from '#lib/api/types.js';
vi.mock('#lib/api/wanted.js', () => ({
createCandidatesQuery: vi.fn(),
grabCandidate: vi.fn().mockResolvedValue({ added: true, refused: false, gone: false, unconfigured: false, kept: false, replaced: false }),
blockCandidate: vi.fn().mockResolvedValue(undefined),
unblockRelease: vi.fn().mockResolvedValue(undefined)
}));
vi.mock('#lib/stores/toast.svelte.js', () => ({ pushToast: vi.fn() }));
import WantCandidates from './WantCandidates.svelte';
import { blockCandidate, createCandidatesQuery, grabCandidate, unblockRelease } from '#lib/api/wanted.js';
import { pushToast } from '#lib/stores/toast.svelte.js';
function cand(key: string, extra: Partial<Candidate> = {}): Candidate {
return {
key,
name: `Daft Punk - Random Access Memories (2013) [${key}]`,
info_hash: key,
indexer: 'Knaben',
size_bytes: 450_000_000,
seeders: 12,
files_count: 13,
format: 'FLAC',
quality: 'lossless',
year: 2013,
verdict: 'grab',
reason: 'The album, FLAC.',
reasons: [],
folded_from: [],
searched_at: new Date().toISOString(),
downloading: false,
...extra
};
}
afterEach(() => vi.clearAllMocks());
function setup(items: Candidate[]) {
vi.mocked(createCandidatesQuery).mockReturnValue(
mockQuery({ data: items }) as unknown as ReturnType<typeof createCandidatesQuery>
);
return render(WantCandidates, { wantId: 'w1', title: 'Random Access Memories' });
}
describe('WantCandidates', () => {
test('every release shows its verdict and why', () => {
setup([
cand('aaaa'),
cand('bbbb', { verdict: 'rejected', reason: 'This is a drumless edition (“drumless”).' })
]);
expect(screen.getAllByTestId('candidate-row')).toHaveLength(2);
expect(screen.getByText('Would take')).toBeTruthy();
expect(screen.getByText('Not the album')).toBeTruthy();
expect(screen.getByText(/drumless edition/)).toBeTruthy();
expect(screen.getAllByText(/FLAC · 450 MB · 12 seeders · Knaben/)).toHaveLength(2);
});
test('a release the verdict would take is grabbed; any other is grabbed anyway', async () => {
setup([cand('aaaa'), cand('bbbb', { verdict: 'review', reason: 'Nobody is sharing it right now.' })]);
await fireEvent.click(screen.getByRole('button', { name: /^grab daft punk.*\[aaaa\]/i }));
await waitFor(() => expect(grabCandidate).toHaveBeenCalledWith('w1', 'aaaa', false));
await waitFor(() => expect(pushToast).toHaveBeenCalledWith('Taken for Random Access Memories.', 'info'));
await fireEvent.click(screen.getByRole('button', { name: /^grab anyway daft punk.*\[bbbb\]/i }));
await waitFor(() => expect(grabCandidate).toHaveBeenCalledWith('w1', 'bbbb', true));
});
test('a release can be set aside, and allowed again', async () => {
setup([cand('aaaa'), cand('bbbb', { verdict: 'blocked', reason: 'You set this release aside.' })]);
await fireEvent.click(screen.getByRole('button', { name: /never take .*\[aaaa\]/i }));
await waitFor(() => expect(blockCandidate).toHaveBeenCalledWith('w1', 'aaaa'));
await fireEvent.click(screen.getByRole('button', { name: /allow .*\[bbbb\] again/i }));
await waitFor(() => expect(unblockRelease).toHaveBeenCalledWith('bbbb'));
expect(screen.queryByRole('button', { name: /grab.*\[bbbb\]/i })).toBeNull();
});
test('the release downloading says so and offers nothing', () => {
setup([cand('aaaa', { downloading: true })]);
expect(screen.getByText('Downloading')).toBeTruthy();
expect(screen.queryByRole('button', { name: /grab/i })).toBeNull();
});
test('before any search, it says how to get one', () => {
setup([]);
expect(screen.getByText(/nothing kept from a search yet/i)).toBeTruthy();
});
});
+7 -1
View File
@@ -67,5 +67,11 @@
"profile_is_default": "The default profile can't be deleted. Make another one the default first.",
"want_not_found": "That's no longer on the wanted list. Refresh the page.",
"follow_not_found": "That artist isn't followed any more. Refresh the page.",
"invalid_profile": "That isn't a release-type profile."
"invalid_profile": "That isn't a release-type profile.",
"bad_interval": "Hours between searches must be between 0 and 720.",
"bad_quality": "Choose at least one quality to take.",
"candidate_not_found": "That release is no longer in the list. Search again.",
"no_infohash": "That release can't be set aside: its indexer gave no infohash to know it by.",
"want_held": "The library already holds that album.",
"block_not_found": "That release isn't set aside any more."
}
@@ -30,6 +30,7 @@
import AcoustIDCard from '#lib/components/AcoustIDCard.svelte';
import AcquisitionCard from '#lib/components/AcquisitionCard.svelte';
import ReleaseProfilesCard from '#lib/components/ReleaseProfilesCard.svelte';
import SearchSettingsCard from '#lib/components/SearchSettingsCard.svelte';
import NetworkSettingsCard from '#lib/components/NetworkSettingsCard.svelte';
import PublicAddressCard from '#lib/components/PublicAddressCard.svelte';
import NotificationEmailCard from '#lib/components/NotificationEmailCard.svelte';
@@ -424,6 +425,8 @@
<AcquisitionCard />
<SearchSettingsCard />
<ReleaseProfilesCard />
<!-- Lidarr panel -->
@@ -60,7 +60,22 @@ vi.mock('#lib/api/wanted.js', async () => {
),
createReleaseProfile: vi.fn(),
updateReleaseProfile: vi.fn(),
deleteReleaseProfile: vi.fn()
deleteReleaseProfile: vi.fn(),
// SearchSettingsCard, settled the same way.
createSearchSettingsQuery: vi.fn(() =>
mockQuery({
data: {
auto_grab: true,
search_interval_hours: 24,
// Not "Lossless": a Lidarr quality profile on this page is named that.
allowed_qualities: ['lossy_high'],
tiers: [{ tier: 'lossy_high', label: 'High quality lossy', what: 'MP3 320' }],
last_pass_at: null,
last_pass: {}
}
})
),
putSearchSettings: vi.fn()
};
});
+134 -39
View File
@@ -1,7 +1,7 @@
<script lang="ts">
import { page } from '$app/state';
import { pageTitle } from '#lib/branding.js';
import { Album, Disc3, ListChecks, X } from 'lucide-svelte';
import { Album, ChevronDown, Disc3, ListChecks, Search, X } from 'lucide-svelte';
import { useQueryClient } from '@tanstack/svelte-query';
import {
createWantsQuery,
@@ -9,20 +9,25 @@
createReleaseProfilesQuery,
removeWant,
putFollow,
deleteFollow
deleteFollow,
searchWant
} from '#lib/api/wanted.js';
import { qk } from '#lib/api/queries.js';
import { errMessage } from '#lib/api/errors.js';
import { pushToast } from '#lib/stores/toast.svelte.js';
import { relativeTime } from '#lib/utils/relativeTime.js';
import ListContinuation from '#lib/components/ListContinuation.svelte';
import TabStrip from '#lib/components/TabStrip.svelte';
import type { Follow, Want, WantState } from '#lib/api/types.js';
import WantCandidates from '#lib/components/WantCandidates.svelte';
import type { Follow, SearchResult, Want, WantState } from '#lib/api/types.js';
// The wanted list (M502 #5520): every release group Minstrel is to get,
// whether a user requested it, a followed artist released it, or its files
// went missing. A want is held once the library has the album; the Held tab
// is that history. Following lists the artists whose new releases are
// wanted, each on a release-type profile that says which kinds of release.
// Each open want says what its last search found (#5521); Releases lists
// every candidate with its verdict, and Search runs one now.
type Tab = WantState | 'following';
const fromUrl = page.url.searchParams.get('tab');
@@ -52,6 +57,8 @@
]);
let busy = $state<string | null>(null);
let searching = $state<string | null>(null);
let open = $state<string | null>(null);
async function refresh() {
await Promise.all([
@@ -98,6 +105,62 @@
}
}
function searched(w: Want, r: SearchResult): string {
let said: string;
switch (r.outcome) {
case 'grabbed':
said = `Took ${r.grabbed ?? r.best_release}.`;
break;
case 'waiting':
said = `Found ${r.considered}; nothing taken. Releases shows why.`;
break;
case 'nothing':
said = r.considered ? `Found ${r.considered}, none of them ${w.title}.` : `Nothing found for ${w.title}.`;
break;
default:
said = r.error ?? 'The search failed.';
}
if (r.narrowed.length) said += ` ${r.narrowed.map((n) => n.label).join(', ')} didn't answer.`;
return said;
}
async function onSearch(w: Want) {
searching = w.id;
try {
const r = await searchWant(w.id);
pushToast(searched(w, r), r.outcome === 'error' ? 'error' : 'info');
await Promise.all([
client.invalidateQueries({ queryKey: ['adminWants'] }),
client.invalidateQueries({ queryKey: qk.wantCandidates(w.id) })
]);
} catch (e) {
pushToast(errMessage(e), 'error');
} finally {
searching = null;
}
}
// What the schedule last did about a want, in a line.
function status(w: Want): string {
if (w.grab) {
const who = w.grab.origin === 'operator' ? 'taken by you' : 'taken by the schedule';
return `Downloading ${w.grab.name}, ${who} ${relativeTime(w.grab.grabbed_at)}`;
}
const s = w.search;
if (!s) return 'Not searched yet';
const when = `Searched ${relativeTime(s.searched_at)}`;
switch (s.outcome) {
case 'nothing':
return s.considered ? `${when}: ${s.considered} found, none of them this album` : `${when}: nothing found`;
case 'waiting':
return `${when}: ${s.considered} found, nothing taken`;
case 'grabbed':
return `${when}: took ${s.best_release}`;
default:
return `${when}: ${s.error}`;
}
}
function why(w: Want): string {
if (w.reason === 'request') return w.requested_by ? `Requested by ${w.requested_by}` : 'Requested';
if (w.reason === 'follow') return 'New from a followed artist';
@@ -122,8 +185,10 @@
<h2 class="font-display text-2xl font-medium text-text-primary">Wanted</h2>
<p class="text-text-secondary">
Albums Minstrel is to get: ones users requested, new releases from the artists it follows,
and albums whose files went missing. An album leaves the list once the library holds it.
Minstrel doesn't search indexers for these yet; that comes next.
and albums whose files went missing. Minstrel searches your indexers for each one on a
schedule and hands the best release to your download client; searching and quality are set
under <a href="/admin/integrations" class="text-accent-fg hover:underline">Integrations</a>.
An album leaves the list once the library holds it.
</p>
</header>
@@ -216,43 +281,73 @@
{:else}
<ul class="divide-y divide-border rounded-lg border border-border bg-surface">
{#each items as w (w.id)}
<li class="flex items-center gap-3 p-3" data-testid="want-row">
<div class="relative flex h-14 w-14 shrink-0 items-center justify-center overflow-hidden rounded-md bg-surface-hover" aria-hidden="true">
<Album size={22} strokeWidth={1} class="text-text-muted" />
{#if w.image_url}
<img src={w.image_url} alt="" class="absolute inset-0 h-full w-full object-cover" loading="lazy" onerror={hideBroken} />
<li class="space-y-2 p-3" data-testid="want-row">
<div class="flex items-center gap-3">
<div class="relative flex h-14 w-14 shrink-0 items-center justify-center overflow-hidden rounded-md bg-surface-hover" aria-hidden="true">
<Album size={22} strokeWidth={1} class="text-text-muted" />
{#if w.image_url}
<img src={w.image_url} alt="" class="absolute inset-0 h-full w-full object-cover" loading="lazy" onerror={hideBroken} />
{/if}
</div>
<div class="min-w-0 flex-1 space-y-0.5">
<div class="truncate text-sm font-medium text-text-primary">
{#if w.album_id && w.held_at}
<a href="/albums/{w.album_id}" class="hover:underline">{w.title}</a>
{:else}
{w.title}
{/if}
</div>
<div class="truncate text-xs text-text-secondary">
{#if w.artist_id}
<a href="/artists/{w.artist_id}" class="hover:text-text-primary hover:underline">{w.artist_name}</a>
{:else}
{w.artist_name}
{/if}
{#if kind(w)}<span class="text-text-muted"> · {kind(w)}</span>{/if}
</div>
<div class="text-xs text-text-muted" data-testid="want-why">
{why(w)} · {w.held_at ? `held ${relativeTime(w.held_at)}` : `wanted ${relativeTime(w.wanted_at)}`}
</div>
{#if !w.held_at}
<div class="truncate text-xs text-text-secondary" data-testid="want-search">{status(w)}</div>
{/if}
</div>
{#if !w.held_at}
<div class="flex shrink-0 flex-wrap justify-end gap-1.5">
<button
type="button"
onclick={() => onSearch(w)}
disabled={searching !== null || !!w.grab}
aria-label="Search for {w.title} now"
class="inline-flex items-center gap-1 rounded-md border border-border px-2.5 py-1 text-xs text-text-secondary hover:bg-surface-hover hover:text-text-primary disabled:opacity-50"
>
<Search size={12} strokeWidth={2} /> {searching === w.id ? 'Searching…' : 'Search'}
</button>
<button
type="button"
onclick={() => (open = open === w.id ? null : w.id)}
aria-expanded={open === w.id}
aria-label="Releases found for {w.title}"
class="inline-flex items-center gap-1 rounded-md border border-border px-2.5 py-1 text-xs text-text-secondary hover:bg-surface-hover hover:text-text-primary"
>
Releases <ChevronDown size={12} strokeWidth={2} class={open === w.id ? 'rotate-180' : ''} />
</button>
<button
type="button"
onclick={() => onRemove(w)}
disabled={busy === w.id}
aria-label="Stop wanting {w.title}"
class="inline-flex items-center gap-1 rounded-md border border-border px-2.5 py-1 text-xs text-text-secondary hover:bg-surface-hover hover:text-text-primary disabled:opacity-50"
>
<X size={12} strokeWidth={2} /> Remove
</button>
</div>
{/if}
</div>
<div class="min-w-0 flex-1 space-y-0.5">
<div class="truncate text-sm font-medium text-text-primary">
{#if w.album_id && w.held_at}
<a href="/albums/{w.album_id}" class="hover:underline">{w.title}</a>
{:else}
{w.title}
{/if}
{#if open === w.id && !w.held_at}
<div class="sm:pl-[4.25rem]">
<WantCandidates wantId={w.id} title={w.title} />
</div>
<div class="truncate text-xs text-text-secondary">
{#if w.artist_id}
<a href="/artists/{w.artist_id}" class="hover:text-text-primary hover:underline">{w.artist_name}</a>
{:else}
{w.artist_name}
{/if}
{#if kind(w)}<span class="text-text-muted"> · {kind(w)}</span>{/if}
</div>
<div class="text-xs text-text-muted" data-testid="want-why">
{why(w)} · {w.held_at ? `held ${relativeTime(w.held_at)}` : `wanted ${relativeTime(w.wanted_at)}`}
</div>
</div>
{#if !w.held_at}
<button
type="button"
onclick={() => onRemove(w)}
disabled={busy === w.id}
aria-label="Stop wanting {w.title}"
class="inline-flex shrink-0 items-center gap-1 rounded-md border border-border px-2.5 py-1 text-xs text-text-secondary hover:bg-surface-hover hover:text-text-primary disabled:opacity-50"
>
<X size={12} strokeWidth={2} /> Remove
</button>
{/if}
</li>
{/each}
+51 -2
View File
@@ -9,17 +9,28 @@ vi.mock('#lib/api/wanted.js', () => ({
createReleaseProfilesQuery: vi.fn(),
removeWant: vi.fn().mockResolvedValue(undefined),
putFollow: vi.fn().mockResolvedValue({ following: true }),
deleteFollow: vi.fn().mockResolvedValue(undefined)
deleteFollow: vi.fn().mockResolvedValue(undefined),
searchWant: vi.fn().mockResolvedValue({
outcome: 'nothing', considered: 0, best_release: '', best_verdict: '', narrowed: []
}),
createCandidatesQuery: vi.fn(),
grabCandidate: vi.fn(),
blockCandidate: vi.fn(),
unblockRelease: vi.fn()
}));
vi.mock('#lib/stores/toast.svelte.js', () => ({ pushToast: vi.fn() }));
import AdminWantedPage from './+page.svelte';
import { pushToast } from '#lib/stores/toast.svelte.js';
import {
createCandidatesQuery,
createFollowsQuery,
createReleaseProfilesQuery,
createWantsQuery,
deleteFollow,
putFollow,
removeWant
removeWant,
searchWant
} from '#lib/api/wanted.js';
const DAY = 24 * 3_600_000;
@@ -87,6 +98,9 @@ function setup(items: Want[] = [want('w1'), want('w2', { reason: 'request', requ
vi.mocked(createReleaseProfilesQuery).mockReturnValue(
mockQuery({ data: profiles }) as unknown as ReturnType<typeof createReleaseProfilesQuery>
);
vi.mocked(createCandidatesQuery).mockReturnValue(
mockQuery({ data: [] }) as unknown as ReturnType<typeof createCandidatesQuery>
);
return render(AdminWantedPage);
}
@@ -142,4 +156,39 @@ describe('/admin/wanted', () => {
setup([]);
expect(screen.getByText(/nothing is wanted/i)).toBeTruthy();
});
test('each open want says what its last search found', () => {
setup([
want('w1'),
want('w2', {
title: 'Parklife',
search: { searched_at: ago(3_600_000), outcome: 'nothing', considered: 4, best_release: 'x', best_verdict: 'rejected', error: '' }
}),
want('w3', {
title: 'Blur',
grab: { name: 'Blur - Blur (1997) [FLAC]', grabbed_at: ago(60_000), origin: 'schedule', quality: 'lossless' }
})
]);
const lines = screen.getAllByTestId('want-search').map((e) => e.textContent ?? '');
expect(lines[0]).toMatch(/not searched yet/i);
expect(lines[1]).toMatch(/4 found, none of them this album/i);
expect(lines[2]).toMatch(/downloading blur - blur \(1997\) \[flac\], taken by the schedule/i);
});
test('Search runs one now and says what came of it', async () => {
setup();
await fireEvent.click(screen.getByRole('button', { name: /search for parklife now/i }));
await waitFor(() => expect(searchWant).toHaveBeenCalledWith('w2'));
await waitFor(() => expect(pushToast).toHaveBeenCalledWith('Nothing found for Parklife.', 'info'));
});
test('Releases opens what the last search found', async () => {
setup();
expect(screen.queryByTestId('want-candidates')).toBeNull();
const toggle = screen.getByRole('button', { name: /releases found for parklife/i });
await fireEvent.click(toggle);
expect(toggle.getAttribute('aria-expanded')).toBe('true');
expect(screen.getByTestId('want-candidates')).toBeTruthy();
expect(createCandidatesQuery).toHaveBeenCalledWith('w2');
});
});