diff --git a/cmd/minstrel/main.go b/cmd/minstrel/main.go index 49afe10c..ef6a0e9d 100644 --- a/cmd/minstrel/main.go +++ b/cmd/minstrel/main.go @@ -322,6 +322,12 @@ func run() error { // daily for new releases and marks wants held once the library has them. go wanted.New(pool, cat, logger.With("component", "wanted")).Run(ctx) + // Search and grab (M502 #5521): every ten minutes, searches the + // configured indexers for wants due, paced by each source's + // searches-per-hour, and hands the best release to the download client + // when auto-grab is on. + go wanted.NewSearcher(pool, cat, logger.With("component", "search")).Run(ctx) + // Duplicate resolver (M498, M502): classifies the sweep's groups and, with // the operator's auto-resolve on, merges the copies that are safe to merge. // Started after SetNotifier so its summaries reach admins. See diff --git a/internal/acquire/build.go b/internal/acquire/build.go new file mode 100644 index 00000000..882d528c --- /dev/null +++ b/internal/acquire/build.go @@ -0,0 +1,45 @@ +package acquire + +import ( + "context" + "fmt" + + "github.com/jackc/pgx/v5/pgxpool" + + "git.fabledsword.com/bvandeusen/minstrel/internal/db/dbq" + "git.fabledsword.com/bvandeusen/minstrel/internal/release" +) + +// Build makes a Service from the settings as they stand: every enabled +// search source behind one paced Fanout, the first enabled download client, +// and the quality profile. Built afresh for each pass and each request, so a +// changed setting takes effect on the next one. +func Build(ctx context.Context, pool *pgxpool.Pool) (*Service, dbq.AcquisitionSetting, error) { + q := dbq.New(pool) + settings, err := q.GetAcquisitionSettings(ctx) + if err != nil { + return nil, settings, fmt.Errorf("acquire: read settings: %w", err) + } + sources, err := Sources(ctx, pool) + if err != nil { + return nil, settings, fmt.Errorf("acquire: read search sources: %w", err) + } + clients, err := q.ListUsableAcquisitionServices(ctx, JobDownload) + if err != nil { + return nil, settings, fmt.Errorf("acquire: read download clients: %w", err) + } + s := &Service{Profile: release.NewProfile(settings.AllowedQualities), Memory: NewMemory(pool)} + fan := &Fanout{Sources: sources, Pace: SearchPace} + if len(sources) > 0 { + s.Search = fan + } + if len(clients) > 0 { + row := clients[0] + client, err := DownloadClient(row) + if err != nil { + return nil, settings, fmt.Errorf("acquire: download client %s: %w", row.Code, err) + } + s.Grabber = &Downloader{Client: client, Code: row.Code, Landing: row.LocalPath, Fetch: fan} + } + return s, settings, nil +} diff --git a/internal/acquire/fanout.go b/internal/acquire/fanout.go new file mode 100644 index 00000000..3d0594b3 --- /dev/null +++ b/internal/acquire/fanout.go @@ -0,0 +1,165 @@ +package acquire + +import ( + "context" + "errors" + "fmt" + "strings" + "sync" + + "git.fabledsword.com/bvandeusen/minstrel/internal/indexer" +) + +// Searching every source at once (M502 #5521), ported from Fabled Librarian +// (#4182). A source that is down narrows a search rather than failing it: the +// results come back with the list of who did not answer. + +// SourceFailure is one source that could not be reached. +type SourceFailure struct { + Code string + Label string + Err error +} + +// PartialError is a search that reached some sources and not others. It is +// returned alongside the results, not instead of them. +type PartialError struct { + Failures []SourceFailure + // Answered is how many sources did answer. Zero means the search failed. + Answered int +} + +func (e *PartialError) Error() string { + if e == nil || len(e.Failures) == 0 { + return "some sources did not answer" + } + parts := make([]string, 0, len(e.Failures)) + for _, f := range e.Failures { + parts = append(parts, fmt.Sprintf("%s: %s", f.Label, indexer.FirstLine(f.Err.Error()))) + } + return "some sources did not answer — " + strings.Join(parts, "; ") +} + +// Unwrap lets errors.Is find indexer.ErrUnreachable through a partial failure. +func (e *PartialError) Unwrap() []error { + if e == nil { + return nil + } + out := make([]error, 0, len(e.Failures)) + for _, f := range e.Failures { + out = append(out, f.Err) + } + return out +} + +// Fanout searches every source and merges what comes back. It is both the +// Searcher and the Fetcher, routing a .torrent fetch to the source that +// offered the result. +type Fanout struct { + // Sources in code order: the fold keeps a duplicate's first arrival, so + // the order decides which source a grab is fetched through. + Sources []Source + // Pace spaces each source's searches. Nil paces nothing. + Pace *Pacer +} + +// Search asks every source concurrently. Each client bounds its own call by +// its configured timeout, so there is no shared deadline here. +func (f *Fanout) Search(ctx context.Context, query indexer.Query) ([]indexer.Result, error) { + if f == nil || len(f.Sources) == 0 { + return nil, indexer.ErrUnconfigured + } + found := make([][]indexer.Result, len(f.Sources)) + errs := make([]error, len(f.Sources)) + var wg sync.WaitGroup + for i, src := range f.Sources { + if src.Search == nil { + continue + } + f.Pace.spend(src) + wg.Add(1) + go func(i int, src Source) { + defer wg.Done() + found[i], errs[i] = src.Search.Search(ctx, query) + }(i, src) + } + wg.Wait() + + out := make([]indexer.Result, 0, 32) + var failures []SourceFailure + answered, unconfigured := 0, 0 + for i, src := range f.Sources { + if src.Search == nil { + continue + } + if errors.Is(errs[i], indexer.ErrUnconfigured) { + unconfigured++ + continue + } + if rl, ok := indexer.AsRateLimited(errs[i]); ok { + f.Pace.hold(src.Code, rl) + } + if errs[i] != nil { + failures = append(failures, SourceFailure{Code: src.Code, Label: src.Label, Err: errs[i]}) + continue + } + answered++ + for j := range found[i] { + if found[i][j].Source == "" { + found[i][j].Source = src.Code + } + } + out = append(out, found[i]...) + } + switch { + case answered == 0 && len(failures) > 0: + return nil, &PartialError{Failures: failures} + case len(failures) > 0: + return out, &PartialError{Failures: failures, Answered: answered} + case answered == 0 && unconfigured > 0: + return nil, indexer.ErrUnconfigured + } + return out, nil +} + +// Download fetches a result's .torrent from the source that offered it: a +// Prowlarr link carries Prowlarr's key and only Prowlarr can redeem it. +func (f *Fanout) Download(ctx context.Context, r indexer.Result) ([]byte, error) { + if f == nil { + return nil, indexer.ErrUnconfigured + } + for _, src := range f.Sources { + if src.Code != r.Source { + continue + } + if src.Fetch == nil { + return nil, fmt.Errorf("%w: %s cannot fetch torrent files", indexer.ErrUnconfigured, src.Label) + } + return src.Fetch.Download(ctx, r) + } + if r.Source == "" { + return nil, fmt.Errorf("%w: this result does not say which source it came from", indexer.ErrUnconfigured) + } + return nil, fmt.Errorf("%w: %q is no longer a configured source", indexer.ErrUnconfigured, r.Source) +} + +// Narrowing is one source that did not answer, as a page shows it. +type Narrowing struct { + Source string `json:"source"` + Label string `json:"label"` + Reason string `json:"reason"` +} + +// narrowings reads a PartialError out of err, and whether any source +// answered. Nil when err is not a partial failure. +func narrowings(err error) ([]Narrowing, bool) { + var partial *PartialError + if !errors.As(err, &partial) || len(partial.Failures) == 0 { + return nil, false + } + out := make([]Narrowing, 0, len(partial.Failures)) + for _, f := range partial.Failures { + out = append(out, Narrowing{Source: f.Code, Label: f.Label, Reason: indexer.FirstLine(f.Err.Error())}) + } + return out, partial.Answered > 0 +} diff --git a/internal/acquire/grab.go b/internal/acquire/grab.go new file mode 100644 index 00000000..6186ebd9 --- /dev/null +++ b/internal/acquire/grab.go @@ -0,0 +1,307 @@ +package acquire + +import ( + "context" + "errors" + "path" + "strings" + "sync" + + "github.com/jackc/pgx/v5/pgtype" + + "git.fabledsword.com/bvandeusen/minstrel/internal/db/dbq" + "git.fabledsword.com/bvandeusen/minstrel/internal/downloader" + "git.fabledsword.com/bvandeusen/minstrel/internal/indexer" + "git.fabledsword.com/bvandeusen/minstrel/internal/release" +) + +// Taking a release (M502 #5521), ported from Fabled Librarian's grab.go and +// rivals.go. One live grab per want: a second is tie-broken against the +// download already running (FL #4470). + +// Tag goes on every torrent Minstrel adds, so the operator can tell them +// from their own in the client. +const Tag = "minstrel" + +// MissingClient is Taken.Missing when no download client is set up. +const MissingClient = "download_client" + +// nearlyDone is how far along a running download stays whatever would +// replace it. +const nearlyDone = 0.9 + +// Downloader is the grab half of a Service. +type Downloader struct { + Client downloader.Driver + // Code is the client's acquisition_services row. + Code string + // Landing is the client's downloads folder as Minstrel sees it; each + // grab lands in a folder of its own under it. + Landing string + // Fetch gets a result's .torrent from the source that offered it. + Fetch Fetcher +} + +// Origin says who took a release. +type Origin struct { + By string // OriginSchedule or OriginOperator + User pgtype.UUID +} + +// Contest is how a grab fared against the download already running. +type Contest struct { + // Kept: the running one stays and this grab was not made. Replaced: + // this grab took its place. + Kept, Replaced bool + Reason string + Rival dbq.Grab +} + +// Taken is what came of a grab. +type Taken struct { + Chosen *Candidate + // Added is the only field that means a download started. + Added bool + SavePath string + Grab dbq.Grab + // Gone: the search no longer offers the release. + Gone bool + // Refused: not taken; the reason is on Chosen. + Refused bool + // Unconfigured: Missing names what is not set up ("" for search). + Unconfigured bool + Missing string + Contest *Contest + Error string +} + +// Take searches again and grabs the candidate named by key: a person's +// grab. Searching again keeps the indexer's download link (which can carry +// its key) on the server, and finds a release gone since it was shown. +// force takes what the verdict would not. +func (s *Service) Take(ctx context.Context, want Want, key string, force bool, origin Origin) Taken { + if s.Grabber == nil { + return Taken{Unconfigured: true, Missing: MissingClient} + } + found := s.For(ctx, want) + switch { + case found.Unconfigured: + return Taken{Unconfigured: true} + case found.Error != "": + return Taken{Error: found.Error} + } + chosen := pick(found.Candidates, key) + if chosen == nil { + return Taken{Gone: true} + } + return s.Grab(ctx, want, chosen, found.Candidates, force, origin) +} + +// Grab hands a judged candidate to the download client. The schedule calls +// it with the candidate it has just judged; Take calls it after searching +// again. +func (s *Service) Grab(ctx context.Context, want Want, chosen *Candidate, all []Candidate, force bool, origin Origin) Taken { + out := Taken{Chosen: chosen} + if s.Grabber == nil || s.Grabber.Client == nil { + out.Unconfigured, out.Missing = true, MissingClient + return out + } + if chosen.Verdict == Blocked || (chosen.Verdict != Grab && !force) { + out.Refused = true + return out + } + + unlock := lockWant(want.ID) + defer unlock() + contest := s.contest(ctx, want, chosen, all, origin) + if contest != nil && contest.Kept { + out.Contest = contest + return out + } + + add := downloader.Add{Magnet: chosen.Source.Magnet, Tags: []string{Tag}} + hash := chosen.InfoHash + if add.Magnet == "" { + if s.Grabber.Fetch == nil { + out.Error = "This release is a .torrent link, and nothing is set up to fetch it." + return out + } + body, err := s.Grabber.Fetch.Download(ctx, chosen.Source) + if magnet, ok := indexer.AsMagnet(err); ok { + add.Magnet, err = magnet, nil + } + if err != nil { + out.Error = indexer.FirstLine(err.Error()) + return out + } + if add.Magnet == "" { + add.Torrent = body + if meta, err := release.ParseTorrent(body); err == nil { + if hash == "" { + hash = meta.InfoHash + } + if refused := s.judgeFiles(ctx, want, chosen, hash, meta.Files, force); refused { + out.Refused = true + return out + } + } + } + } + if hash == "" && add.Magnet != "" { + hash = indexer.MagnetHash(add.Magnet) + } + add.SavePath = landingFor(s.Grabber.Landing, hash, chosen.Name) + + if err := s.Grabber.Client.AddTorrent(ctx, add); err != nil && !errors.Is(err, downloader.ErrDuplicate) { + if errors.Is(err, downloader.ErrUnconfigured) { + out.Unconfigured, out.Missing = true, MissingClient + } else { + out.Error = indexer.FirstLine(err.Error()) + } + return out + } + out.Added, out.SavePath = true, add.SavePath + if s.Memory == nil { + return out + } + + var replaced pgtype.UUID + why := "" + if contest != nil && contest.Replaced { + out.Contest = contest + replaced, why = contest.Rival.ID, "Replaced by "+chosen.Name+": "+contest.Reason+"." + } + g, err := s.Memory.Grabbed(ctx, dbq.InsertGrabParams{ + WantID: want.ID, InfoHash: hash, Name: chosen.Name, Key: Key(*chosen), Indexer: chosen.Indexer, + Source: chosen.Source.Source, Client: s.Grabber.Code, SavePath: add.SavePath, + Quality: string(chosen.Tier), Origin: origin.By, Forced: force, GrabbedBy: origin.User, + }, replaced, why) + if err != nil { + out.Error = "The download started, but could not be recorded: " + indexer.FirstLine(err.Error()) + return out + } + out.Grab = g + // Removed only after the new download was added and recorded, so a + // failure never leaves the want with nothing running. + if replaced.Valid && contest.Rival.InfoHash != "" { + if err := s.Grabber.Client.Delete(ctx, true, contest.Rival.InfoHash); err != nil { + contest.Reason += ". The download it replaced could not be removed from the client: " + indexer.FirstLine(err.Error()) + } + } + return out +} + +// judgeFiles judges a chosen release again on its .torrent's file list, and +// sets it aside when the files say it is not the album, so no later search +// offers it again. force still takes it. +func (s *Service) judgeFiles(ctx context.Context, want Want, chosen *Candidate, hash string, files []release.File, force bool) bool { + chosen.Files = files + if chosen.FilesCount == 0 { + chosen.FilesCount = len(files) + } + veto, reason, _ := release.JudgeFiles(want.Want, files) + if veto == "" || force { + return false + } + set(chosen, Rejected, Hard, veto, reason) + if s.Memory != nil && hash != "" { + _ = s.Memory.Block(ctx, dbq.BlockReleaseParams{ + InfoHash: hash, Name: chosen.Name, Reason: BlockFiles, Detail: reason, WantID: want.ID, + }) + } + return true +} + +var wantLocks sync.Map // want id → *sync.Mutex + +// lockWant settles two grabs for one want one after the other. The +// one-live-grab index is the backstop for anything outside this process. +func lockWant(id pgtype.UUID) func() { + if !id.Valid { + return func() {} + } + m, _ := wantLocks.LoadOrStore(id.Bytes, &sync.Mutex{}) + mu := m.(*sync.Mutex) + mu.Lock() + return mu.Unlock +} + +// contest settles a grab against the download running for its want: a +// person's grab beats the schedule's; between two of a kind the better +// candidate wins; a tie, a download nearly done, or one that cannot be +// removed stays. Nil when nothing is running. +func (s *Service) contest(ctx context.Context, want Want, chosen *Candidate, all []Candidate, origin Origin) *Contest { + if s.Memory == nil || !want.ID.Valid { + return nil + } + running, ok, err := s.Memory.LiveGrab(ctx, want.ID) + if err != nil || !ok { + return nil + } + c := &Contest{Rival: running} + if running.InfoHash != "" && running.InfoHash == chosen.InfoHash { + c.Kept, c.Reason = true, "That release is already downloading for this album." + return c + } + switch { + case origin.By == OriginOperator && running.Origin == OriginSchedule: + c.Reason = "you chose it over the one the schedule started" + case origin.By == OriginSchedule && running.Origin == OriginOperator: + c.Kept, c.Reason = true, "A download you started for this album is already running." + return c + default: + var theirs *Candidate + for i := range all { + if (running.InfoHash != "" && all[i].InfoHash == running.InfoHash) || Key(all[i]) == running.Key { + theirs = &all[i] + break + } + } + if theirs == nil || !better(*chosen, *theirs) { + c.Kept, c.Reason = true, "A download for this album is already running, and this release is no better." + return c + } + c.Reason = "it ranks above the one that was running" + } + if running.InfoHash == "" { + c.Kept, c.Reason = true, "A download for this album is already running, and it cannot be identified to replace it." + return c + } + if ts, err := s.Grabber.Client.Torrents(ctx, running.InfoHash); err == nil && len(ts) > 0 && ts[0].Progress >= nearlyDone { + c.Kept, c.Reason = true, "A download for this album is already running and nearly done." + return c + } + c.Replaced = true + return c +} + +// landingFor is the folder a download lands in: one per torrent, named by +// its infohash, so two grabs never unpack into the same place. +func landingFor(root, hash, name string) string { + dir := hash + if dir == "" { + dir = slug(name) + } + if dir == "" { + dir = "unnamed" + } + return path.Join(root, dir) +} + +// slug keeps a folder name to characters every filesystem in the chain +// takes. +func slug(s string) string { + var b strings.Builder + for _, r := range strings.ToLower(strings.TrimSpace(s)) { + if b.Len() >= 80 { + break + } + switch { + case r >= 'a' && r <= 'z', r >= '0' && r <= '9': + b.WriteRune(r) + case r == '-', r == '_', r == ' ', r == '.': + b.WriteRune('-') + } + } + return strings.Trim(b.String(), "-") +} diff --git a/internal/acquire/memory.go b/internal/acquire/memory.go new file mode 100644 index 00000000..a9b0354a --- /dev/null +++ b/internal/acquire/memory.go @@ -0,0 +1,168 @@ +package acquire + +import ( + "context" + "encoding/json" + "errors" + "fmt" + + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgtype" + "github.com/jackc/pgx/v5/pgxpool" + + "git.fabledsword.com/bvandeusen/minstrel/internal/db/dbq" + "git.fabledsword.com/bvandeusen/minstrel/internal/indexer" + "git.fabledsword.com/bvandeusen/minstrel/internal/release" +) + +// Block reasons, matching release_blocks_reason_check. +const ( + BlockOperator = "operator" + BlockFiles = "files" +) + +// Grab origins, matching grabs_origin_check. +const ( + OriginSchedule = "schedule" + OriginOperator = "operator" +) + +// Memory is what searches and grabs leave behind: every candidate of a +// want's last search, the grabs, and the releases set aside. +type Memory struct { + pool *pgxpool.Pool +} + +// NewMemory returns a Memory over pool. +func NewMemory(pool *pgxpool.Pool) *Memory { return &Memory{pool: pool} } + +// Keep replaces a want's candidates with this search's, in order. +func (m *Memory) Keep(ctx context.Context, wantID pgtype.UUID, cands []Candidate) error { + return pgx.BeginFunc(ctx, m.pool, func(tx pgx.Tx) error { + q := dbq.New(tx) + if err := q.DeleteWantCandidates(ctx, wantID); err != nil { + return err + } + for i, c := range cands { + reasons, err := json.Marshal(c.Reasons) + if err != nil { + return err + } + if c.Reasons == nil { + reasons = []byte("[]") + } + folded := c.FoldedFrom + if folded == nil { + folded = []string{} + } + if err := q.InsertWantCandidate(ctx, dbq.InsertWantCandidateParams{ + WantID: wantID, Key: Key(c), Name: c.Name, InfoHash: c.InfoHash, Indexer: c.Indexer, + Source: c.Source.Source, SizeBytes: c.SizeBytes, Seeders: int32(c.Seeders), + FilesCount: int32(c.FilesCount), Format: release.Describe(c.Release), Quality: string(c.Tier), + Year: int32(c.Year), Verdict: string(c.Verdict), Reason: c.Reason, Reasons: reasons, + FoldedFrom: folded, Rank: int32(i), + }); err != nil { + return err + } + } + return nil + }) +} + +// blocked is every release set aside, by infohash, with the sentence a +// candidate shows. A failed read blocks nothing: offering a release set +// aside costs one grab a person can undo, while blocking everything on a +// database blip would hide every result. +func (m *Memory) blocked(ctx context.Context) map[string]string { + if m == nil { + return nil + } + rows, err := dbq.New(m.pool).ListReleaseBlocks(ctx) + if err != nil { + return nil + } + out := make(map[string]string, len(rows)) + for _, r := range rows { + out[indexer.NormalizeHash(r.InfoHash)] = BlockSentence(r.Reason, r.Detail) + } + return out +} + +// BlockSentence is what a blocked candidate says. +func BlockSentence(reason, detail string) string { + if reason == BlockFiles { + if detail != "" { + return "Set aside for what its files hold: " + detail + } + return "Set aside for what its files hold." + } + return "You set this release aside." +} + +// Block sets a release aside and marks every kept candidate of it blocked. +func (m *Memory) Block(ctx context.Context, p dbq.BlockReleaseParams) error { + p.InfoHash = indexer.NormalizeHash(p.InfoHash) + if p.InfoHash == "" { + return errors.New("a release without an infohash cannot be set aside") + } + return pgx.BeginFunc(ctx, m.pool, func(tx pgx.Tx) error { + q := dbq.New(tx) + if err := q.BlockRelease(ctx, p); err != nil { + return err + } + return q.MarkCandidatesBlocked(ctx, dbq.MarkCandidatesBlockedParams{ + InfoHash: p.InfoHash, Reason: BlockSentence(p.Reason, p.Detail), + }) + }) +} + +// Unblock lets a release be taken again. Its kept candidates wait for a +// person until the next search judges them afresh. +func (m *Memory) Unblock(ctx context.Context, hash string) (bool, error) { + hash = indexer.NormalizeHash(hash) + var n int64 + err := pgx.BeginFunc(ctx, m.pool, func(tx pgx.Tx) error { + q := dbq.New(tx) + var err error + if n, err = q.UnblockRelease(ctx, hash); err != nil { + return err + } + return q.MarkCandidatesUnblocked(ctx, dbq.MarkCandidatesUnblockedParams{ + InfoHash: hash, Reason: "Allowed again. The next search judges it afresh.", + }) + }) + return n > 0, err +} + +// LiveGrab is the download running for a want, if there is one. +func (m *Memory) LiveGrab(ctx context.Context, wantID pgtype.UUID) (dbq.Grab, bool, error) { + g, err := dbq.New(m.pool).GetLiveGrab(ctx, wantID) + if errors.Is(err, pgx.ErrNoRows) { + return g, false, nil + } + if err != nil { + return g, false, err + } + return g, true, nil +} + +// Grabbed records a download started, ending the grab it replaced first so +// the one-live-grab index holds. +func (m *Memory) Grabbed(ctx context.Context, p dbq.InsertGrabParams, replaced pgtype.UUID, why string) (dbq.Grab, error) { + var g dbq.Grab + err := pgx.BeginFunc(ctx, m.pool, func(tx pgx.Tx) error { + q := dbq.New(tx) + if replaced.Valid { + if err := q.EndGrab(ctx, dbq.EndGrabParams{ID: replaced, EndReason: "superseded", EndDetail: why}); err != nil { + return err + } + } + var err error + g, err = q.InsertGrab(ctx, p) + return err + }) + if err != nil { + return g, fmt.Errorf("record grab: %w", err) + } + return g, nil +} diff --git a/internal/acquire/pace.go b/internal/acquire/pace.go new file mode 100644 index 00000000..53900e85 --- /dev/null +++ b/internal/acquire/pace.go @@ -0,0 +1,145 @@ +package acquire + +import ( + "context" + "sync" + "time" + + "git.fabledsword.com/bvandeusen/minstrel/internal/indexer" +) + +// SearchPace is the one Pacer every search shares, ported from Fabled +// Librarian (#4448). Process-wide because a Fanout is built fresh for each +// search; a Pacer per Fanout would start empty and pace nothing. +var SearchPace = &Pacer{} + +// holdWithoutRetryAfter is how long a source that answered 429 without saying +// for how long is left alone: one schedule tick. +const holdWithoutRetryAfter = 10 * time.Minute + +// Pacer spaces the searches sent to each source by its searches_per_hour: a +// spacing, not a burst bucket, because Prowlarr answers 429 to a few requests +// in quick succession (FL #4304). Every search spends, the schedule's and a +// person's alike; only the schedule waits. +type Pacer struct { + // Now and Sleep are the clock, replaceable in tests. + Now func() time.Time + Sleep func(context.Context, time.Duration) error + + mu sync.Mutex + next map[string]time.Time +} + +func (p *Pacer) now() time.Time { + if p.Now != nil { + return p.Now() + } + return time.Now() +} + +func (p *Pacer) sleep(ctx context.Context, d time.Duration) error { + if p.Sleep != nil { + return p.Sleep(ctx, d) + } + t := time.NewTimer(d) + defer t.Stop() + select { + case <-t.C: + return nil + case <-ctx.Done(): + return ctx.Err() + } +} + +// gap is the spacing a source's number asks for; zero is unpaced. +func gap(perHour int) time.Duration { + if perHour <= 0 { + return 0 + } + return time.Hour / time.Duration(perHour) +} + +// readyAt is when every source may next be asked. +func (p *Pacer) readyAt(sources []Source) time.Time { + p.mu.Lock() + defer p.mu.Unlock() + var at time.Time + for _, s := range sources { + if s.Search == nil { + continue + } + if n := p.next[s.Code]; n.After(at) { + at = n + } + } + return at +} + +// Wait holds until every source may be asked, and answers false, without +// waiting, when that is after the deadline or ctx ends first. All of them, +// not any: a search while one is held back would be narrowed, and the want +// recorded as looked for. +func (p *Pacer) Wait(ctx context.Context, sources []Source, deadline time.Time) bool { + if p == nil { + return true + } + at := p.readyAt(sources) + if !deadline.IsZero() && at.After(deadline) { + return false + } + if d := at.Sub(p.now()); d > 0 { + return p.sleep(ctx, d) == nil + } + return ctx.Err() == nil +} + +// spend takes one search's slot from a source. +func (p *Pacer) spend(s Source) { + g := gap(s.PerHour) + if p == nil || g == 0 { + return + } + p.mu.Lock() + defer p.mu.Unlock() + if p.next == nil { + p.next = map[string]time.Time{} + } + at := p.next[s.Code] + if t := p.now(); at.Before(t) { + at = t + } + p.next[s.Code] = at.Add(g) +} + +// hold leaves a source alone after it answered 429, for as long as it asked, +// whatever its searches_per_hour says. +func (p *Pacer) hold(code string, rl *indexer.RateLimited) { + if p == nil { + return + } + d := rl.RetryAfter + if d <= 0 { + d = holdWithoutRetryAfter + } + p.mu.Lock() + defer p.mu.Unlock() + if p.next == nil { + p.next = map[string]time.Time{} + } + if until := p.now().Add(d); until.After(p.next[code]) { + p.next[code] = until + } +} + +// Ready holds until every source this service searches may be asked again, +// and answers false when that is past the deadline. The schedule calls it +// before each want; a person's search never does. +func (s *Service) Ready(ctx context.Context, deadline time.Time) bool { + if s == nil { + return ctx.Err() == nil + } + if f, ok := s.Search.(*Fanout); ok && f != nil { + return f.Pace.Wait(ctx, f.Sources, deadline) + } + return ctx.Err() == nil +} diff --git a/internal/acquire/search.go b/internal/acquire/search.go new file mode 100644 index 00000000..d3edf564 --- /dev/null +++ b/internal/acquire/search.go @@ -0,0 +1,390 @@ +package acquire + +import ( + "context" + "errors" + "sort" + "strings" + + "github.com/jackc/pgx/v5/pgtype" + + "git.fabledsword.com/bvandeusen/minstrel/internal/indexer" + "git.fabledsword.com/bvandeusen/minstrel/internal/release" +) + +// One search for one want, and what could be done about each result (M502 +// #5521), ported from Fabled Librarian's internal/acquire. internal/release +// says whether a name is the album and what quality it is; this package +// asks in the right order and says why in words a person can act on. + +// Verdict is what can be done about one result. +type Verdict string + +const ( + // Grab: the album, in a quality the profile takes. + Grab Verdict = "grab" + // Review: maybe the album; a person decides. + Review Verdict = "review" + // Refused: the album, in a quality the profile does not take. The only + // verdict that names a setting. + Refused Verdict = "refused" + // Rejected: not the album. + Rejected Verdict = "rejected" + // Blocked: a release set aside, by a person or by its files. + Blocked Verdict = "blocked" +) + +// order sorts verdicts: what can be taken first, what was given up on last. +var order = map[Verdict]int{Grab: 0, Review: 1, Refused: 2, Rejected: 3, Blocked: 4} + +// Reason kinds: a hard reason is a fact about the bytes, a soft one about the +// moment or a setting. +const ( + Hard = "hard" + Soft = "soft" +) + +// Reason codes beyond internal/release's vetoes. +const ( + CodeBlocked = "blocked" + CodeProfile = "profile" + CodeNoSeeders = "no_seeders" + CodeDoubt = "doubt" +) + +// Reason is one thing a verdict rests on. +type Reason struct { + Kind string `json:"kind"` + Code string `json:"code"` + Text string `json:"text"` +} + +// Want is the album searched for: its wants row and how the catalogue +// describes it. +type Want struct { + ID pgtype.UUID + release.Want +} + +// Candidate is one result with everything decided about it. +type Candidate struct { + release.Release + // Source is the indexer's own result. It never leaves the server: its + // DownloadURL can carry the indexer's key. + Source indexer.Result + InfoHash string + Indexer string + SizeBytes int64 + Seeders int + SwarmKnown bool + Category int + FilesCount int + // Files is the torrent's file list once its .torrent has been read. + Files []release.File + FoldedFrom []string + + Tier release.Tier + YearAgrees bool + Verdict Verdict + Reason string + Reasons []Reason +} + +// Key is how a candidate is named across a search and a grab: its infohash, +// or its name when the source gave none. +func Key(c Candidate) string { + if c.InfoHash != "" { + return c.InfoHash + } + return c.Name +} + +// Result is one search. +type Result struct { + Query string + // Unconfigured: no search source is set up. + Unconfigured bool + // Error is a search that failed outright. + Error string + // Narrowed names sources that did not answer on a search that otherwise + // did. + Narrowed []Narrowing + Considered int + Folded int + // Looked is how many .torrent files were read for their file lists. + Looked int + Candidates []Candidate + // KeepError: the candidates could not be stored. + KeepError string +} + +// Grabbable is what may be taken, best first. +func (r Result) Grabbable() []Candidate { + var out []Candidate + for _, c := range r.Candidates { + if c.Verdict == Grab { + out = append(out, c) + } + } + return out +} + +// ForAPerson says whether anything found is worth a person's look: a +// release that may be the album, whatever its quality. +func (r Result) ForAPerson() bool { + for _, c := range r.Candidates { + if c.Verdict == Grab || c.Verdict == Review || c.Verdict == Refused { + return true + } + } + return false +} + +// Service searches and grabs. +type Service struct { + // Search is nil when no source is configured. + Search Searcher + // Grabber is nil when no download client is configured. + Grabber *Downloader + Profile release.Profile + // Memory stores candidates, grabs and blocks. Nil keeps nothing. + Memory *Memory +} + +// Query is the text sent to the indexers: the artist and the title, without +// a trailing bracketed part the release names usually leave off. +func Query(w release.Want) string { + title := w.Title + if i := strings.IndexAny(title, "(["); i > 0 { + title = title[:i] + } + return strings.Join(strings.Fields(strings.TrimSpace(w.Artist)+" "+strings.TrimSpace(title)), " ") +} + +// For searches for one want and judges everything that comes back. It never +// returns an error: a search that could not be made is a Result saying so. +func (s *Service) For(ctx context.Context, want Want) Result { + r := Result{Query: Query(want.Want)} + if s == nil || s.Search == nil { + r.Unconfigured = true + return r + } + found, err := s.Search.Search(ctx, indexer.Query{Text: r.Query, Categories: indexer.MusicCategories}) + // A partial search arrives as results and an error together. + if narrowed, answered := narrowings(err); narrowed != nil && answered { + r.Narrowed, err = narrowed, nil + } + switch { + case errors.Is(err, indexer.ErrUnconfigured): + r.Unconfigured = true + return r + case err != nil: + r.Error = indexer.FirstLine(err.Error()) + return r + } + r.Considered = len(found) + cands, folded := fold(found) + r.Folded = folded + + blocked := s.Memory.blocked(ctx) + for i := range cands { + decide(&cands[i], want.Want, s.Profile, blocked) + } + sortCandidates(cands) + for _, i := range s.peek(ctx, cands) { + decide(&cands[i], want.Want, s.Profile, blocked) + r.Looked++ + } + sortCandidates(cands) + r.Candidates = cands + if s.Memory != nil && want.ID.Valid { + if err := s.Memory.Keep(ctx, want.ID, cands); err != nil { + r.KeepError = indexer.FirstLine(err.Error()) + } + } + return r +} + +// decide works out one candidate's verdict, in order: given up on, then +// whether it is the album, then whether the profile takes its quality, then +// whether anything leaves it to a person. A quality refusal said about the +// wrong album would be noise, so identity comes first. +func decide(c *Candidate, w release.Want, profile release.Profile, blocked map[string]string) { + c.Verdict, c.Reason, c.Reasons = "", "", nil + if why, ok := blocked[c.InfoHash]; ok && c.InfoHash != "" { + set(c, Blocked, Hard, CodeBlocked, why) + return + } + m := release.Judge(w, c.Release) + c.YearAgrees = m.YearAgrees + if m.Veto != "" { + set(c, Rejected, Hard, m.Veto, m.Reason) + return + } + doubt := m.Doubt + if len(c.Files) > 0 { + if in := release.Inspect(c.Files); in.Format != "" { + c.Format = in.Format + } + veto, reason, fileDoubt := release.JudgeFiles(w, c.Files) + if veto != "" { + set(c, Rejected, Hard, veto, reason) + return + } + if doubt == "" { + doubt = fileDoubt + } + } + c.Tier = release.TierOf(c.Release, c.Category) + if !profile.Allows(c.Tier) { + set(c, Refused, Soft, CodeProfile, "Your quality settings don’t take "+strings.ToLower(release.Label(c.Tier))+".") + return + } + if doubt != "" { + set(c, Review, Soft, CodeDoubt, doubt) + return + } + if c.Seeders == 0 { + set(c, Review, Soft, CodeNoSeeders, "Nobody is sharing it right now.") + return + } + c.Verdict = Grab + c.Reason = "The album, " + strings.ToLower(release.Label(c.Tier)) + "." + if d := release.Describe(c.Release); d != "" { + c.Reason = "The album, " + d + "." + } +} + +func set(c *Candidate, v Verdict, kind, code, text string) { + c.Verdict, c.Reason = v, text + c.Reasons = append(c.Reasons, Reason{Kind: kind, Code: code, Text: text}) +} + +// sortCandidates puts the best first, stably so the same search proposes the +// same grab twice. +func sortCandidates(cands []Candidate) { + sort.SliceStable(cands, func(a, b int) bool { return better(cands[a], cands[b]) }) +} + +// better says whether x ranks above y: verdict, then quality, then how many +// share it, then whether its year agrees. +func better(x, y Candidate) bool { + if order[x.Verdict] != order[y.Verdict] { + return order[x.Verdict] < order[y.Verdict] + } + if rx, ry := release.Rank(x.Tier), release.Rank(y.Tier); rx != ry { + return rx > ry + } + if x.Seeders != y.Seeders { + return x.Seeders > y.Seeders + } + return x.YearAgrees && !y.YearAgrees +} + +// fold merges results that are the same torrent, by infohash. Prowlarr +// aggregates, so one release arrives once per tracker that lists it. A +// result without a hash is never folded: an absent hash is not a shared one. +func fold(found []indexer.Result) ([]Candidate, int) { + out := make([]Candidate, 0, len(found)) + at := make(map[string]int, len(found)) + folded := 0 + for _, f := range found { + if strings.TrimSpace(f.Title) == "" { + continue + } + hash := indexer.NormalizeHash(f.InfoHash) + if hash == "" && f.Magnet != "" { + hash = indexer.MagnetHash(f.Magnet) + } + if hash != "" { + if i, seen := at[hash]; seen { + folded++ + if f.Indexer != "" { + out[i].FoldedFrom = append(out[i].FoldedFrom, f.Indexer) + } + // A measured swarm beats a reported one; among like, the + // highest count. + switch { + case f.SwarmKnown && !out[i].SwarmKnown: + out[i].Seeders, out[i].SwarmKnown = f.Seeders, true + case f.SwarmKnown == out[i].SwarmKnown && f.Seeders > out[i].Seeders: + out[i].Seeders = f.Seeders + } + if f.FilesCount > 0 && out[i].FilesCount == 0 { + out[i].FilesCount = f.FilesCount + } + continue + } + at[hash] = len(out) + } + out = append(out, Candidate{ + Release: release.Parse(f.Title), Source: f, InfoHash: hash, Indexer: f.Indexer, + SizeBytes: f.SizeBytes, Seeders: f.Seeders, SwarmKnown: f.SwarmKnown, + Category: f.Category, FilesCount: f.FilesCount, + }) + } + return out, folded +} + +// maxPeeks is how many .torrent files one search reads. Few: each goes +// through Prowlarr to a tracker, and Prowlarr answers 429 to a burst. +const maxPeeks = 3 + +// peek reads the .torrent of candidates that are the album but whose name +// states no format, so their files can say what they are; it returns which +// now carry a file list. A failed fetch leaves the candidate as it was. +func (s *Service) peek(ctx context.Context, cands []Candidate) []int { + f, ok := s.Search.(Fetcher) + if !ok { + return nil + } + var looked []int + for i := range cands { + if len(looked) >= maxPeeks || ctx.Err() != nil { + break + } + c := &cands[i] + if c.Tier != release.TierUnknown || (c.Verdict != Grab && c.Verdict != Refused) || + c.Files != nil || c.Source.DownloadURL == "" || c.Source.Magnet != "" { + continue + } + body, err := f.Download(ctx, c.Source) + if err != nil { + continue + } + meta, err := release.ParseTorrent(body) + if err != nil { + continue + } + c.Files = meta.Files + if c.InfoHash == "" { + c.InfoHash = meta.InfoHash + } + if c.FilesCount == 0 { + c.FilesCount = len(meta.Files) + } + looked = append(looked, i) + } + return looked +} + +// pick finds a candidate by key: the infohash first, the exact name for a +// result that has none. +func pick(cands []Candidate, key string) *Candidate { + key = strings.TrimSpace(key) + if key == "" { + return nil + } + hash := indexer.NormalizeHash(key) + for i := range cands { + if cands[i].InfoHash != "" && cands[i].InfoHash == hash { + return &cands[i] + } + } + for i := range cands { + if cands[i].InfoHash == "" && cands[i].Name == key { + return &cands[i] + } + } + return nil +} diff --git a/internal/acquire/search_test.go b/internal/acquire/search_test.go new file mode 100644 index 00000000..f158b345 --- /dev/null +++ b/internal/acquire/search_test.go @@ -0,0 +1,241 @@ +package acquire + +import ( + "context" + "errors" + "strings" + "testing" + "time" + + "git.fabledsword.com/bvandeusen/minstrel/internal/downloader" + "git.fabledsword.com/bvandeusen/minstrel/internal/indexer" + "git.fabledsword.com/bvandeusen/minstrel/internal/release" +) + +type fakeSearch struct { + results []indexer.Result + err error + asked []indexer.Query +} + +func (f *fakeSearch) Search(_ context.Context, q indexer.Query) ([]indexer.Result, error) { + f.asked = append(f.asked, q) + return f.results, f.err +} + +var ram = Want{Want: release.Want{ + Artist: "Daft Punk", Title: "Random Access Memories", Year: 2013, PrimaryType: "Album", + MinTracks: 13, MaxTracks: 15, +}} + +var anyQuality = release.NewProfile([]string{ + "lossless_hires", "lossless", "lossy_high", "lossy_standard", "lossy_low", "lossy_poor", "lossy_trash", +}) + +func result(title, hash string, seeders int) indexer.Result { + return indexer.Result{Title: title, InfoHash: hash, Seeders: seeders, Indexer: "Knaben", Source: "prowlarr", + Magnet: "magnet:?xt=urn:btih:" + hash} +} + +func TestFor_JudgesSortsAndFolds(t *testing.T) { + search := &fakeSearch{results: []indexer.Result{ + result("Daft Punk - Random Access Memories (2013) [MP3 320]", "aaaa", 40), + result("Daft Punk Random Access Memories (Drumless Edition 2023) [Flac 24-88]", "bbbb", 90), + result("Daft Punk - Random Access Memories (2013) [FLAC]", "cccc", 12), + result("Daft Punk - Random Access Memories (2013) [FLAC]", "CCCC", 30), // the same torrent from another tracker + result("Daft Punk - Discovery (2001) [FLAC]", "dddd", 50), + }} + s := &Service{Search: search, Profile: anyQuality} + r := s.For(context.Background(), ram) + + if len(search.asked) != 1 || search.asked[0].Text != "Daft Punk Random Access Memories" { + t.Fatalf("asked %+v", search.asked) + } + if r.Considered != 5 || r.Folded != 1 || len(r.Candidates) != 4 { + t.Fatalf("considered %d folded %d candidates %d", r.Considered, r.Folded, len(r.Candidates)) + } + // Lossless before high-quality lossy among what can be taken; the + // folded copy keeps the higher seeder count. + first, second := r.Candidates[0], r.Candidates[1] + if first.InfoHash != "cccc" || first.Verdict != Grab || first.Seeders != 30 || first.Tier != release.TierLossless { + t.Errorf("first = %s %s %d %s", first.Name, first.Verdict, first.Seeders, first.Tier) + } + if second.InfoHash != "aaaa" || second.Verdict != Grab { + t.Errorf("second = %s %s", second.Name, second.Verdict) + } + for _, c := range r.Candidates[2:] { + if c.Verdict != Rejected || len(c.Reasons) != 1 || c.Reasons[0].Kind != Hard { + t.Errorf("%s: %s %+v", c.Name, c.Verdict, c.Reasons) + } + } + if len(r.Grabbable()) != 2 || !r.ForAPerson() { + t.Errorf("grabbable %d", len(r.Grabbable())) + } +} + +func TestFor_ProfileSeedersAndDoubt(t *testing.T) { + search := &fakeSearch{results: []indexer.Result{ + result("Daft Punk - Random Access Memories [MP3 128]", "aaaa", 5), + result("Daft Punk - Random Access Memories [FLAC]", "bbbb", 0), + result("Daft Punk - Random Access Memories", "cccc", 9), + }} + s := &Service{Search: search, Profile: release.NewProfile([]string{"lossless", "lossy_high"})} + r := s.For(context.Background(), ram) + verdicts := map[string]Verdict{} + for _, c := range r.Candidates { + verdicts[c.InfoHash] = c.Verdict + } + if verdicts["aaaa"] != Refused || verdicts["bbbb"] != Review || verdicts["cccc"] != Refused { + t.Errorf("verdicts = %v", verdicts) + } + if len(r.Grabbable()) != 0 || !r.ForAPerson() { + t.Error("nothing grabbable, but something for a person") + } +} + +func TestFor_BlockedReleasesAreSetAside(t *testing.T) { + c := Candidate{Release: release.Parse("Daft Punk - Random Access Memories [FLAC]"), InfoHash: "aaaa", Seeders: 3} + decide(&c, ram.Want, anyQuality, map[string]string{"aaaa": "You set this release aside."}) + if c.Verdict != Blocked || c.Reason != "You set this release aside." { + t.Errorf("verdict %s %q", c.Verdict, c.Reason) + } +} + +func TestFor_UnconfiguredAndFailedSearches(t *testing.T) { + if r := (&Service{}).For(context.Background(), ram); !r.Unconfigured { + t.Error("no source is unconfigured") + } + r := (&Service{Search: &fakeSearch{err: indexer.ErrUnconfigured}}).For(context.Background(), ram) + if !r.Unconfigured { + t.Error("ErrUnconfigured is unconfigured") + } + r = (&Service{Search: &fakeSearch{err: errors.New("dial tcp: refused")}}).For(context.Background(), ram) + if r.Error == "" { + t.Error("a failed search says so") + } +} + +func TestFanout_ANarrowedSearchKeepsWhatAnswered(t *testing.T) { + up := &fakeSearch{results: []indexer.Result{result("Daft Punk - Random Access Memories [FLAC]", "aaaa", 3)}} + down := &fakeSearch{err: indexer.ErrUnreachable} + f := &Fanout{Sources: []Source{{Code: "prowlarr", Label: "Prowlarr", Search: up}, {Code: "torznab", Label: "Feed", Search: down}}} + r := (&Service{Search: f, Profile: anyQuality}).For(context.Background(), ram) + if r.Error != "" || len(r.Candidates) != 1 || len(r.Narrowed) != 1 || r.Narrowed[0].Source != "torznab" { + t.Errorf("narrowed search: error %q, %d candidates, narrowed %+v", r.Error, len(r.Candidates), r.Narrowed) + } + f.Sources = f.Sources[1:] + if r := (&Service{Search: f}).For(context.Background(), ram); r.Error == "" { + t.Error("every source down is a failed search") + } +} + +func TestPacer_SpacesSearchesAndHoldsAfterA429(t *testing.T) { + now := time.Unix(0, 0) + var slept time.Duration + p := &Pacer{Now: func() time.Time { return now }, Sleep: func(_ context.Context, d time.Duration) error { slept += d; return nil }} + src := []Source{{Code: "prowlarr", PerHour: 120, Search: &fakeSearch{}}} + p.spend(src[0]) + if !p.Wait(context.Background(), src, now.Add(time.Hour)) || slept != 30*time.Second { + t.Errorf("slept %s, want 30s", slept) + } + if p.Wait(context.Background(), src, now.Add(time.Second)) { + t.Error("waited past the deadline") + } + p.hold("prowlarr", &indexer.RateLimited{RetryAfter: 5 * time.Minute}) + if at := p.readyAt(src); at.Sub(now) != 5*time.Minute { + t.Errorf("held until %s", at.Sub(now)) + } +} + +type fakeClient struct { + downloader.Driver + added []downloader.Add + addErr error + deleted []string + progress float64 +} + +func (c *fakeClient) AddTorrent(_ context.Context, a downloader.Add) error { + c.added = append(c.added, a) + return c.addErr +} + +func (c *fakeClient) Torrents(_ context.Context, hashes ...string) ([]downloader.Torrent, error) { + return []downloader.Torrent{{Hash: hashes[0], Progress: c.progress}}, nil +} + +func (c *fakeClient) Delete(_ context.Context, _ bool, hashes ...string) error { + c.deleted = append(c.deleted, hashes...) + return nil +} + +func TestGrab_AddsAMagnetToItsOwnFolder(t *testing.T) { + client := &fakeClient{} + s := &Service{Search: &fakeSearch{results: []indexer.Result{ + result("Daft Punk - Random Access Memories (2013) [FLAC]", "ABCD", 8), + }}, Profile: anyQuality, Grabber: &Downloader{Client: client, Code: "qbittorrent", Landing: "/downloads"}} + r := s.For(context.Background(), ram) + took := s.Grab(context.Background(), ram, &r.Candidates[0], r.Candidates, false, Origin{By: OriginSchedule}) + if !took.Added || took.SavePath != "/downloads/abcd" { + t.Fatalf("took %+v", took) + } + if len(client.added) != 1 || !strings.Contains(strings.ToLower(client.added[0].Magnet), "abcd") || client.added[0].Tags[0] != Tag { + t.Errorf("added %+v", client.added) + } +} + +func TestGrab_RefusesWhatTheVerdictWouldNotTake(t *testing.T) { + client := &fakeClient{} + s := &Service{Grabber: &Downloader{Client: client, Landing: "/downloads"}} + review := Candidate{Verdict: Review, InfoHash: "aaaa", Source: indexer.Result{Magnet: "magnet:?xt=urn:btih:aaaa"}} + if took := s.Grab(context.Background(), ram, &review, nil, false, Origin{By: OriginSchedule}); !took.Refused { + t.Error("a review was taken unforced") + } + if took := s.Grab(context.Background(), ram, &review, nil, true, Origin{By: OriginOperator}); !took.Added { + t.Errorf("a forced review was not taken: %+v", took) + } + blocked := Candidate{Verdict: Blocked, InfoHash: "bbbb"} + if took := s.Grab(context.Background(), ram, &blocked, nil, true, Origin{By: OriginOperator}); !took.Refused { + t.Error("forcing took a release set aside") + } + if took := (&Service{}).Grab(context.Background(), ram, &review, nil, true, Origin{}); !took.Unconfigured || took.Missing != MissingClient { + t.Errorf("no client: %+v", took) + } +} + +func TestGrab_AClientThatAlreadyHasItCounts(t *testing.T) { + client := &fakeClient{addErr: downloader.ErrDuplicate} + s := &Service{Grabber: &Downloader{Client: client, Landing: "/downloads"}} + c := Candidate{Verdict: Grab, InfoHash: "aaaa", Source: indexer.Result{Magnet: "magnet:?xt=urn:btih:aaaa"}} + if took := s.Grab(context.Background(), ram, &c, nil, false, Origin{By: OriginSchedule}); !took.Added { + t.Errorf("a duplicate: %+v", took) + } +} + +func TestBetter_QualityThenSeedersThenYear(t *testing.T) { + flac := Candidate{Verdict: Grab, Tier: release.TierLossless, Seeders: 2} + mp3 := Candidate{Verdict: Grab, Tier: release.TierLossyHigh, Seeders: 90} + if !better(flac, mp3) || better(mp3, flac) { + t.Error("quality first among grabs") + } + busy := Candidate{Verdict: Grab, Tier: release.TierLossless, Seeders: 20} + if !better(busy, flac) { + t.Error("then seeders") + } + dated := Candidate{Verdict: Grab, Tier: release.TierLossless, Seeders: 2, YearAgrees: true} + if !better(dated, flac) { + t.Error("then the year") + } + if better(Candidate{Verdict: Review}, flac) || !better(flac, Candidate{Verdict: Review}) { + t.Error("a grab ranks above a review") + } +} + +func TestLandingFor_OneFolderPerTorrent(t *testing.T) { + if got := landingFor("/downloads", "abcd", "x"); got != "/downloads/abcd" { + t.Error(got) + } + if got := landingFor("/downloads", "", "Daft Punk: R.A.M. [FLAC]"); got != "/downloads/daft-punk-r-a-m--flac" { + t.Error(got) + } +} diff --git a/internal/api/admin_search.go b/internal/api/admin_search.go new file mode 100644 index 00000000..53b7dbe0 --- /dev/null +++ b/internal/api/admin_search.go @@ -0,0 +1,299 @@ +package api + +import ( + "context" + "encoding/json" + "errors" + "net/http" + "time" + + "github.com/go-chi/chi/v5" + "github.com/jackc/pgx/v5" + + "git.fabledsword.com/bvandeusen/minstrel/internal/acquire" + "git.fabledsword.com/bvandeusen/minstrel/internal/apierror" + "git.fabledsword.com/bvandeusen/minstrel/internal/db/dbq" + "git.fabledsword.com/bvandeusen/minstrel/internal/release" + "git.fabledsword.com/bvandeusen/minstrel/internal/wanted" +) + +// Searching for wants and grabbing releases (M502 #5521): the schedule's +// settings, each want's candidates, and a person's Search, Grab and "never +// this one". + +// searchTimeout bounds a search a person waits on: a Prowlarr fanning out to +// many trackers can take most of a minute. +const searchTimeout = 2 * time.Minute + +type searchSettingsView struct { + AutoGrab bool `json:"auto_grab"` + SearchIntervalHours int32 `json:"search_interval_hours"` + AllowedQualities []string `json:"allowed_qualities"` + Tiers []release.TierInfo `json:"tiers"` + LastPassAt *string `json:"last_pass_at"` + LastPass json.RawMessage `json:"last_pass"` +} + +func searchSettings(s dbq.AcquisitionSetting) searchSettingsView { + v := searchSettingsView{ + AutoGrab: s.AutoGrab, SearchIntervalHours: s.SearchIntervalHours, + AllowedQualities: s.AllowedQualities, Tiers: release.Tiers, LastPass: s.LastPass, + } + if v.AllowedQualities == nil { + v.AllowedQualities = []string{} + } + if len(v.LastPass) == 0 { + v.LastPass = json.RawMessage("{}") + } + if s.LastPassAt.Valid { + at := formatTimestamp(s.LastPassAt) + v.LastPassAt = &at + } + return v +} + +// handleGetSearchSettings implements GET /api/admin/acquisition/settings. +func (h *handlers) handleGetSearchSettings(w http.ResponseWriter, r *http.Request) { + s, err := dbq.New(h.pool).GetAcquisitionSettings(r.Context()) + if err != nil { + writeErrWithLog(w, h.logger, "admin search settings: get", apierror.Internal(err)) + return + } + writeJSON(w, http.StatusOK, searchSettings(s)) +} + +type putSearchSettingsBody struct { + AutoGrab bool `json:"auto_grab"` + SearchIntervalHours int32 `json:"search_interval_hours"` + AllowedQualities []string `json:"allowed_qualities"` +} + +// handlePutSearchSettings implements PUT /api/admin/acquisition/settings. +func (h *handlers) handlePutSearchSettings(w http.ResponseWriter, r *http.Request) { + var body putSearchSettingsBody + if !decodeBody(w, r, &body) { + return + } + if body.SearchIntervalHours < 0 || body.SearchIntervalHours > 720 { + writeErr(w, apierror.BadRequest("bad_interval", "search_interval_hours must be between 0 and 720")) + return + } + if len(body.AllowedQualities) == 0 { + writeErr(w, apierror.BadRequest("bad_quality", "allow at least one quality")) + return + } + for _, t := range body.AllowedQualities { + if !release.KnownTier(t) { + writeErr(w, apierror.BadRequest("bad_quality", "unknown quality "+t)) + return + } + } + s, err := dbq.New(h.pool).UpdateAcquisitionSettings(r.Context(), dbq.UpdateAcquisitionSettingsParams{ + AutoGrab: body.AutoGrab, SearchIntervalHours: body.SearchIntervalHours, AllowedQualities: body.AllowedQualities, + }) + if err != nil { + writeErrWithLog(w, h.logger, "admin search settings: put", apierror.Internal(err)) + return + } + writeJSON(w, http.StatusOK, searchSettings(s)) +} + +// candidateView is one release a want's last search found. +type candidateView struct { + Key string `json:"key"` + Name string `json:"name"` + InfoHash string `json:"info_hash"` + Indexer string `json:"indexer"` + SizeBytes int64 `json:"size_bytes"` + Seeders int32 `json:"seeders"` + FilesCount int32 `json:"files_count"` + Format string `json:"format"` + Quality string `json:"quality"` + Year int32 `json:"year"` + Verdict string `json:"verdict"` + Reason string `json:"reason"` + Reasons json.RawMessage `json:"reasons"` + FoldedFrom []string `json:"folded_from"` + SearchedAt string `json:"searched_at"` + Downloading bool `json:"downloading"` +} + +// handleListCandidates implements GET /api/admin/wanted/{id}/candidates. +func (h *handlers) handleListCandidates(w http.ResponseWriter, r *http.Request) { + id, ok := requireURLUUID(w, r, "id") + if !ok { + return + } + q := dbq.New(h.pool) + rows, err := q.ListWantCandidates(r.Context(), id) + if err != nil { + writeErrWithLog(w, h.logger, "admin wanted: candidates", apierror.Internal(err)) + return + } + live, err := q.GetLiveGrab(r.Context(), id) + if err != nil && !errors.Is(err, pgx.ErrNoRows) { + writeErrWithLog(w, h.logger, "admin wanted: live grab", apierror.Internal(err)) + return + } + out := make([]candidateView, 0, len(rows)) + for _, c := range rows { + v := candidateView{ + Key: c.Key, Name: c.Name, InfoHash: c.InfoHash, Indexer: c.Indexer, SizeBytes: c.SizeBytes, + Seeders: c.Seeders, FilesCount: c.FilesCount, Format: c.Format, Quality: c.Quality, Year: c.Year, + Verdict: c.Verdict, Reason: c.Reason, Reasons: c.Reasons, FoldedFrom: c.FoldedFrom, + SearchedAt: formatTimestamp(c.SearchedAt), + Downloading: live.ID.Valid && live.Key == c.Key, + } + if v.FoldedFrom == nil { + v.FoldedFrom = []string{} + } + if len(v.Reasons) == 0 { + v.Reasons = json.RawMessage("[]") + } + out = append(out, v) + } + writeJSON(w, http.StatusOK, map[string]any{"items": out}) +} + +// handleSearchWant implements POST /api/admin/wanted/{id}/search: search +// now, as the schedule would, taking the best release when auto-grab is on. +func (h *handlers) handleSearchWant(w http.ResponseWriter, r *http.Request) { + id, ok := requireURLUUID(w, r, "id") + if !ok { + return + } + ctx, cancel := context.WithTimeout(r.Context(), searchTimeout) + defer cancel() + res, err := h.searcher.SearchNow(ctx, id) + if h.writeSearchErr(w, err) { + return + } + writeJSON(w, http.StatusOK, res) +} + +type grabBody struct { + Key string `json:"key"` + Force bool `json:"force"` +} + +// grabView is what came of a person's grab. +type grabView struct { + Added bool `json:"added"` + Refused bool `json:"refused"` + Gone bool `json:"gone"` + Unconfigured bool `json:"unconfigured"` + Missing string `json:"missing,omitempty"` + Kept bool `json:"kept"` + Replaced bool `json:"replaced"` + Verdict string `json:"verdict,omitempty"` + Reason string `json:"reason,omitempty"` + Error string `json:"error,omitempty"` +} + +// handleGrabCandidate implements POST /api/admin/wanted/{id}/grab: take the +// release named by key. The search runs again first, so the indexer's link +// never leaves the server; force takes what the verdict would not. +func (h *handlers) handleGrabCandidate(w http.ResponseWriter, r *http.Request) { + admin, ok := requireUser(w, r) + if !ok { + return + } + id, ok := requireURLUUID(w, r, "id") + if !ok { + return + } + var body grabBody + if !decodeBody(w, r, &body) { + return + } + ctx, cancel := context.WithTimeout(r.Context(), searchTimeout) + defer cancel() + took, err := h.searcher.Take(ctx, id, body.Key, body.Force, admin.ID) + if h.writeSearchErr(w, err) { + return + } + v := grabView{ + Added: took.Added, Refused: took.Refused, Gone: took.Gone, Unconfigured: took.Unconfigured, + Missing: took.Missing, Error: took.Error, + } + if took.Chosen != nil { + v.Verdict, v.Reason = string(took.Chosen.Verdict), took.Chosen.Reason + } + if took.Contest != nil { + v.Kept, v.Replaced = took.Contest.Kept, took.Contest.Replaced + if took.Contest.Kept { + v.Reason = took.Contest.Reason + } + } + writeJSON(w, http.StatusOK, v) +} + +type blockBody struct { + Key string `json:"key"` +} + +// handleBlockCandidate implements POST /api/admin/wanted/{id}/block: never +// take this release again, for any want. +func (h *handlers) handleBlockCandidate(w http.ResponseWriter, r *http.Request) { + admin, ok := requireUser(w, r) + if !ok { + return + } + id, ok := requireURLUUID(w, r, "id") + if !ok { + return + } + var body blockBody + if !decodeBody(w, r, &body) { + return + } + c, err := dbq.New(h.pool).GetWantCandidate(r.Context(), dbq.GetWantCandidateParams{WantID: id, Key: body.Key}) + if errors.Is(err, pgx.ErrNoRows) { + writeErr(w, apierror.NotFound("candidate")) + return + } + if err != nil { + writeErrWithLog(w, h.logger, "admin wanted: block", apierror.Internal(err)) + return + } + if c.InfoHash == "" { + writeErr(w, apierror.BadRequest("no_infohash", "this release has no infohash to set aside by")) + return + } + if err := acquire.NewMemory(h.pool).Block(r.Context(), dbq.BlockReleaseParams{ + InfoHash: c.InfoHash, Name: c.Name, Reason: acquire.BlockOperator, WantID: id, BlockedBy: admin.ID, + }); err != nil { + writeErrWithLog(w, h.logger, "admin wanted: block", apierror.Internal(err)) + return + } + w.WriteHeader(http.StatusNoContent) +} + +// handleUnblockRelease implements DELETE /api/admin/release-blocks/{hash}. +func (h *handlers) handleUnblockRelease(w http.ResponseWriter, r *http.Request) { + removed, err := acquire.NewMemory(h.pool).Unblock(r.Context(), chi.URLParam(r, "hash")) + if err != nil { + writeErrWithLog(w, h.logger, "admin wanted: unblock", apierror.Internal(err)) + return + } + if !removed { + writeErr(w, apierror.NotFound("block")) + return + } + w.WriteHeader(http.StatusNoContent) +} + +// writeSearchErr answers a Searcher error, and says whether there was one. +func (h *handlers) writeSearchErr(w http.ResponseWriter, err error) bool { + switch { + case err == nil: + return false + case errors.Is(err, pgx.ErrNoRows): + writeErr(w, apierror.NotFound("want")) + case errors.Is(err, wanted.ErrWantHeld): + writeErr(w, apierror.Conflict("want_held", "the library already holds this album")) + default: + writeErrWithLog(w, h.logger, "admin wanted: search", apierror.Internal(err)) + } + return true +} diff --git a/internal/api/admin_wanted.go b/internal/api/admin_wanted.go index ab4a3eb2..6c740ab6 100644 --- a/internal/api/admin_wanted.go +++ b/internal/api/admin_wanted.go @@ -35,6 +35,25 @@ type wantView struct { WantedAt string `json:"wanted_at"` HeldAt *string `json:"held_at"` ImageURL string `json:"image_url"` + // Search is the schedule's last look; Grab the download running (#5521). + Search *wantSearchView `json:"search"` + Grab *wantGrabView `json:"grab"` +} + +type wantSearchView struct { + SearchedAt string `json:"searched_at"` + Outcome string `json:"outcome"` + Considered int32 `json:"considered"` + BestRelease string `json:"best_release"` + BestVerdict string `json:"best_verdict"` + Error string `json:"error"` +} + +type wantGrabView struct { + Name string `json:"name"` + GrabbedAt string `json:"grabbed_at"` + Origin string `json:"origin"` + Quality string `json:"quality"` } // wantCounts sizes each tab of the Wanted page. @@ -118,6 +137,22 @@ func (h *handlers) handleListWants(w http.ResponseWriter, r *http.Request) { if v.SecondaryTypes == nil { v.SecondaryTypes = []string{} } + if row.SearchedAt.Valid { + v.Search = &wantSearchView{ + SearchedAt: formatTimestamp(row.SearchedAt), Outcome: row.SearchOutcome, + Considered: row.SearchConsidered, BestRelease: row.BestRelease, + BestVerdict: row.BestVerdict, Error: row.SearchError, + } + } + if row.GrabName != nil { + v.Grab = &wantGrabView{Name: *row.GrabName, GrabbedAt: formatTimestamp(row.GrabbedAt)} + if row.GrabOrigin != nil { + v.Grab.Origin = *row.GrabOrigin + } + if row.GrabQuality != nil { + v.Grab.Quality = *row.GrabQuality + } + } out.Items = append(out.Items, v) } writeJSON(w, http.StatusOK, out) diff --git a/internal/api/api.go b/internal/api/api.go index 9d3e2721..08d2d231 100644 --- a/internal/api/api.go +++ b/internal/api/api.go @@ -77,6 +77,7 @@ func Mount(r chi.Router, pool *pgxpool.Pool, logger *slog.Logger, events *playev acquisition: acquire.NewStore(pool), catalogue: cat, wanted: wanted.New(pool, cat, logger), + searcher: wanted.NewSearcher(pool, cat, logger), catalogueSearches: newSearchCache(), catalogueArt: catalogue.NewArt(dataDir, artSources(coverSettings)), leveled: newLeveledRenderer(dataDir, loudSettings, logger), @@ -248,6 +249,14 @@ func Mount(r chi.Router, pool *pgxpool.Pool, logger *slog.Logger, events *playev // profiles that decide which of their releases count. admin.Get("/wanted", h.handleListWants) admin.Delete("/wanted/{id}", h.handleRemoveWant) + // Searching and grabbing (M502 #5521). + admin.Get("/acquisition/settings", h.handleGetSearchSettings) + admin.Put("/acquisition/settings", h.handlePutSearchSettings) + admin.Get("/wanted/{id}/candidates", h.handleListCandidates) + admin.Post("/wanted/{id}/search", h.handleSearchWant) + admin.Post("/wanted/{id}/grab", h.handleGrabCandidate) + admin.Post("/wanted/{id}/block", h.handleBlockCandidate) + admin.Delete("/release-blocks/{hash}", h.handleUnblockRelease) admin.Get("/follows", h.handleListFollows) admin.Get("/follows/{mbid}", h.handleGetFollow) admin.Put("/follows/{mbid}", h.handlePutFollow) @@ -416,6 +425,8 @@ type handlers struct { // wanted is the wanted list: follows, release-type profiles and wants // (M502 #5520). Its worker runs in main; the API reads and edits. wanted *wanted.Service + // searcher searches for wants and grabs releases for them (#5521). + searcher *wanted.Searcher // catalogueSearches reuses recent MusicBrainz answers; nil asks every // time, which is what handler tests get. catalogueSearches *searchCache diff --git a/internal/db/dbq/models.go b/internal/db/dbq/models.go index 75d88561..0d4214dd 100644 --- a/internal/db/dbq/models.go +++ b/internal/db/dbq/models.go @@ -217,6 +217,16 @@ type AcquisitionService struct { UpdatedAt pgtype.Timestamptz } +type AcquisitionSetting struct { + ID bool + AutoGrab bool + SearchIntervalHours int32 + AllowedQualities []string + LastPassAt pgtype.Timestamptz + LastPass []byte + UpdatedAt pgtype.Timestamptz +} + type Album struct { ID pgtype.UUID Title string @@ -420,6 +430,26 @@ type GeneralLikesArtist struct { LikedAt pgtype.Timestamptz } +type Grab struct { + ID pgtype.UUID + WantID pgtype.UUID + InfoHash string + Name string + Key string + Indexer string + Source string + Client string + SavePath string + Quality string + Origin string + Forced bool + GrabbedBy pgtype.UUID + GrabbedAt pgtype.Timestamptz + EndedAt pgtype.Timestamptz + EndReason string + EndDetail string +} + type LibraryChange struct { ID int64 EntityType string @@ -652,6 +682,16 @@ type RegistrationSetting struct { Mode string } +type ReleaseBlock struct { + InfoHash string + Name string + Reason string + Detail string + WantID pgtype.UUID + BlockedBy pgtype.UUID + BlockedAt pgtype.Timestamptz +} + type ReleaseTypeProfile struct { ID pgtype.UUID Name string @@ -976,6 +1016,33 @@ type Want struct { FulfilledAt pgtype.Timestamptz CreatedAt pgtype.Timestamptz UpdatedAt pgtype.Timestamptz + SearchedAt pgtype.Timestamptz + SearchOutcome string + SearchConsidered int32 + BestRelease string + BestVerdict string + SearchError string +} + +type WantCandidate struct { + WantID pgtype.UUID + Key string + Name string + InfoHash string + Indexer string + Source string + SizeBytes int64 + Seeders int32 + FilesCount int32 + Format string + Quality string + Year int32 + Verdict string + Reason string + Reasons []byte + FoldedFrom []string + Rank int32 + SearchedAt pgtype.Timestamptz } type YouMightLikeAlbum struct { diff --git a/internal/db/dbq/reacquisition.sql.go b/internal/db/dbq/reacquisition.sql.go index c549005b..c4ea7208 100644 --- a/internal/db/dbq/reacquisition.sql.go +++ b/internal/db/dbq/reacquisition.sql.go @@ -212,7 +212,7 @@ ON CONFLICT (release_group_mbid) DO UPDATE fulfilled_at = NULL, updated_at = now() WHERE wants.fulfilled_at IS NOT NULL -RETURNING id, release_group_mbid, artist_mbid, artist_name, title, primary_type, first_release_date, reason, album_id, created_by, wanted_at, fulfilled_at, created_at, updated_at +RETURNING id, release_group_mbid, artist_mbid, artist_name, title, primary_type, first_release_date, reason, album_id, created_by, wanted_at, fulfilled_at, created_at, updated_at, searched_at, search_outcome, search_considered, best_release, best_verdict, search_error ` type WantMissingAlbumParams struct { @@ -249,6 +249,12 @@ func (q *Queries) WantMissingAlbum(ctx context.Context, arg WantMissingAlbumPara &i.FulfilledAt, &i.CreatedAt, &i.UpdatedAt, + &i.SearchedAt, + &i.SearchOutcome, + &i.SearchConsidered, + &i.BestRelease, + &i.BestVerdict, + &i.SearchError, ) return i, err } diff --git a/internal/db/dbq/search.sql.go b/internal/db/dbq/search.sql.go new file mode 100644 index 00000000..b1043de2 --- /dev/null +++ b/internal/db/dbq/search.sql.go @@ -0,0 +1,585 @@ +// Code generated by sqlc. DO NOT EDIT. +// versions: +// sqlc v1.31.1 +// source: search.sql + +package dbq + +import ( + "context" + + "github.com/jackc/pgx/v5/pgtype" +) + +const blockRelease = `-- name: BlockRelease :exec +INSERT INTO release_blocks (info_hash, name, reason, detail, want_id, blocked_by) +VALUES ($1, $2, $3, $4, + $5, $6) +ON CONFLICT (info_hash) DO UPDATE + SET reason = EXCLUDED.reason, detail = EXCLUDED.detail, blocked_at = now() +` + +type BlockReleaseParams struct { + InfoHash string + Name string + Reason string + Detail string + WantID pgtype.UUID + BlockedBy pgtype.UUID +} + +func (q *Queries) BlockRelease(ctx context.Context, arg BlockReleaseParams) error { + _, err := q.db.Exec(ctx, blockRelease, + arg.InfoHash, + arg.Name, + arg.Reason, + arg.Detail, + arg.WantID, + arg.BlockedBy, + ) + return err +} + +const deleteWantCandidates = `-- name: DeleteWantCandidates :exec +DELETE FROM want_candidates WHERE want_id = $1 +` + +func (q *Queries) DeleteWantCandidates(ctx context.Context, wantID pgtype.UUID) error { + _, err := q.db.Exec(ctx, deleteWantCandidates, wantID) + return err +} + +const editionTrackSpan = `-- name: EditionTrackSpan :one +SELECT COALESCE(MIN(track_count) FILTER (WHERE status = 'Official'), MIN(track_count), 0)::int AS min_tracks, + COALESCE(MAX(track_count), 0)::int AS max_tracks + FROM mb_releases + WHERE release_group_mbid = $1 AND track_count > 0 +` + +type EditionTrackSpanRow struct { + MinTracks int32 + MaxTracks int32 +} + +// The fewest tracks an official edition of the group has (any edition when +// none is official), and the most any has. Zero when none is catalogued. +func (q *Queries) EditionTrackSpan(ctx context.Context, releaseGroupMbid string) (EditionTrackSpanRow, error) { + row := q.db.QueryRow(ctx, editionTrackSpan, releaseGroupMbid) + var i EditionTrackSpanRow + err := row.Scan(&i.MinTracks, &i.MaxTracks) + return i, err +} + +const endGrab = `-- name: EndGrab :exec +UPDATE grabs + SET ended_at = now(), end_reason = $1, end_detail = $2 + WHERE id = $3 AND ended_at IS NULL +` + +type EndGrabParams struct { + EndReason string + EndDetail string + ID pgtype.UUID +} + +func (q *Queries) EndGrab(ctx context.Context, arg EndGrabParams) error { + _, err := q.db.Exec(ctx, endGrab, arg.EndReason, arg.EndDetail, arg.ID) + return err +} + +const getAcquisitionSettings = `-- name: GetAcquisitionSettings :one + +SELECT id, auto_grab, search_interval_hours, allowed_qualities, last_pass_at, last_pass, updated_at FROM acquisition_settings WHERE id = true +` + +// Searching for wants and grabbing releases (M502 #5521). +func (q *Queries) GetAcquisitionSettings(ctx context.Context) (AcquisitionSetting, error) { + row := q.db.QueryRow(ctx, getAcquisitionSettings) + var i AcquisitionSetting + err := row.Scan( + &i.ID, + &i.AutoGrab, + &i.SearchIntervalHours, + &i.AllowedQualities, + &i.LastPassAt, + &i.LastPass, + &i.UpdatedAt, + ) + return i, err +} + +const getLiveGrab = `-- name: GetLiveGrab :one +SELECT id, want_id, info_hash, name, key, indexer, source, client, save_path, quality, origin, forced, grabbed_by, grabbed_at, ended_at, end_reason, end_detail FROM grabs WHERE want_id = $1 AND ended_at IS NULL +` + +func (q *Queries) GetLiveGrab(ctx context.Context, wantID pgtype.UUID) (Grab, error) { + row := q.db.QueryRow(ctx, getLiveGrab, wantID) + var i Grab + err := row.Scan( + &i.ID, + &i.WantID, + &i.InfoHash, + &i.Name, + &i.Key, + &i.Indexer, + &i.Source, + &i.Client, + &i.SavePath, + &i.Quality, + &i.Origin, + &i.Forced, + &i.GrabbedBy, + &i.GrabbedAt, + &i.EndedAt, + &i.EndReason, + &i.EndDetail, + ) + return i, err +} + +const getWantCandidate = `-- name: GetWantCandidate :one +SELECT want_id, key, name, info_hash, indexer, source, size_bytes, seeders, files_count, format, quality, year, verdict, reason, reasons, folded_from, rank, searched_at FROM want_candidates WHERE want_id = $1 AND key = $2 +` + +type GetWantCandidateParams struct { + WantID pgtype.UUID + Key string +} + +func (q *Queries) GetWantCandidate(ctx context.Context, arg GetWantCandidateParams) (WantCandidate, error) { + row := q.db.QueryRow(ctx, getWantCandidate, arg.WantID, arg.Key) + var i WantCandidate + err := row.Scan( + &i.WantID, + &i.Key, + &i.Name, + &i.InfoHash, + &i.Indexer, + &i.Source, + &i.SizeBytes, + &i.Seeders, + &i.FilesCount, + &i.Format, + &i.Quality, + &i.Year, + &i.Verdict, + &i.Reason, + &i.Reasons, + &i.FoldedFrom, + &i.Rank, + &i.SearchedAt, + ) + return i, err +} + +const getWantForSearch = `-- name: GetWantForSearch :one +SELECT w.id, w.release_group_mbid, w.artist_name, w.title, w.reason, w.fulfilled_at, + COALESCE(NULLIF(w.primary_type, ''), g.primary_type, '')::text AS primary_type, + COALESCE(g.secondary_types, '{}')::text[] AS secondary_types, + COALESCE(NULLIF(w.first_release_date, ''), g.first_release_date, '')::text AS first_release_date + FROM wants w + LEFT JOIN mb_release_groups g ON g.mbid = w.release_group_mbid + WHERE w.id = $1 +` + +type GetWantForSearchRow struct { + ID pgtype.UUID + ReleaseGroupMbid string + ArtistName string + Title string + Reason string + FulfilledAt pgtype.Timestamptz + PrimaryType string + SecondaryTypes []string + FirstReleaseDate string +} + +func (q *Queries) GetWantForSearch(ctx context.Context, id pgtype.UUID) (GetWantForSearchRow, error) { + row := q.db.QueryRow(ctx, getWantForSearch, id) + var i GetWantForSearchRow + err := row.Scan( + &i.ID, + &i.ReleaseGroupMbid, + &i.ArtistName, + &i.Title, + &i.Reason, + &i.FulfilledAt, + &i.PrimaryType, + &i.SecondaryTypes, + &i.FirstReleaseDate, + ) + return i, err +} + +const insertGrab = `-- name: InsertGrab :one +INSERT INTO grabs (want_id, info_hash, name, key, indexer, source, client, save_path, quality, + origin, forced, grabbed_by) +VALUES ($1, $2, $3, $4, $5, + $6, $7, $8, $9, + $10, $11, $12) +RETURNING id, want_id, info_hash, name, key, indexer, source, client, save_path, quality, origin, forced, grabbed_by, grabbed_at, ended_at, end_reason, end_detail +` + +type InsertGrabParams struct { + WantID pgtype.UUID + InfoHash string + Name string + Key string + Indexer string + Source string + Client string + SavePath string + Quality string + Origin string + Forced bool + GrabbedBy pgtype.UUID +} + +func (q *Queries) InsertGrab(ctx context.Context, arg InsertGrabParams) (Grab, error) { + row := q.db.QueryRow(ctx, insertGrab, + arg.WantID, + arg.InfoHash, + arg.Name, + arg.Key, + arg.Indexer, + arg.Source, + arg.Client, + arg.SavePath, + arg.Quality, + arg.Origin, + arg.Forced, + arg.GrabbedBy, + ) + var i Grab + err := row.Scan( + &i.ID, + &i.WantID, + &i.InfoHash, + &i.Name, + &i.Key, + &i.Indexer, + &i.Source, + &i.Client, + &i.SavePath, + &i.Quality, + &i.Origin, + &i.Forced, + &i.GrabbedBy, + &i.GrabbedAt, + &i.EndedAt, + &i.EndReason, + &i.EndDetail, + ) + return i, err +} + +const insertWantCandidate = `-- name: InsertWantCandidate :exec +INSERT INTO want_candidates (want_id, key, name, info_hash, indexer, source, size_bytes, seeders, + files_count, format, quality, year, verdict, reason, reasons, folded_from, rank) +VALUES ($1, $2, $3, $4, $5, + $6, $7, $8, $9, $10, + $11, $12, $13, $14, $15, + $16, $17) +ON CONFLICT (want_id, key) DO NOTHING +` + +type InsertWantCandidateParams struct { + WantID pgtype.UUID + Key string + Name string + InfoHash string + Indexer string + Source string + SizeBytes int64 + Seeders int32 + FilesCount int32 + Format string + Quality string + Year int32 + Verdict string + Reason string + Reasons []byte + FoldedFrom []string + Rank int32 +} + +func (q *Queries) InsertWantCandidate(ctx context.Context, arg InsertWantCandidateParams) error { + _, err := q.db.Exec(ctx, insertWantCandidate, + arg.WantID, + arg.Key, + arg.Name, + arg.InfoHash, + arg.Indexer, + arg.Source, + arg.SizeBytes, + arg.Seeders, + arg.FilesCount, + arg.Format, + arg.Quality, + arg.Year, + arg.Verdict, + arg.Reason, + arg.Reasons, + arg.FoldedFrom, + arg.Rank, + ) + return err +} + +const listReleaseBlocks = `-- name: ListReleaseBlocks :many +SELECT info_hash, name, reason, detail, want_id, blocked_by, blocked_at FROM release_blocks ORDER BY blocked_at DESC +` + +func (q *Queries) ListReleaseBlocks(ctx context.Context) ([]ReleaseBlock, error) { + rows, err := q.db.Query(ctx, listReleaseBlocks) + if err != nil { + return nil, err + } + defer rows.Close() + var items []ReleaseBlock + for rows.Next() { + var i ReleaseBlock + if err := rows.Scan( + &i.InfoHash, + &i.Name, + &i.Reason, + &i.Detail, + &i.WantID, + &i.BlockedBy, + &i.BlockedAt, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const listWantCandidates = `-- name: ListWantCandidates :many +SELECT want_id, key, name, info_hash, indexer, source, size_bytes, seeders, files_count, format, quality, year, verdict, reason, reasons, folded_from, rank, searched_at FROM want_candidates WHERE want_id = $1 ORDER BY rank, key +` + +func (q *Queries) ListWantCandidates(ctx context.Context, wantID pgtype.UUID) ([]WantCandidate, error) { + rows, err := q.db.Query(ctx, listWantCandidates, wantID) + if err != nil { + return nil, err + } + defer rows.Close() + var items []WantCandidate + for rows.Next() { + var i WantCandidate + if err := rows.Scan( + &i.WantID, + &i.Key, + &i.Name, + &i.InfoHash, + &i.Indexer, + &i.Source, + &i.SizeBytes, + &i.Seeders, + &i.FilesCount, + &i.Format, + &i.Quality, + &i.Year, + &i.Verdict, + &i.Reason, + &i.Reasons, + &i.FoldedFrom, + &i.Rank, + &i.SearchedAt, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const listWantsDueForSearch = `-- name: ListWantsDueForSearch :many +SELECT w.id, w.release_group_mbid, w.artist_name, w.title, w.reason, + COALESCE(NULLIF(w.primary_type, ''), g.primary_type, '')::text AS primary_type, + COALESCE(g.secondary_types, '{}')::text[] AS secondary_types, + COALESCE(NULLIF(w.first_release_date, ''), g.first_release_date, '')::text AS first_release_date + FROM wants w + LEFT JOIN mb_release_groups g ON g.mbid = w.release_group_mbid + WHERE w.fulfilled_at IS NULL + AND NOT EXISTS (SELECT 1 FROM grabs gr WHERE gr.want_id = w.id AND gr.ended_at IS NULL) + AND (w.searched_at IS NULL OR w.searched_at < now() - make_interval(hours => $1::int)) + ORDER BY w.searched_at IS NOT NULL, + CASE w.reason WHEN 'request' THEN 0 WHEN 'missing' THEN 1 ELSE 2 END, + w.searched_at NULLS FIRST, w.wanted_at, w.id + LIMIT $2 +` + +type ListWantsDueForSearchParams struct { + Hours int32 + MaxRows int32 +} + +type ListWantsDueForSearchRow struct { + ID pgtype.UUID + ReleaseGroupMbid string + ArtistName string + Title string + Reason string + PrimaryType string + SecondaryTypes []string + FirstReleaseDate string +} + +// Open wants with no download running that were never searched or not since +// @hours ago: never-searched first, then requests, then missing files, then +// followed artists' releases. +func (q *Queries) ListWantsDueForSearch(ctx context.Context, arg ListWantsDueForSearchParams) ([]ListWantsDueForSearchRow, error) { + rows, err := q.db.Query(ctx, listWantsDueForSearch, arg.Hours, arg.MaxRows) + if err != nil { + return nil, err + } + defer rows.Close() + var items []ListWantsDueForSearchRow + for rows.Next() { + var i ListWantsDueForSearchRow + if err := rows.Scan( + &i.ID, + &i.ReleaseGroupMbid, + &i.ArtistName, + &i.Title, + &i.Reason, + &i.PrimaryType, + &i.SecondaryTypes, + &i.FirstReleaseDate, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const markCandidatesBlocked = `-- name: MarkCandidatesBlocked :exec +UPDATE want_candidates + SET verdict = 'blocked', reason = $1, + reasons = jsonb_build_array(jsonb_build_object('kind', 'hard', 'code', 'blocked', 'text', $1::text)) + WHERE info_hash = $2 +` + +type MarkCandidatesBlockedParams struct { + Reason string + InfoHash string +} + +func (q *Queries) MarkCandidatesBlocked(ctx context.Context, arg MarkCandidatesBlockedParams) error { + _, err := q.db.Exec(ctx, markCandidatesBlocked, arg.Reason, arg.InfoHash) + return err +} + +const markCandidatesUnblocked = `-- name: MarkCandidatesUnblocked :exec +UPDATE want_candidates + SET verdict = 'review', reason = $1, + reasons = jsonb_build_array(jsonb_build_object('kind', 'soft', 'code', 'unblocked', 'text', $1::text)) + WHERE info_hash = $2 AND verdict = 'blocked' +` + +type MarkCandidatesUnblockedParams struct { + Reason string + InfoHash string +} + +func (q *Queries) MarkCandidatesUnblocked(ctx context.Context, arg MarkCandidatesUnblockedParams) error { + _, err := q.db.Exec(ctx, markCandidatesUnblocked, arg.Reason, arg.InfoHash) + return err +} + +const recordSearchPass = `-- name: RecordSearchPass :exec +UPDATE acquisition_settings SET last_pass_at = now(), last_pass = $1 WHERE id = true +` + +func (q *Queries) RecordSearchPass(ctx context.Context, report []byte) error { + _, err := q.db.Exec(ctx, recordSearchPass, report) + return err +} + +const recordWantSearch = `-- name: RecordWantSearch :exec +UPDATE wants + SET searched_at = now(), + search_outcome = $1, + search_considered = $2, + best_release = $3, + best_verdict = $4, + search_error = $5, + updated_at = now() + WHERE id = $6 +` + +type RecordWantSearchParams struct { + Outcome string + Considered int32 + BestRelease string + BestVerdict string + SearchError string + ID pgtype.UUID +} + +func (q *Queries) RecordWantSearch(ctx context.Context, arg RecordWantSearchParams) error { + _, err := q.db.Exec(ctx, recordWantSearch, + arg.Outcome, + arg.Considered, + arg.BestRelease, + arg.BestVerdict, + arg.SearchError, + arg.ID, + ) + return err +} + +const unblockRelease = `-- name: UnblockRelease :execrows +DELETE FROM release_blocks WHERE info_hash = $1 +` + +func (q *Queries) UnblockRelease(ctx context.Context, infoHash string) (int64, error) { + result, err := q.db.Exec(ctx, unblockRelease, infoHash) + if err != nil { + return 0, err + } + return result.RowsAffected(), nil +} + +const updateAcquisitionSettings = `-- name: UpdateAcquisitionSettings :one +UPDATE acquisition_settings + SET auto_grab = $1, + search_interval_hours = $2, + allowed_qualities = $3, + updated_at = now() + WHERE id = true +RETURNING id, auto_grab, search_interval_hours, allowed_qualities, last_pass_at, last_pass, updated_at +` + +type UpdateAcquisitionSettingsParams struct { + AutoGrab bool + SearchIntervalHours int32 + AllowedQualities []string +} + +func (q *Queries) UpdateAcquisitionSettings(ctx context.Context, arg UpdateAcquisitionSettingsParams) (AcquisitionSetting, error) { + row := q.db.QueryRow(ctx, updateAcquisitionSettings, arg.AutoGrab, arg.SearchIntervalHours, arg.AllowedQualities) + var i AcquisitionSetting + err := row.Scan( + &i.ID, + &i.AutoGrab, + &i.SearchIntervalHours, + &i.AllowedQualities, + &i.LastPassAt, + &i.LastPass, + &i.UpdatedAt, + ) + return i, err +} diff --git a/internal/db/dbq/wanted.sql.go b/internal/db/dbq/wanted.sql.go index 38414108..1374189c 100644 --- a/internal/db/dbq/wanted.sql.go +++ b/internal/db/dbq/wanted.sql.go @@ -206,7 +206,7 @@ func (q *Queries) GetReleaseTypeProfile(ctx context.Context, id pgtype.UUID) (Re } const getWantByReleaseGroup = `-- name: GetWantByReleaseGroup :one -SELECT id, release_group_mbid, artist_mbid, artist_name, title, primary_type, first_release_date, reason, album_id, created_by, wanted_at, fulfilled_at, created_at, updated_at FROM wants WHERE release_group_mbid = $1 +SELECT id, release_group_mbid, artist_mbid, artist_name, title, primary_type, first_release_date, reason, album_id, created_by, wanted_at, fulfilled_at, created_at, updated_at, searched_at, search_outcome, search_considered, best_release, best_verdict, search_error FROM wants WHERE release_group_mbid = $1 ` func (q *Queries) GetWantByReleaseGroup(ctx context.Context, releaseGroupMbid string) (Want, error) { @@ -227,6 +227,12 @@ func (q *Queries) GetWantByReleaseGroup(ctx context.Context, releaseGroupMbid st &i.FulfilledAt, &i.CreatedAt, &i.UpdatedAt, + &i.SearchedAt, + &i.SearchOutcome, + &i.SearchConsidered, + &i.BestRelease, + &i.BestVerdict, + &i.SearchError, ) return i, err } @@ -406,7 +412,7 @@ func (q *Queries) ListFollowsDue(ctx context.Context, arg ListFollowsDueParams) } const listOpenWants = `-- name: ListOpenWants :many -SELECT id, release_group_mbid, artist_mbid, artist_name, title, primary_type, first_release_date, reason, album_id, created_by, wanted_at, fulfilled_at, created_at, updated_at FROM wants +SELECT id, release_group_mbid, artist_mbid, artist_name, title, primary_type, first_release_date, reason, album_id, created_by, wanted_at, fulfilled_at, created_at, updated_at, searched_at, search_outcome, search_considered, best_release, best_verdict, search_error FROM wants WHERE fulfilled_at IS NULL ORDER BY wanted_at, id LIMIT $1 @@ -436,6 +442,12 @@ func (q *Queries) ListOpenWants(ctx context.Context, limit int32) ([]Want, error &i.FulfilledAt, &i.CreatedAt, &i.UpdatedAt, + &i.SearchedAt, + &i.SearchOutcome, + &i.SearchConsidered, + &i.BestRelease, + &i.BestVerdict, + &i.SearchError, ); err != nil { return nil, err } @@ -505,10 +517,14 @@ SELECT w.id, w.release_group_mbid, w.artist_mbid, w.artist_name, w.title, w.reas COALESCE(g.secondary_types, '{}')::text[] AS secondary_types, COALESCE(NULLIF(w.first_release_date, ''), g.first_release_date, '')::text AS first_release_date, u.username AS requested_by, - (SELECT ar.id FROM artists ar WHERE ar.mbid = w.artist_mbid) AS library_artist_id + (SELECT ar.id FROM artists ar WHERE ar.mbid = w.artist_mbid) AS library_artist_id, + -- The search schedule's last look (#5521), and the download running. + w.searched_at, w.search_outcome, w.search_considered, w.best_release, w.best_verdict, w.search_error, + gr.name AS grab_name, gr.grabbed_at, gr.origin AS grab_origin, gr.quality AS grab_quality FROM wants w LEFT JOIN users u ON u.id = w.created_by LEFT JOIN mb_release_groups g ON g.mbid = w.release_group_mbid + LEFT JOIN grabs gr ON gr.want_id = w.id AND gr.ended_at IS NULL WHERE CASE $1::text WHEN 'held' THEN w.fulfilled_at IS NOT NULL WHEN 'missing' THEN w.fulfilled_at IS NULL AND w.reason = 'missing' @@ -539,6 +555,16 @@ type ListWantsRow struct { FirstReleaseDate string RequestedBy *string LibraryArtistID pgtype.UUID + SearchedAt pgtype.Timestamptz + SearchOutcome string + SearchConsidered int32 + BestRelease string + BestVerdict string + SearchError string + GrabName *string + GrabbedAt pgtype.Timestamptz + GrabOrigin *string + GrabQuality *string } // The Wanted page: open wants (all but missing ones, or only missing ones), @@ -567,6 +593,16 @@ func (q *Queries) ListWants(ctx context.Context, arg ListWantsParams) ([]ListWan &i.FirstReleaseDate, &i.RequestedBy, &i.LibraryArtistID, + &i.SearchedAt, + &i.SearchOutcome, + &i.SearchConsidered, + &i.BestRelease, + &i.BestVerdict, + &i.SearchError, + &i.GrabName, + &i.GrabbedAt, + &i.GrabOrigin, + &i.GrabQuality, ); err != nil { return nil, err } @@ -791,7 +827,7 @@ INSERT INTO wants (release_group_mbid, artist_mbid, artist_name, title, primary_ VALUES ($1, $2, $3, $4, $5, $6, $7, $8) ON CONFLICT (release_group_mbid) DO UPDATE SET updated_at = wants.updated_at -RETURNING id, release_group_mbid, artist_mbid, artist_name, title, primary_type, first_release_date, reason, album_id, created_by, wanted_at, fulfilled_at, created_at, updated_at +RETURNING id, release_group_mbid, artist_mbid, artist_name, title, primary_type, first_release_date, reason, album_id, created_by, wanted_at, fulfilled_at, created_at, updated_at, searched_at, search_outcome, search_considered, best_release, best_verdict, search_error ` type WantReleaseGroupParams struct { @@ -834,6 +870,12 @@ func (q *Queries) WantReleaseGroup(ctx context.Context, arg WantReleaseGroupPara &i.FulfilledAt, &i.CreatedAt, &i.UpdatedAt, + &i.SearchedAt, + &i.SearchOutcome, + &i.SearchConsidered, + &i.BestRelease, + &i.BestVerdict, + &i.SearchError, ) return i, err } diff --git a/internal/db/migrations/0082_search_and_grab.down.sql b/internal/db/migrations/0082_search_and_grab.down.sql new file mode 100644 index 00000000..630d58e6 --- /dev/null +++ b/internal/db/migrations/0082_search_and_grab.down.sql @@ -0,0 +1,12 @@ +DROP TABLE release_blocks; +DROP TABLE grabs; +DROP TABLE want_candidates; +DROP INDEX wants_search_due_idx; +ALTER TABLE wants + DROP COLUMN searched_at, + DROP COLUMN search_outcome, + DROP COLUMN search_considered, + DROP COLUMN best_release, + DROP COLUMN best_verdict, + DROP COLUMN search_error; +DROP TABLE acquisition_settings; diff --git a/internal/db/migrations/0082_search_and_grab.up.sql b/internal/db/migrations/0082_search_and_grab.up.sql new file mode 100644 index 00000000..4c20ef3f --- /dev/null +++ b/internal/db/migrations/0082_search_and_grab.up.sql @@ -0,0 +1,102 @@ +-- 0082_search_and_grab.up.sql — searching for wants and grabbing releases +-- (Scribe milestone #502, #5521). +-- +-- acquisition_settings one row: whether the schedule takes the best release +-- by itself, how often a want is searched again, which +-- quality tiers are taken, and the last pass's report. +-- wants (columns) when each want was last searched and what came of it. +-- want_candidates every release the last search found for a want, with +-- its verdict and the reasons behind it. +-- grabs downloads started for wants; at most one live per want. +-- release_blocks releases never to take again, by infohash. + +CREATE TABLE acquisition_settings ( + id boolean PRIMARY KEY DEFAULT true CONSTRAINT acquisition_settings_one_row CHECK (id), + auto_grab boolean NOT NULL DEFAULT true, + -- 0 switches the schedule off. + search_interval_hours integer NOT NULL DEFAULT 24 + CONSTRAINT acquisition_settings_interval_check CHECK (search_interval_hours BETWEEN 0 AND 720), + -- Lidarr's "Any" profile (note #5527): trash lossy to lossless; unknown off. + allowed_qualities text[] NOT NULL DEFAULT ARRAY['lossless_hires', 'lossless', 'lossy_high', + 'lossy_standard', 'lossy_low', 'lossy_poor', 'lossy_trash'], + last_pass_at timestamptz, + last_pass jsonb NOT NULL DEFAULT '{}', + updated_at timestamptz NOT NULL DEFAULT now() +); + +INSERT INTO acquisition_settings DEFAULT VALUES; + +ALTER TABLE wants + ADD COLUMN searched_at timestamptz, + ADD COLUMN search_outcome text NOT NULL DEFAULT '' + CONSTRAINT wants_search_outcome_check CHECK (search_outcome IN ('', 'grabbed', 'waiting', 'nothing', 'error')), + ADD COLUMN search_considered integer NOT NULL DEFAULT 0, + ADD COLUMN best_release text NOT NULL DEFAULT '', + ADD COLUMN best_verdict text NOT NULL DEFAULT '', + ADD COLUMN search_error text NOT NULL DEFAULT ''; + +CREATE INDEX wants_search_due_idx ON wants (searched_at NULLS FIRST) WHERE fulfilled_at IS NULL; + +CREATE TABLE want_candidates ( + want_id uuid NOT NULL REFERENCES wants (id) ON DELETE CASCADE, + -- The infohash, or the name when the source gave none. + key text NOT NULL, + name text NOT NULL, + info_hash text NOT NULL DEFAULT '', + indexer text NOT NULL DEFAULT '', + source text NOT NULL DEFAULT '', + size_bytes bigint NOT NULL DEFAULT 0, + seeders integer NOT NULL DEFAULT 0, + files_count integer NOT NULL DEFAULT 0, + format text NOT NULL DEFAULT '', + quality text NOT NULL DEFAULT '', + year integer NOT NULL DEFAULT 0, + verdict text NOT NULL + CONSTRAINT want_candidates_verdict_check CHECK (verdict IN ('grab', 'review', 'refused', 'rejected', 'blocked')), + reason text NOT NULL DEFAULT '', + reasons jsonb NOT NULL DEFAULT '[]', + folded_from text[] NOT NULL DEFAULT '{}', + -- Position in the search's order, best first. + rank integer NOT NULL DEFAULT 0, + searched_at timestamptz NOT NULL DEFAULT now(), + PRIMARY KEY (want_id, key) +); + +CREATE INDEX want_candidates_info_hash_idx ON want_candidates (info_hash) WHERE info_hash <> ''; + +CREATE TABLE grabs ( + id uuid PRIMARY KEY DEFAULT gen_random_uuid(), + want_id uuid REFERENCES wants (id) ON DELETE SET NULL, + info_hash text NOT NULL DEFAULT '', + name text NOT NULL, + key text NOT NULL, + indexer text NOT NULL DEFAULT '', + source text NOT NULL DEFAULT '', + -- The acquisition_services row of the download client it went to. + client text NOT NULL, + save_path text NOT NULL, + quality text NOT NULL DEFAULT '', + origin text NOT NULL CONSTRAINT grabs_origin_check CHECK (origin IN ('schedule', 'operator')), + forced boolean NOT NULL DEFAULT false, + grabbed_by uuid REFERENCES users (id) ON DELETE SET NULL, + grabbed_at timestamptz NOT NULL DEFAULT now(), + ended_at timestamptz, + end_reason text NOT NULL DEFAULT '' + CONSTRAINT grabs_end_reason_check CHECK (end_reason IN ('', 'superseded')), + end_detail text NOT NULL DEFAULT '' +); + +-- One live grab per want; the service decides which, this is the backstop. +CREATE UNIQUE INDEX grabs_one_live_per_want ON grabs (want_id) WHERE ended_at IS NULL AND want_id IS NOT NULL; +CREATE INDEX grabs_info_hash_idx ON grabs (info_hash) WHERE info_hash <> ''; + +CREATE TABLE release_blocks ( + info_hash text PRIMARY KEY, + name text NOT NULL DEFAULT '', + -- operator: "never this one"; files: its .torrent held the wrong thing. + reason text NOT NULL CONSTRAINT release_blocks_reason_check CHECK (reason IN ('operator', 'files')), + detail text NOT NULL DEFAULT '', + want_id uuid REFERENCES wants (id) ON DELETE SET NULL, + blocked_by uuid REFERENCES users (id) ON DELETE SET NULL, + blocked_at timestamptz NOT NULL DEFAULT now() +); diff --git a/internal/db/queries/search.sql b/internal/db/queries/search.sql new file mode 100644 index 00000000..832e35f2 --- /dev/null +++ b/internal/db/queries/search.sql @@ -0,0 +1,121 @@ +-- Searching for wants and grabbing releases (M502 #5521). + +-- name: GetAcquisitionSettings :one +SELECT * FROM acquisition_settings WHERE id = true; + +-- name: UpdateAcquisitionSettings :one +UPDATE acquisition_settings + SET auto_grab = sqlc.arg(auto_grab), + search_interval_hours = sqlc.arg(search_interval_hours), + allowed_qualities = sqlc.arg(allowed_qualities), + updated_at = now() + WHERE id = true +RETURNING *; + +-- name: RecordSearchPass :exec +UPDATE acquisition_settings SET last_pass_at = now(), last_pass = sqlc.arg(report) WHERE id = true; + +-- name: ListWantsDueForSearch :many +-- Open wants with no download running that were never searched or not since +-- @hours ago: never-searched first, then requests, then missing files, then +-- followed artists' releases. +SELECT w.id, w.release_group_mbid, w.artist_name, w.title, w.reason, + COALESCE(NULLIF(w.primary_type, ''), g.primary_type, '')::text AS primary_type, + COALESCE(g.secondary_types, '{}')::text[] AS secondary_types, + COALESCE(NULLIF(w.first_release_date, ''), g.first_release_date, '')::text AS first_release_date + FROM wants w + LEFT JOIN mb_release_groups g ON g.mbid = w.release_group_mbid + WHERE w.fulfilled_at IS NULL + AND NOT EXISTS (SELECT 1 FROM grabs gr WHERE gr.want_id = w.id AND gr.ended_at IS NULL) + AND (w.searched_at IS NULL OR w.searched_at < now() - make_interval(hours => sqlc.arg(hours)::int)) + ORDER BY w.searched_at IS NOT NULL, + CASE w.reason WHEN 'request' THEN 0 WHEN 'missing' THEN 1 ELSE 2 END, + w.searched_at NULLS FIRST, w.wanted_at, w.id + LIMIT sqlc.arg(max_rows); + +-- name: GetWantForSearch :one +SELECT w.id, w.release_group_mbid, w.artist_name, w.title, w.reason, w.fulfilled_at, + COALESCE(NULLIF(w.primary_type, ''), g.primary_type, '')::text AS primary_type, + COALESCE(g.secondary_types, '{}')::text[] AS secondary_types, + COALESCE(NULLIF(w.first_release_date, ''), g.first_release_date, '')::text AS first_release_date + FROM wants w + LEFT JOIN mb_release_groups g ON g.mbid = w.release_group_mbid + WHERE w.id = $1; + +-- name: EditionTrackSpan :one +-- The fewest tracks an official edition of the group has (any edition when +-- none is official), and the most any has. Zero when none is catalogued. +SELECT COALESCE(MIN(track_count) FILTER (WHERE status = 'Official'), MIN(track_count), 0)::int AS min_tracks, + COALESCE(MAX(track_count), 0)::int AS max_tracks + FROM mb_releases + WHERE release_group_mbid = $1 AND track_count > 0; + +-- name: RecordWantSearch :exec +UPDATE wants + SET searched_at = now(), + search_outcome = sqlc.arg(outcome), + search_considered = sqlc.arg(considered), + best_release = sqlc.arg(best_release), + best_verdict = sqlc.arg(best_verdict), + search_error = sqlc.arg(search_error), + updated_at = now() + WHERE id = sqlc.arg(id); + +-- name: DeleteWantCandidates :exec +DELETE FROM want_candidates WHERE want_id = $1; + +-- name: InsertWantCandidate :exec +INSERT INTO want_candidates (want_id, key, name, info_hash, indexer, source, size_bytes, seeders, + files_count, format, quality, year, verdict, reason, reasons, folded_from, rank) +VALUES (sqlc.arg(want_id), sqlc.arg(key), sqlc.arg(name), sqlc.arg(info_hash), sqlc.arg(indexer), + sqlc.arg(source), sqlc.arg(size_bytes), sqlc.arg(seeders), sqlc.arg(files_count), sqlc.arg(format), + sqlc.arg(quality), sqlc.arg(year), sqlc.arg(verdict), sqlc.arg(reason), sqlc.arg(reasons), + sqlc.arg(folded_from), sqlc.arg(rank)) +ON CONFLICT (want_id, key) DO NOTHING; + +-- name: ListWantCandidates :many +SELECT * FROM want_candidates WHERE want_id = $1 ORDER BY rank, key; + +-- name: GetWantCandidate :one +SELECT * FROM want_candidates WHERE want_id = $1 AND key = $2; + +-- name: MarkCandidatesBlocked :exec +UPDATE want_candidates + SET verdict = 'blocked', reason = sqlc.arg(reason), + reasons = jsonb_build_array(jsonb_build_object('kind', 'hard', 'code', 'blocked', 'text', sqlc.arg(reason)::text)) + WHERE info_hash = sqlc.arg(info_hash); + +-- name: MarkCandidatesUnblocked :exec +UPDATE want_candidates + SET verdict = 'review', reason = sqlc.arg(reason), + reasons = jsonb_build_array(jsonb_build_object('kind', 'soft', 'code', 'unblocked', 'text', sqlc.arg(reason)::text)) + WHERE info_hash = sqlc.arg(info_hash) AND verdict = 'blocked'; + +-- name: GetLiveGrab :one +SELECT * FROM grabs WHERE want_id = $1 AND ended_at IS NULL; + +-- name: InsertGrab :one +INSERT INTO grabs (want_id, info_hash, name, key, indexer, source, client, save_path, quality, + origin, forced, grabbed_by) +VALUES (sqlc.narg(want_id), sqlc.arg(info_hash), sqlc.arg(name), sqlc.arg(key), sqlc.arg(indexer), + sqlc.arg(source), sqlc.arg(client), sqlc.arg(save_path), sqlc.arg(quality), + sqlc.arg(origin), sqlc.arg(forced), sqlc.narg(grabbed_by)) +RETURNING *; + +-- name: EndGrab :exec +UPDATE grabs + SET ended_at = now(), end_reason = sqlc.arg(end_reason), end_detail = sqlc.arg(end_detail) + WHERE id = sqlc.arg(id) AND ended_at IS NULL; + +-- name: ListReleaseBlocks :many +SELECT * FROM release_blocks ORDER BY blocked_at DESC; + +-- name: BlockRelease :exec +INSERT INTO release_blocks (info_hash, name, reason, detail, want_id, blocked_by) +VALUES (sqlc.arg(info_hash), sqlc.arg(name), sqlc.arg(reason), sqlc.arg(detail), + sqlc.narg(want_id), sqlc.narg(blocked_by)) +ON CONFLICT (info_hash) DO UPDATE + SET reason = EXCLUDED.reason, detail = EXCLUDED.detail, blocked_at = now(); + +-- name: UnblockRelease :execrows +DELETE FROM release_blocks WHERE info_hash = $1; diff --git a/internal/db/queries/wanted.sql b/internal/db/queries/wanted.sql index e28463d9..bde3219d 100644 --- a/internal/db/queries/wanted.sql +++ b/internal/db/queries/wanted.sql @@ -186,10 +186,14 @@ SELECT w.id, w.release_group_mbid, w.artist_mbid, w.artist_name, w.title, w.reas COALESCE(g.secondary_types, '{}')::text[] AS secondary_types, COALESCE(NULLIF(w.first_release_date, ''), g.first_release_date, '')::text AS first_release_date, u.username AS requested_by, - (SELECT ar.id FROM artists ar WHERE ar.mbid = w.artist_mbid) AS library_artist_id + (SELECT ar.id FROM artists ar WHERE ar.mbid = w.artist_mbid) AS library_artist_id, + -- The search schedule's last look (#5521), and the download running. + w.searched_at, w.search_outcome, w.search_considered, w.best_release, w.best_verdict, w.search_error, + gr.name AS grab_name, gr.grabbed_at, gr.origin AS grab_origin, gr.quality AS grab_quality FROM wants w LEFT JOIN users u ON u.id = w.created_by LEFT JOIN mb_release_groups g ON g.mbid = w.release_group_mbid + LEFT JOIN grabs gr ON gr.want_id = w.id AND gr.ended_at IS NULL WHERE CASE sqlc.arg(state)::text WHEN 'held' THEN w.fulfilled_at IS NOT NULL WHEN 'missing' THEN w.fulfilled_at IS NULL AND w.reason = 'missing' diff --git a/internal/dbtest/reset.go b/internal/dbtest/reset.go index 540276b8..fd1b7865 100644 --- a/internal/dbtest/reset.go +++ b/internal/dbtest/reset.go @@ -109,6 +109,9 @@ var dataTables = []string{ "mb_release_groups", // M502 #5519 "mb_artists", // M502 #5519 "requests", // M502 #5520 (was lidarr_requests) + "release_blocks", // M502 #5521 + "grabs", // M502 #5521 + "want_candidates", // M502 #5521 "wants", // M502 #5520 "follow_seen_groups", // M502 #5520 "follows", // M502 #5520 @@ -180,6 +183,14 @@ func ResetDB(t *testing.T, pool *pgxpool.Pool) { ); err != nil { t.Fatalf("dbtest.ResetDB reset release-type profiles: %v", err) } + // Search and grab settings (M502 #5521), a singleton reset the same way. + if _, err := pool.Exec(ctx, ` + UPDATE acquisition_settings + SET auto_grab = DEFAULT, search_interval_hours = DEFAULT, allowed_qualities = DEFAULT, + last_pass_at = NULL, last_pass = DEFAULT, updated_at = DEFAULT`, + ); err != nil { + t.Fatalf("dbtest.ResetDB reset acquisition settings: %v", err) + } // Notification email settings (M489 #5346), reset the same way. if _, err := pool.Exec(ctx, ` UPDATE notification_email_settings diff --git a/internal/release/files.go b/internal/release/files.go new file mode 100644 index 00000000..2bfeca9f --- /dev/null +++ b/internal/release/files.go @@ -0,0 +1,76 @@ +package release + +import ( + "path" + "strconv" + "strings" +) + +// File is one file inside a torrent. +type File struct { + Index int + Path string + Size int64 +} + +// Contents is what a torrent's file list holds. +type Contents struct { + Audio int + Video int + // Format is the format most of the audio files are in, "" when none. + Format string +} + +var videoExt = map[string]bool{ + "mkv": true, "mp4": true, "avi": true, "m2ts": true, "vob": true, + "mov": true, "webm": true, "wmv": true, "m4v": true, "ts": true, +} + +// Inspect counts a file list's audio and video files. +func Inspect(files []File) Contents { + var c Contents + formats := map[string]int{} + for _, f := range files { + ext := strings.TrimPrefix(strings.ToLower(path.Ext(f.Path)), ".") + if fm := formatOf(ext); fm != "" { + c.Audio++ + formats[fm]++ + } else if videoExt[ext] { + c.Video++ + } + } + best := 0 + for fm, n := range formats { + if n > best || (n == best && formatRank[fm] > formatRank[c.Format]) { + c.Format, best = fm, n + } + } + return c +} + +// JudgeFiles says what a torrent's files say about whether it is the album: +// a veto (code and sentence), or a doubt for a person, or nothing. +func JudgeFiles(w Want, files []File) (veto, reason, doubt string) { + if len(files) == 0 { + return "", "", "" + } + c := Inspect(files) + switch { + case c.Audio == 0 && c.Video > 0: + return VetoVideo, "It holds video files and no audio.", "" + case c.Audio == 0: + return VetoNoAudio, "It holds no audio files Minstrel can read.", "" + case w.MinTracks > 0 && c.Audio < w.MinTracks: + return VetoIncomplete, "It holds " + tracks(c.Audio) + "; the shortest edition has " + tracks(w.MinTracks) + ".", "" + case w.MaxTracks > 0 && c.Audio > 2*w.MaxTracks+4: + return "", "", "It holds " + tracks(c.Audio) + ", more than any edition of the album has: probably more than one album." + } + return "", "", "" +} + +func tracks(n int) string { + if n == 1 { + return "1 track" + } + return strconv.Itoa(n) + " tracks" +} diff --git a/internal/release/match.go b/internal/release/match.go new file mode 100644 index 00000000..77ae1575 --- /dev/null +++ b/internal/release/match.go @@ -0,0 +1,210 @@ +package release + +import "strings" + +// Want is the album a search is for, as the catalogue describes it. +type Want struct { + Artist string + Title string + Year int + PrimaryType string // Album, EP, Single, Broadcast, Other + SecondaryTypes []string + // MinTracks and MaxTracks span the group's editions' track counts; zero + // when the catalogue has not read them. + MinTracks int + MaxTracks int +} + +// Veto codes: why a name is not the album. Kept on a candidate as its +// reason's code. +const ( + VetoArtist = "wrong_artist" + VetoTitle = "wrong_title" + VetoVideo = "video" + VetoDiscography = "discography" + VetoVersion = "other_version" + VetoType = "wrong_type" + VetoNoAudio = "no_audio" + VetoIncomplete = "incomplete" +) + +// Match is what a name says about whether it is the album wanted. +type Match struct { + // Veto is a code from above when the name is not the album, with Reason + // saying why in a sentence. + Veto string + Reason string + // Doubt is set when the name may be the album and cannot say for sure: a + // person decides. It is a sentence. + Doubt string + // YearAgrees: the name states the album's year, give or take one. + YearAgrees bool +} + +// versions are words that make a release a different recording of the +// album, with what to call it. A word the wanted title or artist holds does +// not count ("Live Through This", "Instrumental Core"). +var versions = []struct { + words []string + what string + // typ is the MusicBrainz secondary type that wants this version. + typ string +}{ + {[]string{"instrumental", "instrumentals"}, "an instrumental version", ""}, + {[]string{"drumless"}, "a drumless edition", ""}, + {[]string{"karaoke"}, "a karaoke version", ""}, + {[]string{"acapella", "acappella", "accapella", "acapellas"}, "an a cappella version", ""}, + {[]string{"remix", "remixes", "remixed"}, "a remix release", "Remix"}, + {[]string{"live", "bootleg"}, "a live recording", "Live"}, + {[]string{"demo", "demos"}, "demo recordings", "Demo"}, + // Not "covers": in a release name that is usually the artwork scans. + {[]string{"tribute", "lullaby", "lullabies"}, "a tribute release", ""}, + {[]string{"nightcore", "slowed", "sped"}, "a slowed or sped-up edit", ""}, + {[]string{"commentary", "interview", "interviews"}, "commentary or interviews", "Interview"}, +} + +// collections are words naming more than one album. +var collections = []string{"collection", "anthology", "albums", "boxset", "discography"} + +// Judge says whether a release name is the album wanted. It looks only at +// the name; JudgeFiles looks at what a torrent holds. +func Judge(w Want, r Release) Match { + var m Match + m.YearAgrees = r.Year != 0 && w.Year != 0 && abs(r.Year-w.Year) <= 1 + if r.Video != "" { + m.Veto, m.Reason = VetoVideo, "A video, not audio (“"+r.Video+"”)." + return m + } + artist := Tokens(w.Artist) + title := Tokens(w.Title) + if len(artist) > 0 && !contains(r.Tokens, artist) && !contains(r.Tokens, withoutThe(artist)) { + m.Veto, m.Reason = VetoArtist, "The name doesn’t mention "+w.Artist+"." + return m + } + if len(title) > 0 && !contains(r.Tokens, title) && !contains(r.Tokens, bare(w.Title)) { + m.Veto, m.Reason = VetoTitle, "The name doesn’t contain the title “"+w.Title+"”." + return m + } + own := map[string]bool{} + for _, t := range append(append([]string{}, title...), artist...) { + own[t] = true + } + if r.Discography != "" && !own[r.Discography] { + m.Veto, m.Reason = VetoDiscography, "A discography or collection ("+r.Discography+"), and one album is wanted." + return m + } + for _, c := range collections { + if !own[c] && has(r.Tokens, c) { + m.Veto, m.Reason = VetoDiscography, "A collection of albums (“"+c+"”), and one album is wanted." + return m + } + } + for _, v := range versions { + if v.typ != "" && hasType(w.SecondaryTypes, v.typ) { + continue + } + for _, word := range v.words { + if !own[word] && has(r.Tokens, word) { + m.Veto, m.Reason = VetoVersion, "This is "+v.what+" (“"+word+"”)." + return m + } + } + } + if w.PrimaryType == "Album" || w.PrimaryType == "EP" { + if !own["single"] && has(r.Tokens, "single") { + m.Veto, m.Reason = VetoType, "A single, and the "+strings.ToLower(w.PrimaryType)+" is wanted." + return m + } + } + if w.PrimaryType == "Album" && !own["ep"] && has(r.Tokens, "ep") { + m.Veto, m.Reason = VetoType, "An EP, and the album is wanted." + return m + } + // A self-titled album: every release of the artist names it once. Two + // mentions, or its year, say it is this one. + if len(title) > 0 && equal(title, artist) && count(r.Tokens, title) < 2 && !m.YearAgrees { + m.Doubt = "The album shares the artist’s name, and nothing else in the name says it is this album." + } + return m +} + +func abs(n int) int { + if n < 0 { + return -n + } + return n +} + +func withoutThe(t []string) []string { + if len(t) > 1 && t[0] == "the" { + return t[1:] + } + return t +} + +// bare is a title without a trailing bracketed part ("Album (Remastered)"), +// for names that leave it off. +func bare(title string) []string { + if i := strings.IndexAny(title, "(["); i > 0 { + return Tokens(title[:i]) + } + return nil +} + +// contains says whether want's words appear in name in order and together, +// or, for two or more words, run together ("ACDC" for AC/DC). A single word +// is never looked for run together: "muse" is inside "museum". +func contains(name, want []string) bool { + if len(want) == 0 { + return false + } + if count(name, want) > 0 { + return true + } + joined := strings.Join(want, "") + if len(want) < 2 || len(joined) < 4 { + return false + } + return strings.Contains(strings.Join(name, ""), joined) +} + +// count is how often want's words appear in name in order and together. +func count(name, want []string) int { + n := 0 + for i := 0; i+len(want) <= len(name); i++ { + if equal(name[i:i+len(want)], want) { + n++ + } + } + return n +} + +func equal(a, b []string) bool { + if len(a) != len(b) { + return false + } + for i := range a { + if a[i] != b[i] { + return false + } + } + return true +} + +func has(tokens []string, word string) bool { + for _, t := range tokens { + if t == word { + return true + } + } + return false +} + +func hasType(types []string, t string) bool { + for _, x := range types { + if strings.EqualFold(x, t) { + return true + } + } + return false +} diff --git a/internal/release/parse.go b/internal/release/parse.go new file mode 100644 index 00000000..956d548d --- /dev/null +++ b/internal/release/parse.go @@ -0,0 +1,289 @@ +// Package release reads music release names and judges them against a +// wanted album (M502 #5521): what a name says it is (format, bitrate, bit +// depth, source, year, edition), whether it is the album wanted, and which +// quality tier it sits in. Nothing here touches the network or the database, +// so every rule is testable on a string. +package release + +import ( + "regexp" + "strconv" + "strings" +) + +// Formats a name can state. +const ( + FormatFLAC = "FLAC" + FormatALAC = "ALAC" + FormatAPE = "APE" + FormatWV = "WavPack" + FormatWAV = "WAV" + FormatAIFF = "AIFF" + FormatDSD = "DSD" + FormatMP3 = "MP3" + FormatAAC = "AAC" + FormatVorbis = "Vorbis" + FormatOpus = "Opus" +) + +// Sources a name can state. +const ( + SourceWEB = "WEB" + SourceCD = "CD" + SourceVinyl = "Vinyl" + SourceSACD = "SACD" + SourceCassette = "Cassette" +) + +// Release is what a release name says about itself. Zero values mean the +// name does not say. +type Release struct { + Name string + Year int + Format string + // Bitrate is a constant bitrate in kbps; VBR is a LAME preset (V0, V1, V2). + Bitrate int + VBR string + // BitDepth (16, 24) and SampleRate (kHz) for lossless. + BitDepth int + SampleRate float64 + Source string + // Video is the marker that says this is a video ("x264", "1080p"), or "". + Video string + // Discography: a year range or a word naming many albums. + Discography string + // Tokens are the name's words, normalised for matching. + Tokens []string +} + +var ( + formatRe = regexp.MustCompile(`\b(flac|alac|ape|wavpack|wv|wav|aiff|dsd|dsf|dff|mp3|aac|m4a|ogg|vorbis|opus)\b`) + // "24-96", "24/192", "16-44 1" (44.1 after dots become spaces). + depthRateRe = regexp.MustCompile(`\b(16|24|32) ?[-/ ] ?(44 1|44|48|88 2|88|96|176 4|176|192|352 8|384)\b`) + depthRe = regexp.MustCompile(`\b(16|24|32) ?-?bits?\b`) + rateRe = regexp.MustCompile(`\b(44 1|48|88 2|96|176 4|192) ?khz\b`) + hiResRe = regexp.MustCompile(`\bhi ?-?res\b`) + codecRateRe = regexp.MustCompile(`\b(?:mp3|aac|m4a|cbr|ogg|opus)[ \-@]*(320|256|224|192|160|128|112|96|64)\b`) + kbpsRe = regexp.MustCompile(`\b(320|256|224|192|160|128|112|96|64) ?(?:kbps|kbit|kb/s|k)\b`) + vbrRe = regexp.MustCompile(`\bv([0-2])\b`) + webRe = regexp.MustCompile(`\bweb(?: ?-?dl)?\b`) + cdRe = regexp.MustCompile(`\bcd ?(?:rip)?\b`) + vinylRe = regexp.MustCompile(`\b(?:vinyl|vinylrip|lp ?rip|needle ?drop)\b`) + sacdRe = regexp.MustCompile(`\bsacd\b`) + cassetteRe = regexp.MustCompile(`\b(?:cassette|tape ?rip)\b`) + yearRe = regexp.MustCompile(`\b(19[3-9]\d|20[0-4]\d)\b`) + bracketYear = regexp.MustCompile(`[(\[{] ?(19[3-9]\d|20[0-4]\d) ?[)\]}]`) + yearRangeRe = regexp.MustCompile(`\b(19[3-9]\d|20[0-4]\d) ?(?:-|–|to) ?(19[3-9]\d|20[0-4]\d)\b`) + videoRe = regexp.MustCompile(`\b(x264|x265|h ?264|h ?265|hevc|xvid|divx|1080p|1080i|720p|2160p|bluray|blu ?ray|bdrip|brrip|bdremux|dvdrip|dvd9|dvd5|webrip|hdtv|mkv|avi|m2ts)\b`) + discogRe = regexp.MustCompile(`\b(discography|discografia|discographie|diskografie|discografie)\b`) +) + +// Parse reads what a release name says. +func Parse(name string) Release { + r := Release{Name: name} + s := spaced(name) + + r.Format = bestFormat(formatRe.FindAllStringSubmatch(s, -1)) + + if m := depthRateRe.FindStringSubmatch(s); m != nil { + r.BitDepth, _ = strconv.Atoi(m[1]) + r.SampleRate = rate(m[2]) + } + if r.BitDepth == 0 { + if m := depthRe.FindStringSubmatch(s); m != nil { + r.BitDepth, _ = strconv.Atoi(m[1]) + } + } + if r.SampleRate == 0 { + if m := rateRe.FindStringSubmatch(s); m != nil { + r.SampleRate = rate(m[1]) + } + } + if r.BitDepth == 0 && hiResRe.MatchString(s) { + r.BitDepth = 24 + } + + if m := codecRateRe.FindStringSubmatch(s); m != nil { + r.Bitrate, _ = strconv.Atoi(m[1]) + } else if m := kbpsRe.FindStringSubmatch(s); m != nil { + r.Bitrate, _ = strconv.Atoi(m[1]) + } + // A bare "V0" is a LAME preset only beside MP3 or VBR; elsewhere it can + // be part of a title. + if m := vbrRe.FindStringSubmatch(s); m != nil && (r.Format == FormatMP3 || strings.Contains(s, "vbr")) { + r.VBR = "V" + m[1] + } + if r.Format == "" && (r.Bitrate > 0 || r.VBR != "") { + r.Format = FormatMP3 + } + + switch { + case sacdRe.MatchString(s): + r.Source = SourceSACD + case vinylRe.MatchString(s): + r.Source = SourceVinyl + case webRe.MatchString(s): + r.Source = SourceWEB + case cassetteRe.MatchString(s): + r.Source = SourceCassette + case cdRe.MatchString(s): + r.Source = SourceCD + } + + if m := videoRe.FindStringSubmatch(s); m != nil { + r.Video = m[1] + } + if m := discogRe.FindStringSubmatch(s); m != nil { + r.Discography = m[1] + } else if m := yearRangeRe.FindStringSubmatch(s); m != nil && m[2] > m[1] { + r.Discography = m[1] + "–" + m[2] + } + r.Year = year(s) + r.Tokens = Tokens(name) + return r +} + +// spaced lowercases a name and turns its separators into spaces, keeping +// the slash in "24/96" and "kb/s". +func spaced(name string) string { + var b strings.Builder + for _, c := range strings.ToLower(name) { + switch c { + case '_', '.', ',': + b.WriteByte(' ') + default: + b.WriteRune(c) + } + } + return b.String() +} + +// rate reads a sample rate in kHz; "44", "88" and "176" are the short +// spellings of 44.1, 88.2 and 176.4. +func rate(s string) float64 { + switch s { + case "44": + return 44.1 + case "88": + return 88.2 + case "176": + return 176.4 + } + f, _ := strconv.ParseFloat(strings.ReplaceAll(s, " ", "."), 64) + return f +} + +// formatRank orders formats a name might state together ("FLAC + MP3"): +// the best is the one it is described by. +var formatRank = map[string]int{ + FormatDSD: 11, FormatFLAC: 10, FormatALAC: 9, FormatWV: 8, FormatAPE: 7, FormatWAV: 6, FormatAIFF: 5, + FormatOpus: 4, FormatAAC: 3, FormatVorbis: 2, FormatMP3: 1, +} + +func bestFormat(found [][]string) string { + best := "" + for _, m := range found { + f := formatOf(m[1]) + if formatRank[f] > formatRank[best] { + best = f + } + } + return best +} + +// formatOf names the format a word or file extension stands for. +func formatOf(word string) string { + switch word { + case "flac": + return FormatFLAC + case "alac": + return FormatALAC + case "ape": + return FormatAPE + case "wavpack", "wv": + return FormatWV + case "wav": + return FormatWAV + case "aiff", "aif": + return FormatAIFF + case "dsd", "dsf", "dff": + return FormatDSD + case "mp3": + return FormatMP3 + case "aac", "m4a": + return FormatAAC + case "ogg", "vorbis": + return FormatVorbis + case "opus": + return FormatOpus + } + return "" +} + +// year is the release year a name states: one in brackets when there is +// one (so the title "1989" is not read as its year), else the last. +func year(s string) int { + if m := bracketYear.FindStringSubmatch(s); m != nil { + y, _ := strconv.Atoi(m[1]) + return y + } + all := yearRe.FindAllString(s, -1) + if len(all) == 0 { + return 0 + } + y, _ := strconv.Atoi(all[len(all)-1]) + return y +} + +// fold maps the accented letters music names carry to plain ones. +var fold = map[rune]string{ + 'à': "a", 'á': "a", 'â': "a", 'ã': "a", 'ä': "a", 'å': "a", 'ā': "a", 'ą': "a", + 'æ': "ae", 'ç': "c", 'ć': "c", 'č': "c", + 'è': "e", 'é': "e", 'ê': "e", 'ë': "e", 'ē': "e", 'ė': "e", 'ę': "e", 'ě': "e", + 'ì': "i", 'í': "i", 'î': "i", 'ï': "i", 'ī': "i", 'ı': "i", + 'ñ': "n", 'ń': "n", 'ň': "n", + 'ò': "o", 'ó': "o", 'ô': "o", 'õ': "o", 'ö': "o", 'ø': "o", 'ō': "o", 'ő': "o", + 'œ': "oe", 'ù': "u", 'ú': "u", 'û': "u", 'ü': "u", 'ū': "u", 'ů': "u", 'ű': "u", + 'ý': "y", 'ÿ': "y", 'ß': "ss", 'š': "s", 'ś': "s", 'ş': "s", + 'ž': "z", 'ź': "z", 'ż': "z", 'ł': "l", 'đ': "d", 'ð': "d", 'þ': "th", 'ř': "r", +} + +// abbreviations are spelled out on both sides before comparing. +var abbreviations = map[string]string{"vol": "volume", "pt": "part", "n": "and"} + +// Tokens are a name's words for matching: lowercase, accents folded, "&" +// and "+" read as "and", apostrophes dropped (so "Don't" is "dont"), every +// other mark a word break. +func Tokens(s string) []string { + var b strings.Builder + for _, c := range strings.ToLower(s) { + switch { + case c >= 'a' && c <= 'z', c >= '0' && c <= '9': + b.WriteRune(c) + case c == '\'' || c == '’' || c == '`' || c == '´': + case c == '&' || c == '+': + b.WriteString(" and ") + default: + if f, ok := fold[c]; ok { + b.WriteString(f) + } else if c > 127 && isLetter(c) { + b.WriteRune(c) + } else { + b.WriteByte(' ') + } + } + } + words := strings.Fields(b.String()) + for i, w := range words { + if full, ok := abbreviations[w]; ok { + words[i] = full + } + } + return words +} + +// isLetter keeps letters of scripts the fold table does not cover (Cyrillic, +// CJK) as letters rather than breaks. +func isLetter(c rune) bool { + return (c >= 0x0400 && c <= 0x04FF) || (c >= 0x0370 && c <= 0x03FF) || c >= 0x3040 +} diff --git a/internal/release/quality.go b/internal/release/quality.go new file mode 100644 index 00000000..66590f9a --- /dev/null +++ b/internal/release/quality.go @@ -0,0 +1,169 @@ +package release + +import ( + "fmt" + "strconv" + "strings" +) + +// Tier is a rung of the quality ladder, Lidarr's names for them. +type Tier string + +// The ladder, best first. +const ( + TierLosslessHiRes Tier = "lossless_hires" + TierLossless Tier = "lossless" + TierLossyHigh Tier = "lossy_high" + TierLossyStandard Tier = "lossy_standard" + TierLossyLow Tier = "lossy_low" + TierLossyPoor Tier = "lossy_poor" + TierLossyTrash Tier = "lossy_trash" + TierUnknown Tier = "unknown" +) + +// losslessCategory is Newznab's Audio/Lossless. +const losslessCategory = 3040 + +// TierInfo is a rung as the settings page shows it. +type TierInfo struct { + Tier Tier `json:"tier"` + Label string `json:"label"` + What string `json:"what"` +} + +// Tiers is the ladder, best first. +var Tiers = []TierInfo{ + {TierLosslessHiRes, "Lossless, hi-res", "24-bit or above 48 kHz: FLAC, ALAC, DSD"}, + {TierLossless, "Lossless", "FLAC, ALAC, APE, WavPack, WAV at CD quality"}, + {TierLossyHigh, "High quality lossy", "MP3 320 or V0, AAC 256 and up"}, + {TierLossyStandard, "Standard quality lossy", "MP3 224–256 or V1/V2, AAC 192, Vorbis, Opus, or a lossy format with no bitrate stated"}, + {TierLossyLow, "Low quality lossy", "160–192 kbps"}, + {TierLossyPoor, "Poor quality lossy", "128 kbps"}, + {TierLossyTrash, "Trash quality lossy", "Under 128 kbps"}, + {TierUnknown, "Unknown", "Nothing says what format it is"}, +} + +// DefaultTiers is what a new install allows: Lidarr's "Any" profile, every +// format from trash lossy to lossless (note #5527). Unknown is left out; a +// release that names no format has its .torrent read to find out. +var DefaultTiers = []Tier{ + TierLosslessHiRes, TierLossless, TierLossyHigh, TierLossyStandard, + TierLossyLow, TierLossyPoor, TierLossyTrash, +} + +// KnownTier says whether t is a rung of the ladder. +func KnownTier(t string) bool { + for _, info := range Tiers { + if string(info.Tier) == t { + return true + } + } + return false +} + +// Rank orders tiers: higher is better, Unknown lowest. +func Rank(t Tier) int { + for i, info := range Tiers { + if info.Tier == t { + return len(Tiers) - i + } + } + return 0 +} + +// Label is a tier's name for a person. +func Label(t Tier) string { + for _, info := range Tiers { + if info.Tier == t { + return info.Label + } + } + return string(t) +} + +// TierOf places a release on the ladder. category is the indexer's; 3040 +// (Lossless) stands in for a name that states no format. +func TierOf(r Release, category int) Tier { + switch r.Format { + case FormatDSD: + return TierLosslessHiRes + case FormatFLAC, FormatALAC, FormatAPE, FormatWV, FormatWAV, FormatAIFF: + if r.BitDepth >= 24 || r.SampleRate > 48 { + return TierLosslessHiRes + } + return TierLossless + case FormatMP3: + switch r.VBR { + case "V0": + return TierLossyHigh + case "V1", "V2": + return TierLossyStandard + } + return byBitrate(r.Bitrate, 320) + case FormatAAC, FormatVorbis, FormatOpus: + return byBitrate(r.Bitrate, 256) + } + if category == losslessCategory { + return TierLossless + } + return TierUnknown +} + +// byBitrate places a lossy bitrate; high is the bitrate from which the codec +// counts as high quality. An unstated bitrate is standard. +func byBitrate(kbps, high int) Tier { + switch { + case kbps == 0: + return TierLossyStandard + case kbps >= high: + return TierLossyHigh + case kbps >= 224 || (high == 256 && kbps >= 192): + return TierLossyStandard + case kbps >= 160: + return TierLossyLow + case kbps >= 128: + return TierLossyPoor + } + return TierLossyTrash +} + +// Describe is a release's format in a few words: "FLAC 24-bit/96 kHz", +// "MP3 V0", "AAC 256". Empty when the name states none. +func Describe(r Release) string { + if r.Format == "" { + return "" + } + parts := []string{r.Format} + switch { + case r.BitDepth > 0 && r.SampleRate > 0: + parts = append(parts, fmt.Sprintf("%d-bit/%s kHz", r.BitDepth, khz(r.SampleRate))) + case r.BitDepth > 0: + parts = append(parts, fmt.Sprintf("%d-bit", r.BitDepth)) + case r.VBR != "": + parts = append(parts, r.VBR) + case r.Bitrate > 0: + parts = append(parts, strconv.Itoa(r.Bitrate)) + } + return strings.Join(parts, " ") +} + +func khz(f float64) string { + return strings.TrimSuffix(strconv.FormatFloat(f, 'f', 1, 64), ".0") +} + +// Profile is which tiers the operator takes. +type Profile struct { + Allowed map[Tier]bool +} + +// NewProfile builds a profile from the stored tier list. +func NewProfile(tiers []string) Profile { + p := Profile{Allowed: map[Tier]bool{}} + for _, t := range tiers { + p.Allowed[Tier(t)] = true + } + return p +} + +// Allows says whether the profile takes a tier. +func (p Profile) Allows(t Tier) bool { return p.Allowed[t] } diff --git a/internal/release/release_test.go b/internal/release/release_test.go new file mode 100644 index 00000000..a5c07015 --- /dev/null +++ b/internal/release/release_test.go @@ -0,0 +1,251 @@ +package release + +import ( + "crypto/sha1" + "encoding/hex" + "fmt" + "strings" + "testing" +) + +func TestParse_ReadsWhatTheNameSays(t *testing.T) { + for _, c := range []struct { + name string + format, vbr, src string + bitrate, depth, yr int + rate float64 + tier Tier + }{ + // The wrong edition Lidarr grabbed on 2026-10-09 (note #5527). + {"Daft Punk Random Access Memories (Drumless Edition 2023) [Flac 24-88]", FormatFLAC, "", "", 0, 24, 2023, 88.2, TierLosslessHiRes}, + {"Gorillaz - Humanz - (Japanese Edition) - [CD-FLAC] 2017", FormatFLAC, "", SourceCD, 0, 0, 2017, 0, TierLossless}, + {"Radiohead - OK Computer (1997) [MP3 320]", FormatMP3, "", "", 320, 0, 1997, 0, TierLossyHigh}, + {"Artist - Album 2019 MP3 V0", FormatMP3, "V0", "", 0, 0, 2019, 0, TierLossyHigh}, + {"Artist - Album (2020) [WEB] [AAC 256]", FormatAAC, "", SourceWEB, 256, 0, 2020, 0, TierLossyHigh}, + {"Artist_-_Album-(2004)-192kbps", FormatMP3, "", "", 192, 0, 2004, 0, TierLossyLow}, + {"Artist - Album [24bit-96kHz] FLAC Vinyl", FormatFLAC, "", SourceVinyl, 0, 24, 0, 96, TierLosslessHiRes}, + {"Taylor Swift - 1989 (2014) [FLAC]", FormatFLAC, "", "", 0, 0, 2014, 0, TierLossless}, + {"Artist - Album", "", "", "", 0, 0, 0, 0, TierUnknown}, + } { + r := Parse(c.name) + got := fmt.Sprint(r.Format, r.VBR, r.Source, r.Bitrate, r.BitDepth, r.Year, r.SampleRate, TierOf(r, 0)) + want := fmt.Sprint(c.format, c.vbr, c.src, c.bitrate, c.depth, c.yr, c.rate, c.tier) + if got != want { + t.Errorf("Parse(%q)\n got %s\nwant %s", c.name, got, want) + } + } +} + +func TestParse_VideoAndDiscographies(t *testing.T) { + if v := Parse("Artist.Album.2018.1080p.BluRay.x264").Video; v != "1080p" { + t.Errorf("video = %q", v) + } + if d := Parse("Artist Discography 1994-2010 FLAC").Discography; d != "discography" { + t.Errorf("discography = %q", d) + } + if d := Parse("Artist - Studio Albums 1994 - 2010 [MP3]").Discography; d != "1994–2010" { + t.Errorf("year range = %q", d) + } + if d := Parse("Artist - Album (2010) [FLAC]").Discography; d != "" { + t.Errorf("one year read as a range: %q", d) + } +} + +func judge(w Want, name string) Match { return Judge(w, Parse(name)) } + +func TestJudge_TheAlbumAndWhatIsNot(t *testing.T) { + ram := Want{Artist: "Daft Punk", Title: "Random Access Memories", Year: 2013, PrimaryType: "Album"} + for name, veto := range map[string]string{ + "Daft Punk - Random Access Memories (2013) [FLAC]": "", + "Daft_Punk-Random_Access_Memories-WEB-2013": "", + "Daft Punk Random Access Memories (Drumless Edition 2023) [Flac 24-88]": VetoVersion, + "Daft Punk - Random Access Memories (Instrumentals) [MP3]": VetoVersion, + "Daft Punk - Random Access Memories Remixes": VetoVersion, + "Daft Punk - Random Access Memories (Live) FLAC": VetoVersion, + "Daft Punk - Discovery (2001) [FLAC]": VetoTitle, + "Random Access Memories (2013) [FLAC]": VetoArtist, + "Daft Punk - Discography 1997-2013 FLAC": VetoTitle, + "Daft Punk - Random Access Memories + Discovery + Homework Collection": VetoDiscography, + "Daft Punk - Random Access Memories (2013) 1080p BluRay": VetoVideo, + "Daft Punk - Random Access Memories (Single) 2013": VetoType, + "Daft Punk - Random Access Memories (Deluxe 10th Anniversary Edition) 2023": "", + } { + if got := judge(ram, name).Veto; got != veto { + t.Errorf("%q: veto %q, want %q", name, got, veto) + } + } + if !judge(ram, "Daft Punk - Random Access Memories (2013) [FLAC]").YearAgrees { + t.Error("the year did not agree") + } +} + +func TestJudge_WordsTheWantedTitleHoldsDoNotVeto(t *testing.T) { + hole := Want{Artist: "Hole", Title: "Live Through This", PrimaryType: "Album"} + if m := judge(hole, "Hole - Live Through This (1994) [FLAC]"); m.Veto != "" { + t.Errorf("Live Through This: %s", m.Reason) + } + alive := Want{Artist: "Daft Punk", Title: "Alive 2007", PrimaryType: "Album", SecondaryTypes: []string{"Live"}} + if m := judge(alive, "Daft Punk - Alive 2007 (Live) [FLAC]"); m.Veto != "" { + t.Errorf("a wanted live album: %s", m.Reason) + } + alive.SecondaryTypes = nil + if m := judge(alive, "Daft Punk - Alive 2007 (Live) [FLAC]"); m.Veto != VetoVersion { + t.Errorf("live without the Live type: veto %q", m.Veto) + } +} + +func TestJudge_SpellingsOfNames(t *testing.T) { + for _, c := range []struct { + w Want + name string + }{ + {Want{Artist: "AC/DC", Title: "Back in Black"}, "ACDC - Back In Black (1980) FLAC"}, + {Want{Artist: "AC/DC", Title: "Back in Black"}, "AC-DC - Back in Black [MP3 320]"}, + {Want{Artist: "Sigur Rós", Title: "Ágætis byrjun"}, "Sigur Ros - Agaetis Byrjun (1999) [FLAC]"}, + {Want{Artist: "Simon & Garfunkel", Title: "Bookends"}, "Simon and Garfunkel - Bookends"}, + {Want{Artist: "The Beatles", Title: "Abbey Road"}, "Beatles - Abbey Road (Remastered 2009)"}, + {Want{Artist: "Florence + the Machine", Title: "Lungs"}, "Florence and The Machine - Lungs (2009)"}, + {Want{Artist: "Guns N' Roses", Title: "Appetite for Destruction"}, "Guns N Roses - Appetite For Destruction"}, + } { + if m := judge(c.w, c.name); m.Veto != "" { + t.Errorf("%q for %s / %s: %s", c.name, c.w.Artist, c.w.Title, m.Reason) + } + } + // One word is never matched run together: Muse is not inside Museum. + if m := judge(Want{Artist: "Muse", Title: "Absolution"}, "Museum - Absolution"); m.Veto != VetoArtist { + t.Errorf("Museum read as Muse: %q", m.Veto) + } +} + +func TestJudge_ASelfTitledAlbumNeedsMoreThanTheName(t *testing.T) { + weezer := Want{Artist: "Weezer", Title: "Weezer", Year: 1994, PrimaryType: "Album"} + if m := judge(weezer, "Weezer - Weezer (Blue Album) [FLAC]"); m.Doubt != "" || m.Veto != "" { + t.Errorf("named twice: %+v", m) + } + if m := judge(weezer, "Weezer (1994) [FLAC]"); m.Doubt != "" { + t.Errorf("the year agrees: %+v", m) + } + if m := judge(weezer, "Weezer (Green Album) FLAC"); m.Doubt == "" { + t.Error("a self-titled name with nothing else was taken as sure") + } +} + +func TestTierOf_TheLadder(t *testing.T) { + for _, c := range []struct { + r Release + cat int + want Tier + }{ + {Release{Format: FormatMP3, Bitrate: 256}, 0, TierLossyStandard}, + {Release{Format: FormatMP3, Bitrate: 128}, 0, TierLossyPoor}, + {Release{Format: FormatMP3, Bitrate: 96}, 0, TierLossyTrash}, + {Release{Format: FormatMP3, VBR: "V2"}, 0, TierLossyStandard}, + {Release{Format: FormatMP3}, 0, TierLossyStandard}, + {Release{Format: FormatAAC, Bitrate: 192}, 0, TierLossyStandard}, + {Release{Format: FormatALAC, SampleRate: 96}, 0, TierLosslessHiRes}, + {Release{Format: FormatDSD}, 0, TierLosslessHiRes}, + {Release{}, 3040, TierLossless}, + {Release{}, 3010, TierUnknown}, + } { + if got := TierOf(c.r, c.cat); got != c.want { + t.Errorf("TierOf(%+v, %d) = %s, want %s", c.r, c.cat, got, c.want) + } + } + if Rank(TierLossless) <= Rank(TierLossyHigh) || Rank(TierUnknown) >= Rank(TierLossyTrash) { + t.Error("the ladder is out of order") + } + if !NewProfile([]string{"lossless"}).Allows(TierLossless) || NewProfile([]string{"lossless"}).Allows(TierUnknown) { + t.Error("profile") + } +} + +func files(paths ...string) []File { + out := make([]File, len(paths)) + for i, p := range paths { + out[i] = File{Index: i, Path: p, Size: 1 << 20} + } + return out +} + +func numbered(n int, ext string) []string { + var out []string + for i := 1; i <= n; i++ { + out = append(out, fmt.Sprintf("Album/%02d - Track.%s", i, ext)) + } + return out +} + +func TestJudgeFiles_WhatTheTorrentHolds(t *testing.T) { + w := Want{MinTracks: 10, MaxTracks: 13} + if veto, _, _ := JudgeFiles(w, files(numbered(13, "flac")...)); veto != "" { + t.Errorf("a whole album: %s", veto) + } + if veto, reason, _ := JudgeFiles(w, files(numbered(3, "flac")...)); veto != VetoIncomplete || !strings.Contains(reason, "3 tracks") { + t.Errorf("three tracks: %s %s", veto, reason) + } + if veto, _, _ := JudgeFiles(w, files("Concert/Concert.mkv", "Concert/cover.jpg")); veto != VetoVideo { + t.Errorf("a video: %s", veto) + } + if veto, _, _ := JudgeFiles(w, files("Album.rar", "Album.r00")); veto != VetoNoAudio { + t.Errorf("an archive: %s", veto) + } + if _, _, doubt := JudgeFiles(w, files(numbered(60, "mp3")...)); doubt == "" { + t.Error("sixty tracks for a thirteen-track album raised no doubt") + } + if c := Inspect(files(append(numbered(12, "flac"), "Album/cover.jpg", "Album/album.cue")...)); c.Format != FormatFLAC || c.Audio != 12 { + t.Errorf("inspect = %+v", c) + } +} + +// bencode writes the few shapes a test torrent needs. +func bencode(v any) string { + switch x := v.(type) { + case string: + return fmt.Sprintf("%d:%s", len(x), x) + case int: + return fmt.Sprintf("i%de", x) + case []any: + var b strings.Builder + b.WriteString("l") + for _, e := range x { + b.WriteString(bencode(e)) + } + return b.String() + "e" + case [][2]any: // an ordered dictionary + var b strings.Builder + b.WriteString("d") + for _, kv := range x { + b.WriteString(bencode(kv[0]) + bencode(kv[1])) + } + return b.String() + "e" + } + panic(fmt.Sprintf("bencode %T", v)) +} + +func TestParseTorrent_FilesAndInfohash(t *testing.T) { + info := bencode([][2]any{ + {"files", []any{ + [][2]any{{"length", 300}, {"path", []any{"01 - One.flac"}}}, + [][2]any{{"attr", "p"}, {"length", 12}, {"path", []any{".pad", "12"}}}, + [][2]any{{"length", 200}, {"path", []any{"Scans", "front.jpg"}}}, + }}, + {"name", "Artist - Album"}, + {"piece length", 16384}, + {"pieces", strings.Repeat("x", 20)}, + }) + body := "d" + bencode("announce") + bencode("http://tracker/") + bencode("info") + info + "e" + meta, err := ParseTorrent([]byte(body)) + if err != nil { + t.Fatal(err) + } + sum := sha1.Sum([]byte(info)) + if meta.InfoHash != hex.EncodeToString(sum[:]) { + t.Errorf("infohash = %s", meta.InfoHash) + } + if len(meta.Files) != 2 || meta.Files[0].Path != "Artist - Album/01 - One.flac" || meta.Files[1].Path != "Artist - Album/Scans/front.jpg" { + t.Errorf("files = %+v", meta.Files) + } + if _, err := ParseTorrent([]byte("not a torrent")); err == nil { + t.Error("HTML parsed as a torrent") + } +} diff --git a/internal/release/torrent.go b/internal/release/torrent.go new file mode 100644 index 00000000..578528de --- /dev/null +++ b/internal/release/torrent.go @@ -0,0 +1,178 @@ +package release + +import ( + "bytes" + "crypto/sha1" + "encoding/hex" + "errors" + "fmt" + "strconv" + "strings" +) + +// Reading a .torrent file, ported from Fabled Librarian (#4463). A .torrent +// is the metadata: its info dictionary names every file, and its SHA-1 is the +// infohash. So a release offered as a .torrent link can be judged on what it +// holds before any of it is downloaded. Bencode is four shapes; walking them +// here costs less than a dependency. + +// TorrentMeta is what a .torrent says about itself. +type TorrentMeta struct { + // InfoHash is the v1 infohash: the SHA-1 of the info dictionary's own + // bytes, lowercase hex. + InfoHash string + Name string + Files []File +} + +// ErrNotTorrent is bytes that are not a .torrent. +var ErrNotTorrent = errors.New("not a .torrent file") + +// maxTorrentBytes bounds what is read, as Exchange bounds what it accepts. +const maxTorrentBytes = 10 << 20 + +// ParseTorrent reads a .torrent's info dictionary. +func ParseTorrent(b []byte) (TorrentMeta, error) { + if len(b) == 0 || len(b) > maxTorrentBytes || b[0] != 'd' { + return TorrentMeta{}, ErrNotTorrent + } + d := &bdecoder{b: b} + root, err := d.value(0) + if err != nil { + return TorrentMeta{}, fmt.Errorf("%w: %v", ErrNotTorrent, err) + } + top, ok := root.(map[string]any) + if !ok || d.infoEnd == 0 { + return TorrentMeta{}, fmt.Errorf("%w: no info dictionary", ErrNotTorrent) + } + info, ok := top["info"].(map[string]any) + if !ok { + return TorrentMeta{}, fmt.Errorf("%w: no info dictionary", ErrNotTorrent) + } + sum := sha1.Sum(b[d.infoStart:d.infoEnd]) + out := TorrentMeta{InfoHash: hex.EncodeToString(sum[:])} + out.Name, _ = info["name"].(string) + + files, multi := info["files"].([]any) + if !multi { + // A single-file torrent: the name IS the file. + length, _ := info["length"].(int64) + out.Files = []File{{Path: out.Name, Size: length}} + return out, nil + } + for index, f := range files { + entry, ok := f.(map[string]any) + if !ok { + continue + } + parts, _ := entry["path"].([]any) + segs := []string{out.Name} + for _, p := range parts { + if s, ok := p.(string); ok { + segs = append(segs, s) + } + } + length, _ := entry["length"].(int64) + // BEP 47 padding files carry no content. + if attr, _ := entry["attr"].(string); strings.Contains(attr, "p") { + continue + } + out.Files = append(out.Files, File{Index: index, Path: strings.Join(segs, "/"), Size: length}) + } + return out, nil +} + +// bdecoder walks bencode, and remembers where the top-level info dictionary +// sits so its raw bytes can be hashed exactly as published. +type bdecoder struct { + b []byte + i int + infoStart, infoEnd int +} + +// maxDepth: a real torrent nests four deep. Anything past this is hostile. +const maxDepth = 32 + +func (d *bdecoder) value(depth int) (any, error) { + if depth > maxDepth { + return nil, errors.New("nested too deep") + } + if d.i >= len(d.b) { + return nil, errors.New("truncated") + } + switch c := d.b[d.i]; { + case c == 'i': + end := bytes.IndexByte(d.b[d.i:], 'e') + if end < 0 { + return nil, errors.New("unterminated integer") + } + n, err := strconv.ParseInt(string(d.b[d.i+1:d.i+end]), 10, 64) + if err != nil { + return nil, err + } + d.i += end + 1 + return n, nil + case c == 'l': + d.i++ + var out []any + for d.i < len(d.b) && d.b[d.i] != 'e' { + v, err := d.value(depth + 1) + if err != nil { + return nil, err + } + out = append(out, v) + } + if d.i >= len(d.b) { + return nil, errors.New("unterminated list") + } + d.i++ + return out, nil + case c == 'd': + d.i++ + out := map[string]any{} + for d.i < len(d.b) && d.b[d.i] != 'e' { + k, err := d.str() + if err != nil { + return nil, err + } + start := d.i + v, err := d.value(depth + 1) + if err != nil { + return nil, err + } + if depth == 0 && k == "info" { + d.infoStart, d.infoEnd = start, d.i + } + // Pieces are 20 bytes a piece and never read: not kept. + if k != "pieces" { + out[k] = v + } + } + if d.i >= len(d.b) { + return nil, errors.New("unterminated dictionary") + } + d.i++ + return out, nil + case c >= '0' && c <= '9': + return d.str() + default: + return nil, fmt.Errorf("unexpected %q", c) + } +} + +func (d *bdecoder) str() (string, error) { + colon := bytes.IndexByte(d.b[d.i:], ':') + if colon < 0 { + return "", errors.New("unterminated string length") + } + n, err := strconv.Atoi(string(d.b[d.i : d.i+colon])) + if err != nil || n < 0 { + return "", errors.New("bad string length") + } + start := d.i + colon + 1 + if start+n > len(d.b) { + return "", errors.New("string runs past the end") + } + d.i = start + n + return string(d.b[start : start+n]), nil +} diff --git a/internal/wanted/search.go b/internal/wanted/search.go new file mode 100644 index 00000000..3b5efb2e --- /dev/null +++ b/internal/wanted/search.go @@ -0,0 +1,322 @@ +package wanted + +import ( + "context" + "encoding/json" + "errors" + "log/slog" + "strconv" + "time" + + "github.com/jackc/pgx/v5/pgtype" + "github.com/jackc/pgx/v5/pgxpool" + + "git.fabledsword.com/bvandeusen/minstrel/internal/acquire" + "git.fabledsword.com/bvandeusen/minstrel/internal/catalogue" + "git.fabledsword.com/bvandeusen/minstrel/internal/db/dbq" + "git.fabledsword.com/bvandeusen/minstrel/internal/indexer" + "git.fabledsword.com/bvandeusen/minstrel/internal/release" +) + +// Searching for wants on a schedule (M502 #5521), ported from Fabled +// Librarian's internal/wanted. The schedule decides which wants and when; +// internal/acquire judges what a search finds and grabs. + +// Search outcomes, matching wants_search_outcome_check. +const ( + OutcomeGrabbed = "grabbed" + OutcomeWaiting = "waiting" + OutcomeNothing = "nothing" + OutcomeError = "error" +) + +const ( + // searchTick is how often the schedule looks for wants due; the + // operator's setting is how long a want waits between searches. + searchTick = 10 * time.Minute + // searchWindow is how long one pass searches: most of a tick, so a pass + // ends before the next begins. + searchWindow = 9 * time.Minute + // duePage bounds what one pass reads. The sources' pace is the real + // limit; this caps a pass whose sources are all unpaced. + duePage = 200 + // grabTries is how many grabbable releases one search tries in turn + // when the first is refused for its files. + grabTries = 3 + // passTimeout backs up the window for a call none of the clients' + // own timeouts cover, such as a wedged database read. + passTimeout = 15 * time.Minute +) + +// ErrWantHeld: the want is already answered by the library. +var ErrWantHeld = errors.New("wanted: the library already holds this album") + +// Searcher searches for wants and grabs what they need. +type Searcher struct { + pool *pgxpool.Pool + cat *catalogue.Service + logger *slog.Logger +} + +// NewSearcher returns a Searcher reading editions through cat (nil reads +// only what the catalogue already holds). +func NewSearcher(pool *pgxpool.Pool, cat *catalogue.Service, logger *slog.Logger) *Searcher { + if logger == nil { + logger = slog.Default() + } + return &Searcher{pool: pool, cat: cat, logger: logger} +} + +// Report is one pass. +type Report struct { + Looked int `json:"looked"` + Grabbed int `json:"grabbed"` + Waiting int `json:"waiting"` + Nothing int `json:"nothing"` + Failed int `json:"failed"` + // Paced: the pass stopped because the next search would wait past its + // window for a source's pace. + Paced bool `json:"paced"` + // Off: the schedule is switched off. Unconfigured: nothing to search + // with. Both are states, not faults. + Off bool `json:"off"` + Unconfigured bool `json:"unconfigured"` + AutoGrab bool `json:"auto_grab"` + Error string `json:"error,omitempty"` +} + +// Run searches every searchTick until ctx ends. +func (s *Searcher) Run(ctx context.Context) { + ticker := time.NewTicker(searchTick) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + } + rep := s.Pass(ctx) + if rep.Looked > 0 || rep.Error != "" { + s.logger.Info("wanted: search pass", "looked", rep.Looked, "grabbed", rep.Grabbed, + "waiting", rep.Waiting, "nothing", rep.Nothing, "failed", rep.Failed, + "paced", rep.Paced, "error", rep.Error) + } + } +} + +// Pass searches for the wants due, paced by the sources, until the window +// closes, and records what it did. +func (s *Searcher) Pass(ctx context.Context) Report { + rep := s.pass(ctx) + if body, err := json.Marshal(rep); err == nil { + if err := dbq.New(s.pool).RecordSearchPass(ctx, body); err != nil && ctx.Err() == nil { + s.logger.Warn("wanted: record search pass", "err", err) + } + } + return rep +} + +func (s *Searcher) pass(ctx context.Context) Report { + var rep Report + ctx, cancel := context.WithTimeout(ctx, passTimeout) + defer cancel() + svc, settings, err := acquire.Build(ctx, s.pool) + if err != nil { + rep.Error = indexer.FirstLine(err.Error()) + return rep + } + rep.AutoGrab = settings.AutoGrab + switch { + case settings.SearchIntervalHours <= 0: + rep.Off = true + return rep + case svc.Search == nil: + rep.Unconfigured = true + return rep + } + deadline := time.Now().Add(searchWindow) + due, err := dbq.New(s.pool).ListWantsDueForSearch(ctx, dbq.ListWantsDueForSearchParams{ + Hours: settings.SearchIntervalHours, MaxRows: duePage, + }) + if err != nil { + rep.Error = indexer.FirstLine(err.Error()) + return rep + } + for _, row := range due { + if !svc.Ready(ctx, deadline) { + rep.Paced = ctx.Err() == nil + return rep + } + want := s.wantFor(ctx, row.ID, row.ArtistName, row.Title, row.PrimaryType, row.SecondaryTypes, + row.FirstReleaseDate, row.ReleaseGroupMbid) + out := s.one(ctx, svc, want, settings.AutoGrab) + rep.Looked++ + switch out.Outcome { + case OutcomeGrabbed: + rep.Grabbed++ + case OutcomeWaiting: + rep.Waiting++ + case OutcomeNothing: + rep.Nothing++ + default: + rep.Failed++ + } + } + return rep +} + +// wantFor describes a want for the matcher, with its editions' track counts +// from the catalogue (read from MusicBrainz the first time when it can be). +func (s *Searcher) wantFor(ctx context.Context, id pgtype.UUID, artist, title, primary string, secondary []string, date, group string) acquire.Want { + w := acquire.Want{ID: id, Want: release.Want{ + Artist: artist, Title: title, Year: yearOf(date), PrimaryType: primary, SecondaryTypes: secondary, + }} + q := dbq.New(s.pool) + span, err := q.EditionTrackSpan(ctx, group) + if err == nil && span.MinTracks == 0 && s.cat != nil { + if _, rerr := s.cat.Releases(ctx, group); rerr == nil { + span, err = q.EditionTrackSpan(ctx, group) + } + } + if err == nil { + w.MinTracks, w.MaxTracks = int(span.MinTracks), int(span.MaxTracks) + } + return w +} + +func yearOf(date string) int { + if len(date) < 4 { + return 0 + } + y, _ := strconv.Atoi(date[:4]) + return y +} + +// Searched is what one want's search came to. +type Searched struct { + Outcome string `json:"outcome"` + Considered int `json:"considered"` + Best string `json:"best_release"` + Verdict string `json:"best_verdict"` + Grabbed string `json:"grabbed,omitempty"` + Narrowed []acquire.Narrowing `json:"narrowed"` + Error string `json:"error,omitempty"` +} + +// one searches for a want, takes the best release when auto-grab is on, and +// records what came of it on the want. +func (s *Searcher) one(ctx context.Context, svc *acquire.Service, want acquire.Want, autoGrab bool) Searched { + found := svc.For(ctx, want) + if found.KeepError != "" { + s.logger.Warn("wanted: searched, but could not keep the candidates", "want", want.Title, "err", found.KeepError) + } + out := Searched{Considered: found.Considered, Narrowed: found.Narrowed} + if out.Narrowed == nil { + out.Narrowed = []acquire.Narrowing{} + } + if len(found.Candidates) > 0 { + out.Best, out.Verdict = found.Candidates[0].Name, string(found.Candidates[0].Verdict) + } + switch { + case found.Unconfigured: + out.Outcome, out.Error = OutcomeError, "No indexer is set up, so there is nothing to search with." + case found.Error != "": + out.Outcome, out.Error = OutcomeError, found.Error + case !found.ForAPerson(): + out.Outcome = OutcomeNothing + case len(found.Grabbable()) == 0 || !autoGrab: + out.Outcome = OutcomeWaiting + default: + out.Outcome = OutcomeWaiting + s.grabBest(ctx, svc, want, found, &out) + } + s.record(ctx, want.ID, out) + return out +} + +// grabBest takes the best grabbable release, trying the next when one is +// refused for what its files hold. +func (s *Searcher) grabBest(ctx context.Context, svc *acquire.Service, want acquire.Want, found acquire.Result, out *Searched) { + for i, c := range found.Grabbable() { + if i == grabTries { + return + } + took := svc.Grab(ctx, want, &c, found.Candidates, false, acquire.Origin{By: acquire.OriginSchedule}) + switch { + case took.Added: + out.Outcome, out.Grabbed, out.Best, out.Verdict = OutcomeGrabbed, c.Name, c.Name, string(acquire.Grab) + if took.Error != "" { + s.logger.Warn("wanted: took a release but could not record it", "want", want.Title, "err", took.Error) + } + s.logger.Info("wanted: took a release", "want", want.Title, "artist", want.Artist, + "release", c.Name, "quality", c.Tier, "save_path", took.SavePath) + return + case took.Refused: + continue + case took.Unconfigured: + out.Outcome, out.Error = OutcomeError, "No download client is set up, so nothing was taken." + return + case took.Contest != nil: + return + case took.Error != "": + out.Outcome, out.Error = OutcomeError, took.Error + return + } + } +} + +func (s *Searcher) record(ctx context.Context, id pgtype.UUID, out Searched) { + if err := dbq.New(s.pool).RecordWantSearch(ctx, dbq.RecordWantSearchParams{ + ID: id, Outcome: out.Outcome, Considered: int32(out.Considered), + BestRelease: out.Best, BestVerdict: out.Verdict, SearchError: out.Error, + }); err != nil && ctx.Err() == nil { + s.logger.Warn("wanted: record search", "err", err) + } +} + +// SearchNow searches for one want at once, as the schedule would, for a +// person pressing Search: it spends the sources' pace but never waits on it. +func (s *Searcher) SearchNow(ctx context.Context, id pgtype.UUID) (Searched, error) { + want, err := s.load(ctx, id) + if err != nil { + return Searched{}, err + } + svc, settings, err := acquire.Build(ctx, s.pool) + if err != nil { + return Searched{}, err + } + return s.one(ctx, svc, want, settings.AutoGrab), nil +} + +// Take grabs the release named by key for a person, searching again first. +func (s *Searcher) Take(ctx context.Context, id pgtype.UUID, key string, force bool, by pgtype.UUID) (acquire.Taken, error) { + want, err := s.load(ctx, id) + if err != nil { + return acquire.Taken{}, err + } + svc, _, err := acquire.Build(ctx, s.pool) + if err != nil { + return acquire.Taken{}, err + } + took := svc.Take(ctx, want, key, force, acquire.Origin{By: acquire.OriginOperator, User: by}) + if took.Added && took.Chosen != nil { + s.record(ctx, id, Searched{ + Outcome: OutcomeGrabbed, Best: took.Chosen.Name, Verdict: string(took.Chosen.Verdict), + Considered: 0, Grabbed: took.Chosen.Name, + }) + } + return took, nil +} + +// load reads an open want for a search. +func (s *Searcher) load(ctx context.Context, id pgtype.UUID) (acquire.Want, error) { + row, err := dbq.New(s.pool).GetWantForSearch(ctx, id) + if err != nil { + return acquire.Want{}, err + } + if row.FulfilledAt.Valid { + return acquire.Want{}, ErrWantHeld + } + return s.wantFor(ctx, row.ID, row.ArtistName, row.Title, row.PrimaryType, row.SecondaryTypes, + row.FirstReleaseDate, row.ReleaseGroupMbid), nil +} diff --git a/internal/wanted/search_test.go b/internal/wanted/search_test.go new file mode 100644 index 00000000..1e96274f --- /dev/null +++ b/internal/wanted/search_test.go @@ -0,0 +1,195 @@ +package wanted + +import ( + "context" + "encoding/json" + "io" + "log/slog" + "net/http" + "net/http/httptest" + "strings" + "sync/atomic" + "testing" + + "git.fabledsword.com/bvandeusen/minstrel/internal/acquire" + "git.fabledsword.com/bvandeusen/minstrel/internal/db/dbq" +) + +const ramHash = "0123456789abcdef0123456789abcdef01234567" + +const ramFeed = ` + +Daft Punk - Random Access Memories (2013) [FLAC] + magnet:?xt=urn:btih:` + ramHash + `400000000 + +Daft Punk Random Access Memories (Drumless Edition 2023) [Flac 24-88] + magnet:?xt=urn:btih:fedcba9876543210fedcba9876543210fedcba98900000000 + +` + +// searchStack is a fake indexer feed and a fake FabledExchange download +// client, configured the way the operator would in Integrations. +func searchStack(t *testing.T, ctx context.Context, store *acquire.Store) *atomic.Int32 { + t.Helper() + var added atomic.Int32 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch { + case r.URL.Path == "/feed": + if q := r.URL.Query().Get("q"); q != "Daft Punk Random Access Memories" { + t.Errorf("searched for %q", q) + } + _, _ = io.WriteString(w, ramFeed) + case r.URL.Path == "/api/downloads" && r.Method == http.MethodPost: + var body struct { + Magnet string `json:"magnet"` + SavePath string `json:"save_path"` + } + _ = json.NewDecoder(r.Body).Decode(&body) + if !strings.Contains(body.Magnet, ramHash) || body.SavePath != "/downloads/"+ramHash { + t.Errorf("added %+v", body) + } + added.Add(1) + _, _ = io.WriteString(w, "{}") + default: + http.NotFound(w, r) + } + })) + t.Cleanup(srv.Close) + + feed, err := store.Create(ctx, acquire.ProtocolTorznab) + if err != nil { + t.Fatal(err) + } + if _, err := store.Save(ctx, feed.Code, acquire.Update{ + Label: "Feed", Enabled: true, BaseURL: srv.URL + "/feed", TimeoutMs: 5000, LocalPath: "/downloads", + }); err != nil { + t.Fatal(err) + } + ex, err := store.Create(ctx, acquire.ProtocolExchange) + if err != nil { + t.Fatal(err) + } + if _, err := store.Save(ctx, ex.Code, acquire.Update{ + Label: "Exchange", Enabled: true, BaseURL: srv.URL, TimeoutMs: 5000, ServesDownloads: true, LocalPath: "/downloads", + }); err != nil { + t.Fatal(err) + } + return &added +} + +func TestSearcher_FindsAndTakesAWant_Integration(t *testing.T) { + ctx := context.Background() + pool := newPool(t) + q := dbq.New(pool) + added := searchStack(t, ctx, acquire.NewStore(pool)) + w, err := WantWith(ctx, q, WantParams{ + GroupMBID: "rg-ram", ArtistMBID: "mb-daft-punk", ArtistName: "Daft Punk", Title: "Random Access Memories", + PrimaryType: "Album", FirstReleaseDate: "2013-05-17", Reason: ReasonRequest, + }) + if err != nil { + t.Fatal(err) + } + s := NewSearcher(pool, nil, slog.New(slog.NewTextHandler(io.Discard, nil))) + + rep := s.Pass(ctx) + if rep.Looked != 1 || rep.Grabbed != 1 || !rep.AutoGrab || added.Load() != 1 { + t.Fatalf("pass = %+v, added %d", rep, added.Load()) + } + got, err := q.GetWantByReleaseGroup(ctx, "rg-ram") + if err != nil { + t.Fatal(err) + } + if got.SearchOutcome != OutcomeGrabbed || got.SearchConsidered != 2 || !strings.Contains(got.BestRelease, "(2013) [FLAC]") { + t.Errorf("want after the pass: %s %d %q", got.SearchOutcome, got.SearchConsidered, got.BestRelease) + } + cands, err := q.ListWantCandidates(ctx, w.ID) + if err != nil { + t.Fatal(err) + } + if len(cands) != 2 || cands[0].Verdict != "grab" || cands[1].Verdict != "rejected" || !strings.Contains(cands[1].Reason, "drumless") { + t.Errorf("candidates = %+v", cands) + } + g, err := q.GetLiveGrab(ctx, w.ID) + if err != nil { + t.Fatal(err) + } + if g.InfoHash != ramHash || g.Client != "exchange" || g.Origin != acquire.OriginSchedule || g.Quality != "lossless" { + t.Errorf("grab = %+v", g) + } + settings, _ := q.GetAcquisitionSettings(ctx) + if !settings.LastPassAt.Valid || !strings.Contains(string(settings.LastPass), `"grabbed":1`) { + t.Errorf("last pass = %s", settings.LastPass) + } + + // A want with a download running is not searched again. + if rep := s.Pass(ctx); rep.Looked != 0 { + t.Errorf("searched a want already downloading: %+v", rep) + } + + // Setting the drumless edition aside marks it on every kept candidate; + // allowing it again leaves it for a person until the next search. + mem := acquire.NewMemory(pool) + if err := mem.Block(ctx, dbq.BlockReleaseParams{ + InfoHash: cands[1].InfoHash, Name: cands[1].Name, Reason: acquire.BlockOperator, WantID: w.ID, + }); err != nil { + t.Fatal(err) + } + c, _ := q.GetWantCandidate(ctx, dbq.GetWantCandidateParams{WantID: w.ID, Key: cands[1].Key}) + if c.Verdict != "blocked" { + t.Errorf("blocked candidate = %s", c.Verdict) + } + if ok, err := mem.Unblock(ctx, cands[1].InfoHash); err != nil || !ok { + t.Fatalf("unblock: %v %v", ok, err) + } + c, _ = q.GetWantCandidate(ctx, dbq.GetWantCandidateParams{WantID: w.ID, Key: cands[1].Key}) + if c.Verdict != "review" { + t.Errorf("unblocked candidate = %s", c.Verdict) + } +} + +func TestSearcher_OffAndUnconfigured_Integration(t *testing.T) { + ctx := context.Background() + pool := newPool(t) + s := NewSearcher(pool, nil, slog.New(slog.NewTextHandler(io.Discard, nil))) + if rep := s.Pass(ctx); !rep.Unconfigured { + t.Errorf("no indexer: %+v", rep) + } + if _, err := dbq.New(pool).UpdateAcquisitionSettings(ctx, dbq.UpdateAcquisitionSettingsParams{ + AutoGrab: true, SearchIntervalHours: 0, AllowedQualities: []string{"lossless"}, + }); err != nil { + t.Fatal(err) + } + if rep := s.Pass(ctx); !rep.Off { + t.Errorf("interval 0: %+v", rep) + } +} + +func TestGrabs_OneLivePerWant_Integration(t *testing.T) { + ctx := context.Background() + pool := newPool(t) + q := dbq.New(pool) + w, err := WantWith(ctx, q, WantParams{GroupMBID: "rg-x", ArtistMBID: "mb-x", ArtistName: "X", Title: "Y", Reason: ReasonFollow}) + if err != nil { + t.Fatal(err) + } + mem := acquire.NewMemory(pool) + grab := func(hash string) dbq.InsertGrabParams { + return dbq.InsertGrabParams{WantID: w.ID, InfoHash: hash, Name: hash, Key: hash, Client: "qbittorrent", + SavePath: "/downloads/" + hash, Origin: acquire.OriginSchedule} + } + first, err := mem.Grabbed(ctx, grab("aaaa"), dbq.Grab{}.ID, "") + if err != nil { + t.Fatal(err) + } + if _, err := mem.Grabbed(ctx, grab("bbbb"), dbq.Grab{}.ID, ""); err == nil { + t.Error("a second live grab for one want was recorded") + } + second, err := mem.Grabbed(ctx, grab("bbbb"), first.ID, "Replaced by bbbb.") + if err != nil { + t.Fatalf("replacing: %v", err) + } + live, ok, err := mem.LiveGrab(ctx, w.ID) + if err != nil || !ok || live.ID != second.ID { + t.Errorf("live = %+v %v %v", live, ok, err) + } +} diff --git a/web/src/lib/api/queries.ts b/web/src/lib/api/queries.ts index f8c99725..8abd41d4 100644 --- a/web/src/lib/api/queries.ts +++ b/web/src/lib/api/queries.ts @@ -45,6 +45,9 @@ export const qk = { adminFollows: () => ['adminFollows'] as const, adminFollow: (artistMbid: string) => ['adminFollow', artistMbid] as const, releaseProfiles: () => ['releaseProfiles'] as const, + // Search and grab (M502 #5521). + wantCandidates: (wantId: string) => ['wantCandidates', wantId] as const, + searchSettings: () => ['searchSettings'] as const, myQuarantine: () => ['myQuarantine'] as const, adminQuarantine: () => ['adminQuarantine'] as const, adminQuarantineActions: (limit?: number) => diff --git a/web/src/lib/api/types.ts b/web/src/lib/api/types.ts index 995f350c..783a5985 100644 --- a/web/src/lib/api/types.ts +++ b/web/src/lib/api/types.ts @@ -590,6 +590,101 @@ export type Want = { wanted_at: string; held_at: string | null; image_url: string; + // The search schedule's last look, and the download running (#5521). + search?: WantSearch | null; + grab?: WantGrab | null; +}; + +export type SearchOutcome = 'grabbed' | 'waiting' | 'nothing' | 'error'; + +export type WantSearch = { + searched_at: string; + outcome: SearchOutcome; + considered: number; + best_release: string; + best_verdict: string; + error: string; +}; + +export type WantGrab = { + name: string; + grabbed_at: string; + origin: 'schedule' | 'operator'; + quality: string; +}; + +// Search and grab (M502 #5521) ---------------------------------------------- + +export type CandidateVerdict = 'grab' | 'review' | 'refused' | 'rejected' | 'blocked'; + +// One release a want's last search found, with what was decided about it. +export type Candidate = { + key: string; + name: string; + info_hash: string; + indexer: string; + size_bytes: number; + seeders: number; + files_count: number; + format: string; + quality: string; + year: number; + verdict: CandidateVerdict; + reason: string; + reasons: { kind: 'hard' | 'soft'; code: string; text: string }[]; + folded_from: string[]; + searched_at: string; + downloading: boolean; +}; + +export type Narrowing = { source: string; label: string; reason: string }; + +export type SearchResult = { + outcome: SearchOutcome; + considered: number; + best_release: string; + best_verdict: string; + grabbed?: string; + narrowed: Narrowing[]; + error?: string; +}; + +export type GrabResult = { + added: boolean; + refused: boolean; + gone: boolean; + unconfigured: boolean; + missing?: string; + kept: boolean; + replaced: boolean; + verdict?: CandidateVerdict; + reason?: string; + error?: string; +}; + +export type QualityTier = { tier: string; label: string; what: string }; + +// The last scheduled pass; empty before the first. +export type SearchPass = { + looked?: number; + grabbed?: number; + waiting?: number; + nothing?: number; + failed?: number; + paced?: boolean; + off?: boolean; + unconfigured?: boolean; + auto_grab?: boolean; + error?: string; +}; + +export type SearchSettings = { + auto_grab: boolean; + search_interval_hours: number; + allowed_qualities: string[]; + tiers: QualityTier[]; + last_pass_at: string | null; + last_pass: SearchPass; }; export type WantCounts = { wanted: number; missing: number; held: number; following: number }; diff --git a/web/src/lib/api/wanted.ts b/web/src/lib/api/wanted.ts index 39f95cec..4d201d45 100644 --- a/web/src/lib/api/wanted.ts +++ b/web/src/lib/api/wanted.ts @@ -2,11 +2,15 @@ import { createInfiniteQuery, createQuery } from '@tanstack/svelte-query'; import { api } from './client'; import { qk } from './queries'; import type { + Candidate, Follow, FollowState, ReleaseProfile, ReleaseProfileInput, ReleaseProfilesResponse, + GrabResult, + SearchResult, + SearchSettings, WantState, WantsPage } from './types'; @@ -100,3 +104,59 @@ export async function updateReleaseProfile(id: string, body: ReleaseProfileInput export async function deleteReleaseProfile(id: string): Promise { await api.del(`/api/admin/release-profiles/${id}`); } + +// Search and grab (M502 #5521): each want's last search, a person's Search, +// Grab and "never this one", and the schedule's settings. + +export async function listCandidates(wantId: string): Promise { + const res = await api.get<{ items: Candidate[] }>(`/api/admin/wanted/${wantId}/candidates`); + return res.items; +} + +export function createCandidatesQuery(wantId: string) { + return createQuery({ + queryKey: qk.wantCandidates(wantId), + queryFn: () => listCandidates(wantId), + staleTime: 30_000 + }); +} + +// Searches now, as the schedule would: takes the best release when +// auto-grab is on. Can take most of a minute. +export async function searchWant(wantId: string): Promise { + return api.post(`/api/admin/wanted/${wantId}/search`, {}); +} + +// Takes the release named by key; force takes what the verdict would not. +export async function grabCandidate(wantId: string, key: string, force: boolean): Promise { + return api.post(`/api/admin/wanted/${wantId}/grab`, { key, force }); +} + +// Never takes this release again, for any want. +export async function blockCandidate(wantId: string, key: string): Promise { + await api.post(`/api/admin/wanted/${wantId}/block`, { key }); +} + +export async function unblockRelease(infoHash: string): Promise { + await api.del(`/api/admin/release-blocks/${infoHash}`); +} + +export async function getSearchSettings(): Promise { + return api.get('/api/admin/acquisition/settings'); +} + +export function createSearchSettingsQuery() { + return createQuery({ + queryKey: qk.searchSettings(), + queryFn: getSearchSettings, + staleTime: 30_000 + }); +} + +export async function putSearchSettings(body: { + auto_grab: boolean; + search_interval_hours: number; + allowed_qualities: string[]; +}): Promise { + return api.put('/api/admin/acquisition/settings', body); +} diff --git a/web/src/lib/components/SearchSettingsCard.svelte b/web/src/lib/components/SearchSettingsCard.svelte new file mode 100644 index 00000000..fb97fdcf --- /dev/null +++ b/web/src/lib/components/SearchSettingsCard.svelte @@ -0,0 +1,162 @@ + + +
+
+

Searching and grabbing

+

+ Minstrel searches your indexers for each album on the + wanted list, judges every + release it finds, and hands the best one to your download client. Each album's releases and + the reason for every verdict are on the Wanted page. +

+
+ + {#if query.isPending} +

Loading…

+ {:else if query.isError || !form || !query.data} +

Couldn't load the search settings.

+ {:else} + + + + +
+ Qualities to take + + The best of these is taken first. A release that names no format has its file list read to + find out. + +
+ {#each query.data.tiers as t (t.tier)} + + {/each} +
+
+ +

{lastPass(query.data)}

+ +
+ +
+ {/if} +
diff --git a/web/src/lib/components/SearchSettingsCard.test.ts b/web/src/lib/components/SearchSettingsCard.test.ts new file mode 100644 index 00000000..c5b919be --- /dev/null +++ b/web/src/lib/components/SearchSettingsCard.test.ts @@ -0,0 +1,69 @@ +import { afterEach, describe, expect, test, vi } from 'vitest'; +import { fireEvent, render, screen, waitFor } from '@testing-library/svelte'; +import { mockQuery } from '../../test-utils/query'; +import type { SearchSettings } from '#lib/api/types.js'; + +vi.mock('#lib/api/wanted.js', () => ({ + createSearchSettingsQuery: vi.fn(), + putSearchSettings: vi.fn() +})); +vi.mock('#lib/stores/toast.svelte.js', () => ({ pushToast: vi.fn() })); + +import SearchSettingsCard from './SearchSettingsCard.svelte'; +import { createSearchSettingsQuery, putSearchSettings } from '#lib/api/wanted.js'; + +const settings: SearchSettings = { + auto_grab: true, + search_interval_hours: 24, + allowed_qualities: ['lossless', 'lossy_high'], + tiers: [ + { tier: 'lossless', label: 'Lossless', what: 'FLAC, ALAC' }, + { tier: 'lossy_high', label: 'High quality lossy', what: 'MP3 320 or V0' }, + { tier: 'unknown', label: 'Unknown', what: 'Nothing says what format it is' } + ], + last_pass_at: new Date(Date.now() - 5 * 60_000).toISOString(), + last_pass: { looked: 12, grabbed: 3, waiting: 2, nothing: 7, failed: 0, paced: true } +}; + +afterEach(() => vi.clearAllMocks()); + +function setup(s: SearchSettings = settings) { + vi.mocked(createSearchSettingsQuery).mockReturnValue( + mockQuery({ data: s }) as unknown as ReturnType + ); + vi.mocked(putSearchSettings).mockImplementation(async (body) => ({ ...s, ...body })); + return render(SearchSettingsCard); +} + +describe('SearchSettingsCard', () => { + test('shows the settings and what the last pass did', () => { + setup(); + expect((screen.getByLabelText(/take the best release without asking/i) as HTMLInputElement).checked).toBe(true); + expect((screen.getByLabelText(/hours between searches/i) as HTMLInputElement).value).toBe('24'); + expect((screen.getByLabelText(/^unknown/i) as HTMLInputElement).checked).toBe(false); + const last = screen.getByTestId('last-pass').textContent ?? ''; + expect(last).toMatch(/searched 12, took 3, 2 waiting for you, 7 with nothing found\./); + expect(last).toMatch(/searches per hour/); + }); + + test('saves what changed', async () => { + setup(); + const save = screen.getByRole('button', { name: /^save$/i }) as HTMLButtonElement; + expect(save.disabled).toBe(true); + await fireEvent.click(screen.getByLabelText(/take the best release without asking/i)); + await fireEvent.click(screen.getByLabelText(/^unknown/i)); + await fireEvent.click(save); + await waitFor(() => + expect(putSearchSettings).toHaveBeenCalledWith({ + auto_grab: false, + search_interval_hours: 24, + allowed_qualities: ['lossless', 'lossy_high', 'unknown'] + }) + ); + }); + + test('before the first pass it says so', () => { + setup({ ...settings, last_pass_at: null, last_pass: {} }); + expect(screen.getByTestId('last-pass').textContent).toMatch(/hasn’t run yet/); + }); +}); diff --git a/web/src/lib/components/WantCandidates.svelte b/web/src/lib/components/WantCandidates.svelte new file mode 100644 index 00000000..878eabc8 --- /dev/null +++ b/web/src/lib/components/WantCandidates.svelte @@ -0,0 +1,184 @@ + + +
+ {#if query.isPending} +

Loading what the last search found…

+ {:else if query.isError} +

Couldn't load the releases.

+ {:else if items.length === 0} +

+ Nothing kept from a search yet. Search now, or wait for the schedule. +

+ {:else} +

+ {items.length === 1 ? '1 release' : `${items.length} releases`} found {relativeTime(items[0].searched_at)}, best first. +

+
    + {#each items as c (c.key)} +
  • +
    +
    {c.name}
    +
    {meta(c)}
    +
    + + {c.downloading ? 'Downloading' : verdictLabel[c.verdict]} + + {c.reason} +
    +
    +
    + {#if c.verdict === 'blocked'} + + {:else if !c.downloading} + + {#if c.info_hash} + + {/if} + {/if} +
    +
    • + {/each} +
+ {/if} +
diff --git a/web/src/lib/components/WantCandidates.test.ts b/web/src/lib/components/WantCandidates.test.ts new file mode 100644 index 00000000..3c7fcecf --- /dev/null +++ b/web/src/lib/components/WantCandidates.test.ts @@ -0,0 +1,90 @@ +import { afterEach, describe, expect, test, vi } from 'vitest'; +import { fireEvent, render, screen, waitFor } from '@testing-library/svelte'; +import { mockQuery } from '../../test-utils/query'; +import type { Candidate } from '#lib/api/types.js'; + +vi.mock('#lib/api/wanted.js', () => ({ + createCandidatesQuery: vi.fn(), + grabCandidate: vi.fn().mockResolvedValue({ added: true, refused: false, gone: false, unconfigured: false, kept: false, replaced: false }), + blockCandidate: vi.fn().mockResolvedValue(undefined), + unblockRelease: vi.fn().mockResolvedValue(undefined) +})); +vi.mock('#lib/stores/toast.svelte.js', () => ({ pushToast: vi.fn() })); + +import WantCandidates from './WantCandidates.svelte'; +import { blockCandidate, createCandidatesQuery, grabCandidate, unblockRelease } from '#lib/api/wanted.js'; +import { pushToast } from '#lib/stores/toast.svelte.js'; + +function cand(key: string, extra: Partial = {}): Candidate { + return { + key, + name: `Daft Punk - Random Access Memories (2013) [${key}]`, + info_hash: key, + indexer: 'Knaben', + size_bytes: 450_000_000, + seeders: 12, + files_count: 13, + format: 'FLAC', + quality: 'lossless', + year: 2013, + verdict: 'grab', + reason: 'The album, FLAC.', + reasons: [], + folded_from: [], + searched_at: new Date().toISOString(), + downloading: false, + ...extra + }; +} + +afterEach(() => vi.clearAllMocks()); + +function setup(items: Candidate[]) { + vi.mocked(createCandidatesQuery).mockReturnValue( + mockQuery({ data: items }) as unknown as ReturnType + ); + return render(WantCandidates, { wantId: 'w1', title: 'Random Access Memories' }); +} + +describe('WantCandidates', () => { + test('every release shows its verdict and why', () => { + setup([ + cand('aaaa'), + cand('bbbb', { verdict: 'rejected', reason: 'This is a drumless edition (“drumless”).' }) + ]); + expect(screen.getAllByTestId('candidate-row')).toHaveLength(2); + expect(screen.getByText('Would take')).toBeTruthy(); + expect(screen.getByText('Not the album')).toBeTruthy(); + expect(screen.getByText(/drumless edition/)).toBeTruthy(); + expect(screen.getAllByText(/FLAC · 450 MB · 12 seeders · Knaben/)).toHaveLength(2); + }); + + test('a release the verdict would take is grabbed; any other is grabbed anyway', async () => { + setup([cand('aaaa'), cand('bbbb', { verdict: 'review', reason: 'Nobody is sharing it right now.' })]); + await fireEvent.click(screen.getByRole('button', { name: /^grab daft punk.*\[aaaa\]/i })); + await waitFor(() => expect(grabCandidate).toHaveBeenCalledWith('w1', 'aaaa', false)); + await waitFor(() => expect(pushToast).toHaveBeenCalledWith('Taken for Random Access Memories.', 'info')); + await fireEvent.click(screen.getByRole('button', { name: /^grab anyway daft punk.*\[bbbb\]/i })); + await waitFor(() => expect(grabCandidate).toHaveBeenCalledWith('w1', 'bbbb', true)); + }); + + test('a release can be set aside, and allowed again', async () => { + setup([cand('aaaa'), cand('bbbb', { verdict: 'blocked', reason: 'You set this release aside.' })]); + await fireEvent.click(screen.getByRole('button', { name: /never take .*\[aaaa\]/i })); + await waitFor(() => expect(blockCandidate).toHaveBeenCalledWith('w1', 'aaaa')); + await fireEvent.click(screen.getByRole('button', { name: /allow .*\[bbbb\] again/i })); + await waitFor(() => expect(unblockRelease).toHaveBeenCalledWith('bbbb')); + expect(screen.queryByRole('button', { name: /grab.*\[bbbb\]/i })).toBeNull(); + }); + + test('the release downloading says so and offers nothing', () => { + setup([cand('aaaa', { downloading: true })]); + expect(screen.getByText('Downloading')).toBeTruthy(); + expect(screen.queryByRole('button', { name: /grab/i })).toBeNull(); + }); + + test('before any search, it says how to get one', () => { + setup([]); + expect(screen.getByText(/nothing kept from a search yet/i)).toBeTruthy(); + }); +}); diff --git a/web/src/lib/styles/error-copy.json b/web/src/lib/styles/error-copy.json index 68a365d9..90abec17 100644 --- a/web/src/lib/styles/error-copy.json +++ b/web/src/lib/styles/error-copy.json @@ -67,5 +67,11 @@ "profile_is_default": "The default profile can't be deleted. Make another one the default first.", "want_not_found": "That's no longer on the wanted list. Refresh the page.", "follow_not_found": "That artist isn't followed any more. Refresh the page.", - "invalid_profile": "That isn't a release-type profile." + "invalid_profile": "That isn't a release-type profile.", + "bad_interval": "Hours between searches must be between 0 and 720.", + "bad_quality": "Choose at least one quality to take.", + "candidate_not_found": "That release is no longer in the list. Search again.", + "no_infohash": "That release can't be set aside: its indexer gave no infohash to know it by.", + "want_held": "The library already holds that album.", + "block_not_found": "That release isn't set aside any more." } diff --git a/web/src/routes/admin/integrations/+page.svelte b/web/src/routes/admin/integrations/+page.svelte index fa9be790..94542a0d 100644 --- a/web/src/routes/admin/integrations/+page.svelte +++ b/web/src/routes/admin/integrations/+page.svelte @@ -30,6 +30,7 @@ import AcoustIDCard from '#lib/components/AcoustIDCard.svelte'; import AcquisitionCard from '#lib/components/AcquisitionCard.svelte'; import ReleaseProfilesCard from '#lib/components/ReleaseProfilesCard.svelte'; + import SearchSettingsCard from '#lib/components/SearchSettingsCard.svelte'; import NetworkSettingsCard from '#lib/components/NetworkSettingsCard.svelte'; import PublicAddressCard from '#lib/components/PublicAddressCard.svelte'; import NotificationEmailCard from '#lib/components/NotificationEmailCard.svelte'; @@ -424,6 +425,8 @@ + + diff --git a/web/src/routes/admin/integrations/integrations.test.ts b/web/src/routes/admin/integrations/integrations.test.ts index e5449fd7..26b475d7 100644 --- a/web/src/routes/admin/integrations/integrations.test.ts +++ b/web/src/routes/admin/integrations/integrations.test.ts @@ -60,7 +60,22 @@ vi.mock('#lib/api/wanted.js', async () => { ), createReleaseProfile: vi.fn(), updateReleaseProfile: vi.fn(), - deleteReleaseProfile: vi.fn() + deleteReleaseProfile: vi.fn(), + // SearchSettingsCard, settled the same way. + createSearchSettingsQuery: vi.fn(() => + mockQuery({ + data: { + auto_grab: true, + search_interval_hours: 24, + // Not "Lossless": a Lidarr quality profile on this page is named that. + allowed_qualities: ['lossy_high'], + tiers: [{ tier: 'lossy_high', label: 'High quality lossy', what: 'MP3 320' }], + last_pass_at: null, + last_pass: {} + } + }) + ), + putSearchSettings: vi.fn() }; }); diff --git a/web/src/routes/admin/wanted/+page.svelte b/web/src/routes/admin/wanted/+page.svelte index 1fed0c87..d791c2d5 100644 --- a/web/src/routes/admin/wanted/+page.svelte +++ b/web/src/routes/admin/wanted/+page.svelte @@ -1,7 +1,7 @@