feat: search and grab — music release names, a quality ladder, a paced search schedule and one live grab per want (M502 #5521)
release / govulncheck (push) Successful in 21s
release / go (push) Failing after 33s
release / web (push) Successful in 56s
release / integration (push) Failing after 4m48s
release / android (push) Successful in 5m6s
release / Build signed APK (releases and dev) (push) Successful in 5m16s
release / Attach APK to the Release (tag releases only) (push) Skipped
release / Build + push container image (push) Skipped
release / Verify release artifacts (tag releases only) (push) Skipped
release / govulncheck (push) Successful in 21s
release / go (push) Failing after 33s
release / web (push) Successful in 56s
release / integration (push) Failing after 4m48s
release / android (push) Successful in 5m6s
release / Build signed APK (releases and dev) (push) Successful in 5m16s
release / Attach APK to the Release (tag releases only) (push) Skipped
release / Build + push container image (push) Skipped
release / Verify release artifacts (tag releases only) (push) Skipped
- internal/release: reads a release name (format, bitrate, bit depth and sample rate, source, year, video, discography) and judges it against the album wanted: the artist and title must be in it; video rips, discographies, other versions (drumless, instrumental, karaoke, remix, live unless the album is live…) and singles or EPs for an album are refused. Quality tiers on Lidarr's ladder; a .torrent's file list is read for format, video files and a track count short of the album. - internal/acquire: fan-out across every search source with a narrowed search kept, a process-wide pacer per source, every candidate judged and kept with its reasons, and the grab: one live grab per want, tie-broken (a person's grab beats the schedule's; a better release replaces a worse one unless it is nearly done). Each download lands in its own folder under the client's downloads folder. - internal/wanted: the search schedule, every ten minutes, never-searched first, then requests, missing files and followed artists, taking the best release when auto-grab is on (the default). - Migration 0082: acquisition_settings (auto-grab, interval, allowed qualities seeded from Lidarr's "Any"), want search columns, want_candidates, grabs, release_blocks. - Web: the Wanted page shows each want's last search, a Search button and every release found with its verdict (Grab, Grab anyway, Never this one, Allow again). Integrations gains "Searching and grabbing". Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,45 @@
|
||||
package acquire
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
|
||||
"git.fabledsword.com/bvandeusen/minstrel/internal/db/dbq"
|
||||
"git.fabledsword.com/bvandeusen/minstrel/internal/release"
|
||||
)
|
||||
|
||||
// Build makes a Service from the settings as they stand: every enabled
|
||||
// search source behind one paced Fanout, the first enabled download client,
|
||||
// and the quality profile. Built afresh for each pass and each request, so a
|
||||
// changed setting takes effect on the next one.
|
||||
func Build(ctx context.Context, pool *pgxpool.Pool) (*Service, dbq.AcquisitionSetting, error) {
|
||||
q := dbq.New(pool)
|
||||
settings, err := q.GetAcquisitionSettings(ctx)
|
||||
if err != nil {
|
||||
return nil, settings, fmt.Errorf("acquire: read settings: %w", err)
|
||||
}
|
||||
sources, err := Sources(ctx, pool)
|
||||
if err != nil {
|
||||
return nil, settings, fmt.Errorf("acquire: read search sources: %w", err)
|
||||
}
|
||||
clients, err := q.ListUsableAcquisitionServices(ctx, JobDownload)
|
||||
if err != nil {
|
||||
return nil, settings, fmt.Errorf("acquire: read download clients: %w", err)
|
||||
}
|
||||
s := &Service{Profile: release.NewProfile(settings.AllowedQualities), Memory: NewMemory(pool)}
|
||||
fan := &Fanout{Sources: sources, Pace: SearchPace}
|
||||
if len(sources) > 0 {
|
||||
s.Search = fan
|
||||
}
|
||||
if len(clients) > 0 {
|
||||
row := clients[0]
|
||||
client, err := DownloadClient(row)
|
||||
if err != nil {
|
||||
return nil, settings, fmt.Errorf("acquire: download client %s: %w", row.Code, err)
|
||||
}
|
||||
s.Grabber = &Downloader{Client: client, Code: row.Code, Landing: row.LocalPath, Fetch: fan}
|
||||
}
|
||||
return s, settings, nil
|
||||
}
|
||||
@@ -0,0 +1,165 @@
|
||||
package acquire
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
"git.fabledsword.com/bvandeusen/minstrel/internal/indexer"
|
||||
)
|
||||
|
||||
// Searching every source at once (M502 #5521), ported from Fabled Librarian
|
||||
// (#4182). A source that is down narrows a search rather than failing it: the
|
||||
// results come back with the list of who did not answer.
|
||||
|
||||
// SourceFailure is one source that could not be reached.
|
||||
type SourceFailure struct {
|
||||
Code string
|
||||
Label string
|
||||
Err error
|
||||
}
|
||||
|
||||
// PartialError is a search that reached some sources and not others. It is
|
||||
// returned alongside the results, not instead of them.
|
||||
type PartialError struct {
|
||||
Failures []SourceFailure
|
||||
// Answered is how many sources did answer. Zero means the search failed.
|
||||
Answered int
|
||||
}
|
||||
|
||||
func (e *PartialError) Error() string {
|
||||
if e == nil || len(e.Failures) == 0 {
|
||||
return "some sources did not answer"
|
||||
}
|
||||
parts := make([]string, 0, len(e.Failures))
|
||||
for _, f := range e.Failures {
|
||||
parts = append(parts, fmt.Sprintf("%s: %s", f.Label, indexer.FirstLine(f.Err.Error())))
|
||||
}
|
||||
return "some sources did not answer — " + strings.Join(parts, "; ")
|
||||
}
|
||||
|
||||
// Unwrap lets errors.Is find indexer.ErrUnreachable through a partial failure.
|
||||
func (e *PartialError) Unwrap() []error {
|
||||
if e == nil {
|
||||
return nil
|
||||
}
|
||||
out := make([]error, 0, len(e.Failures))
|
||||
for _, f := range e.Failures {
|
||||
out = append(out, f.Err)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// Fanout searches every source and merges what comes back. It is both the
|
||||
// Searcher and the Fetcher, routing a .torrent fetch to the source that
|
||||
// offered the result.
|
||||
type Fanout struct {
|
||||
// Sources in code order: the fold keeps a duplicate's first arrival, so
|
||||
// the order decides which source a grab is fetched through.
|
||||
Sources []Source
|
||||
// Pace spaces each source's searches. Nil paces nothing.
|
||||
Pace *Pacer
|
||||
}
|
||||
|
||||
// Search asks every source concurrently. Each client bounds its own call by
|
||||
// its configured timeout, so there is no shared deadline here.
|
||||
func (f *Fanout) Search(ctx context.Context, query indexer.Query) ([]indexer.Result, error) {
|
||||
if f == nil || len(f.Sources) == 0 {
|
||||
return nil, indexer.ErrUnconfigured
|
||||
}
|
||||
found := make([][]indexer.Result, len(f.Sources))
|
||||
errs := make([]error, len(f.Sources))
|
||||
var wg sync.WaitGroup
|
||||
for i, src := range f.Sources {
|
||||
if src.Search == nil {
|
||||
continue
|
||||
}
|
||||
f.Pace.spend(src)
|
||||
wg.Add(1)
|
||||
go func(i int, src Source) {
|
||||
defer wg.Done()
|
||||
found[i], errs[i] = src.Search.Search(ctx, query)
|
||||
}(i, src)
|
||||
}
|
||||
wg.Wait()
|
||||
|
||||
out := make([]indexer.Result, 0, 32)
|
||||
var failures []SourceFailure
|
||||
answered, unconfigured := 0, 0
|
||||
for i, src := range f.Sources {
|
||||
if src.Search == nil {
|
||||
continue
|
||||
}
|
||||
if errors.Is(errs[i], indexer.ErrUnconfigured) {
|
||||
unconfigured++
|
||||
continue
|
||||
}
|
||||
if rl, ok := indexer.AsRateLimited(errs[i]); ok {
|
||||
f.Pace.hold(src.Code, rl)
|
||||
}
|
||||
if errs[i] != nil {
|
||||
failures = append(failures, SourceFailure{Code: src.Code, Label: src.Label, Err: errs[i]})
|
||||
continue
|
||||
}
|
||||
answered++
|
||||
for j := range found[i] {
|
||||
if found[i][j].Source == "" {
|
||||
found[i][j].Source = src.Code
|
||||
}
|
||||
}
|
||||
out = append(out, found[i]...)
|
||||
}
|
||||
switch {
|
||||
case answered == 0 && len(failures) > 0:
|
||||
return nil, &PartialError{Failures: failures}
|
||||
case len(failures) > 0:
|
||||
return out, &PartialError{Failures: failures, Answered: answered}
|
||||
case answered == 0 && unconfigured > 0:
|
||||
return nil, indexer.ErrUnconfigured
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// Download fetches a result's .torrent from the source that offered it: a
|
||||
// Prowlarr link carries Prowlarr's key and only Prowlarr can redeem it.
|
||||
func (f *Fanout) Download(ctx context.Context, r indexer.Result) ([]byte, error) {
|
||||
if f == nil {
|
||||
return nil, indexer.ErrUnconfigured
|
||||
}
|
||||
for _, src := range f.Sources {
|
||||
if src.Code != r.Source {
|
||||
continue
|
||||
}
|
||||
if src.Fetch == nil {
|
||||
return nil, fmt.Errorf("%w: %s cannot fetch torrent files", indexer.ErrUnconfigured, src.Label)
|
||||
}
|
||||
return src.Fetch.Download(ctx, r)
|
||||
}
|
||||
if r.Source == "" {
|
||||
return nil, fmt.Errorf("%w: this result does not say which source it came from", indexer.ErrUnconfigured)
|
||||
}
|
||||
return nil, fmt.Errorf("%w: %q is no longer a configured source", indexer.ErrUnconfigured, r.Source)
|
||||
}
|
||||
|
||||
// Narrowing is one source that did not answer, as a page shows it.
|
||||
type Narrowing struct {
|
||||
Source string `json:"source"`
|
||||
Label string `json:"label"`
|
||||
Reason string `json:"reason"`
|
||||
}
|
||||
|
||||
// narrowings reads a PartialError out of err, and whether any source
|
||||
// answered. Nil when err is not a partial failure.
|
||||
func narrowings(err error) ([]Narrowing, bool) {
|
||||
var partial *PartialError
|
||||
if !errors.As(err, &partial) || len(partial.Failures) == 0 {
|
||||
return nil, false
|
||||
}
|
||||
out := make([]Narrowing, 0, len(partial.Failures))
|
||||
for _, f := range partial.Failures {
|
||||
out = append(out, Narrowing{Source: f.Code, Label: f.Label, Reason: indexer.FirstLine(f.Err.Error())})
|
||||
}
|
||||
return out, partial.Answered > 0
|
||||
}
|
||||
@@ -0,0 +1,307 @@
|
||||
package acquire
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"path"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
"github.com/jackc/pgx/v5/pgtype"
|
||||
|
||||
"git.fabledsword.com/bvandeusen/minstrel/internal/db/dbq"
|
||||
"git.fabledsword.com/bvandeusen/minstrel/internal/downloader"
|
||||
"git.fabledsword.com/bvandeusen/minstrel/internal/indexer"
|
||||
"git.fabledsword.com/bvandeusen/minstrel/internal/release"
|
||||
)
|
||||
|
||||
// Taking a release (M502 #5521), ported from Fabled Librarian's grab.go and
|
||||
// rivals.go. One live grab per want: a second is tie-broken against the
|
||||
// download already running (FL #4470).
|
||||
|
||||
// Tag goes on every torrent Minstrel adds, so the operator can tell them
|
||||
// from their own in the client.
|
||||
const Tag = "minstrel"
|
||||
|
||||
// MissingClient is Taken.Missing when no download client is set up.
|
||||
const MissingClient = "download_client"
|
||||
|
||||
// nearlyDone is how far along a running download stays whatever would
|
||||
// replace it.
|
||||
const nearlyDone = 0.9
|
||||
|
||||
// Downloader is the grab half of a Service.
|
||||
type Downloader struct {
|
||||
Client downloader.Driver
|
||||
// Code is the client's acquisition_services row.
|
||||
Code string
|
||||
// Landing is the client's downloads folder as Minstrel sees it; each
|
||||
// grab lands in a folder of its own under it.
|
||||
Landing string
|
||||
// Fetch gets a result's .torrent from the source that offered it.
|
||||
Fetch Fetcher
|
||||
}
|
||||
|
||||
// Origin says who took a release.
|
||||
type Origin struct {
|
||||
By string // OriginSchedule or OriginOperator
|
||||
User pgtype.UUID
|
||||
}
|
||||
|
||||
// Contest is how a grab fared against the download already running.
|
||||
type Contest struct {
|
||||
// Kept: the running one stays and this grab was not made. Replaced:
|
||||
// this grab took its place.
|
||||
Kept, Replaced bool
|
||||
Reason string
|
||||
Rival dbq.Grab
|
||||
}
|
||||
|
||||
// Taken is what came of a grab.
|
||||
type Taken struct {
|
||||
Chosen *Candidate
|
||||
// Added is the only field that means a download started.
|
||||
Added bool
|
||||
SavePath string
|
||||
Grab dbq.Grab
|
||||
// Gone: the search no longer offers the release.
|
||||
Gone bool
|
||||
// Refused: not taken; the reason is on Chosen.
|
||||
Refused bool
|
||||
// Unconfigured: Missing names what is not set up ("" for search).
|
||||
Unconfigured bool
|
||||
Missing string
|
||||
Contest *Contest
|
||||
Error string
|
||||
}
|
||||
|
||||
// Take searches again and grabs the candidate named by key: a person's
|
||||
// grab. Searching again keeps the indexer's download link (which can carry
|
||||
// its key) on the server, and finds a release gone since it was shown.
|
||||
// force takes what the verdict would not.
|
||||
func (s *Service) Take(ctx context.Context, want Want, key string, force bool, origin Origin) Taken {
|
||||
if s.Grabber == nil {
|
||||
return Taken{Unconfigured: true, Missing: MissingClient}
|
||||
}
|
||||
found := s.For(ctx, want)
|
||||
switch {
|
||||
case found.Unconfigured:
|
||||
return Taken{Unconfigured: true}
|
||||
case found.Error != "":
|
||||
return Taken{Error: found.Error}
|
||||
}
|
||||
chosen := pick(found.Candidates, key)
|
||||
if chosen == nil {
|
||||
return Taken{Gone: true}
|
||||
}
|
||||
return s.Grab(ctx, want, chosen, found.Candidates, force, origin)
|
||||
}
|
||||
|
||||
// Grab hands a judged candidate to the download client. The schedule calls
|
||||
// it with the candidate it has just judged; Take calls it after searching
|
||||
// again.
|
||||
func (s *Service) Grab(ctx context.Context, want Want, chosen *Candidate, all []Candidate, force bool, origin Origin) Taken {
|
||||
out := Taken{Chosen: chosen}
|
||||
if s.Grabber == nil || s.Grabber.Client == nil {
|
||||
out.Unconfigured, out.Missing = true, MissingClient
|
||||
return out
|
||||
}
|
||||
if chosen.Verdict == Blocked || (chosen.Verdict != Grab && !force) {
|
||||
out.Refused = true
|
||||
return out
|
||||
}
|
||||
|
||||
unlock := lockWant(want.ID)
|
||||
defer unlock()
|
||||
contest := s.contest(ctx, want, chosen, all, origin)
|
||||
if contest != nil && contest.Kept {
|
||||
out.Contest = contest
|
||||
return out
|
||||
}
|
||||
|
||||
add := downloader.Add{Magnet: chosen.Source.Magnet, Tags: []string{Tag}}
|
||||
hash := chosen.InfoHash
|
||||
if add.Magnet == "" {
|
||||
if s.Grabber.Fetch == nil {
|
||||
out.Error = "This release is a .torrent link, and nothing is set up to fetch it."
|
||||
return out
|
||||
}
|
||||
body, err := s.Grabber.Fetch.Download(ctx, chosen.Source)
|
||||
if magnet, ok := indexer.AsMagnet(err); ok {
|
||||
add.Magnet, err = magnet, nil
|
||||
}
|
||||
if err != nil {
|
||||
out.Error = indexer.FirstLine(err.Error())
|
||||
return out
|
||||
}
|
||||
if add.Magnet == "" {
|
||||
add.Torrent = body
|
||||
if meta, err := release.ParseTorrent(body); err == nil {
|
||||
if hash == "" {
|
||||
hash = meta.InfoHash
|
||||
}
|
||||
if refused := s.judgeFiles(ctx, want, chosen, hash, meta.Files, force); refused {
|
||||
out.Refused = true
|
||||
return out
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if hash == "" && add.Magnet != "" {
|
||||
hash = indexer.MagnetHash(add.Magnet)
|
||||
}
|
||||
add.SavePath = landingFor(s.Grabber.Landing, hash, chosen.Name)
|
||||
|
||||
if err := s.Grabber.Client.AddTorrent(ctx, add); err != nil && !errors.Is(err, downloader.ErrDuplicate) {
|
||||
if errors.Is(err, downloader.ErrUnconfigured) {
|
||||
out.Unconfigured, out.Missing = true, MissingClient
|
||||
} else {
|
||||
out.Error = indexer.FirstLine(err.Error())
|
||||
}
|
||||
return out
|
||||
}
|
||||
out.Added, out.SavePath = true, add.SavePath
|
||||
if s.Memory == nil {
|
||||
return out
|
||||
}
|
||||
|
||||
var replaced pgtype.UUID
|
||||
why := ""
|
||||
if contest != nil && contest.Replaced {
|
||||
out.Contest = contest
|
||||
replaced, why = contest.Rival.ID, "Replaced by "+chosen.Name+": "+contest.Reason+"."
|
||||
}
|
||||
g, err := s.Memory.Grabbed(ctx, dbq.InsertGrabParams{
|
||||
WantID: want.ID, InfoHash: hash, Name: chosen.Name, Key: Key(*chosen), Indexer: chosen.Indexer,
|
||||
Source: chosen.Source.Source, Client: s.Grabber.Code, SavePath: add.SavePath,
|
||||
Quality: string(chosen.Tier), Origin: origin.By, Forced: force, GrabbedBy: origin.User,
|
||||
}, replaced, why)
|
||||
if err != nil {
|
||||
out.Error = "The download started, but could not be recorded: " + indexer.FirstLine(err.Error())
|
||||
return out
|
||||
}
|
||||
out.Grab = g
|
||||
// Removed only after the new download was added and recorded, so a
|
||||
// failure never leaves the want with nothing running.
|
||||
if replaced.Valid && contest.Rival.InfoHash != "" {
|
||||
if err := s.Grabber.Client.Delete(ctx, true, contest.Rival.InfoHash); err != nil {
|
||||
contest.Reason += ". The download it replaced could not be removed from the client: " + indexer.FirstLine(err.Error())
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// judgeFiles judges a chosen release again on its .torrent's file list, and
|
||||
// sets it aside when the files say it is not the album, so no later search
|
||||
// offers it again. force still takes it.
|
||||
func (s *Service) judgeFiles(ctx context.Context, want Want, chosen *Candidate, hash string, files []release.File, force bool) bool {
|
||||
chosen.Files = files
|
||||
if chosen.FilesCount == 0 {
|
||||
chosen.FilesCount = len(files)
|
||||
}
|
||||
veto, reason, _ := release.JudgeFiles(want.Want, files)
|
||||
if veto == "" || force {
|
||||
return false
|
||||
}
|
||||
set(chosen, Rejected, Hard, veto, reason)
|
||||
if s.Memory != nil && hash != "" {
|
||||
_ = s.Memory.Block(ctx, dbq.BlockReleaseParams{
|
||||
InfoHash: hash, Name: chosen.Name, Reason: BlockFiles, Detail: reason, WantID: want.ID,
|
||||
})
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
var wantLocks sync.Map // want id → *sync.Mutex
|
||||
|
||||
// lockWant settles two grabs for one want one after the other. The
|
||||
// one-live-grab index is the backstop for anything outside this process.
|
||||
func lockWant(id pgtype.UUID) func() {
|
||||
if !id.Valid {
|
||||
return func() {}
|
||||
}
|
||||
m, _ := wantLocks.LoadOrStore(id.Bytes, &sync.Mutex{})
|
||||
mu := m.(*sync.Mutex)
|
||||
mu.Lock()
|
||||
return mu.Unlock
|
||||
}
|
||||
|
||||
// contest settles a grab against the download running for its want: a
|
||||
// person's grab beats the schedule's; between two of a kind the better
|
||||
// candidate wins; a tie, a download nearly done, or one that cannot be
|
||||
// removed stays. Nil when nothing is running.
|
||||
func (s *Service) contest(ctx context.Context, want Want, chosen *Candidate, all []Candidate, origin Origin) *Contest {
|
||||
if s.Memory == nil || !want.ID.Valid {
|
||||
return nil
|
||||
}
|
||||
running, ok, err := s.Memory.LiveGrab(ctx, want.ID)
|
||||
if err != nil || !ok {
|
||||
return nil
|
||||
}
|
||||
c := &Contest{Rival: running}
|
||||
if running.InfoHash != "" && running.InfoHash == chosen.InfoHash {
|
||||
c.Kept, c.Reason = true, "That release is already downloading for this album."
|
||||
return c
|
||||
}
|
||||
switch {
|
||||
case origin.By == OriginOperator && running.Origin == OriginSchedule:
|
||||
c.Reason = "you chose it over the one the schedule started"
|
||||
case origin.By == OriginSchedule && running.Origin == OriginOperator:
|
||||
c.Kept, c.Reason = true, "A download you started for this album is already running."
|
||||
return c
|
||||
default:
|
||||
var theirs *Candidate
|
||||
for i := range all {
|
||||
if (running.InfoHash != "" && all[i].InfoHash == running.InfoHash) || Key(all[i]) == running.Key {
|
||||
theirs = &all[i]
|
||||
break
|
||||
}
|
||||
}
|
||||
if theirs == nil || !better(*chosen, *theirs) {
|
||||
c.Kept, c.Reason = true, "A download for this album is already running, and this release is no better."
|
||||
return c
|
||||
}
|
||||
c.Reason = "it ranks above the one that was running"
|
||||
}
|
||||
if running.InfoHash == "" {
|
||||
c.Kept, c.Reason = true, "A download for this album is already running, and it cannot be identified to replace it."
|
||||
return c
|
||||
}
|
||||
if ts, err := s.Grabber.Client.Torrents(ctx, running.InfoHash); err == nil && len(ts) > 0 && ts[0].Progress >= nearlyDone {
|
||||
c.Kept, c.Reason = true, "A download for this album is already running and nearly done."
|
||||
return c
|
||||
}
|
||||
c.Replaced = true
|
||||
return c
|
||||
}
|
||||
|
||||
// landingFor is the folder a download lands in: one per torrent, named by
|
||||
// its infohash, so two grabs never unpack into the same place.
|
||||
func landingFor(root, hash, name string) string {
|
||||
dir := hash
|
||||
if dir == "" {
|
||||
dir = slug(name)
|
||||
}
|
||||
if dir == "" {
|
||||
dir = "unnamed"
|
||||
}
|
||||
return path.Join(root, dir)
|
||||
}
|
||||
|
||||
// slug keeps a folder name to characters every filesystem in the chain
|
||||
// takes.
|
||||
func slug(s string) string {
|
||||
var b strings.Builder
|
||||
for _, r := range strings.ToLower(strings.TrimSpace(s)) {
|
||||
if b.Len() >= 80 {
|
||||
break
|
||||
}
|
||||
switch {
|
||||
case r >= 'a' && r <= 'z', r >= '0' && r <= '9':
|
||||
b.WriteRune(r)
|
||||
case r == '-', r == '_', r == ' ', r == '.':
|
||||
b.WriteRune('-')
|
||||
}
|
||||
}
|
||||
return strings.Trim(b.String(), "-")
|
||||
}
|
||||
@@ -0,0 +1,168 @@
|
||||
package acquire
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
|
||||
"github.com/jackc/pgx/v5"
|
||||
"github.com/jackc/pgx/v5/pgtype"
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
|
||||
"git.fabledsword.com/bvandeusen/minstrel/internal/db/dbq"
|
||||
"git.fabledsword.com/bvandeusen/minstrel/internal/indexer"
|
||||
"git.fabledsword.com/bvandeusen/minstrel/internal/release"
|
||||
)
|
||||
|
||||
// Block reasons, matching release_blocks_reason_check.
|
||||
const (
|
||||
BlockOperator = "operator"
|
||||
BlockFiles = "files"
|
||||
)
|
||||
|
||||
// Grab origins, matching grabs_origin_check.
|
||||
const (
|
||||
OriginSchedule = "schedule"
|
||||
OriginOperator = "operator"
|
||||
)
|
||||
|
||||
// Memory is what searches and grabs leave behind: every candidate of a
|
||||
// want's last search, the grabs, and the releases set aside.
|
||||
type Memory struct {
|
||||
pool *pgxpool.Pool
|
||||
}
|
||||
|
||||
// NewMemory returns a Memory over pool.
|
||||
func NewMemory(pool *pgxpool.Pool) *Memory { return &Memory{pool: pool} }
|
||||
|
||||
// Keep replaces a want's candidates with this search's, in order.
|
||||
func (m *Memory) Keep(ctx context.Context, wantID pgtype.UUID, cands []Candidate) error {
|
||||
return pgx.BeginFunc(ctx, m.pool, func(tx pgx.Tx) error {
|
||||
q := dbq.New(tx)
|
||||
if err := q.DeleteWantCandidates(ctx, wantID); err != nil {
|
||||
return err
|
||||
}
|
||||
for i, c := range cands {
|
||||
reasons, err := json.Marshal(c.Reasons)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if c.Reasons == nil {
|
||||
reasons = []byte("[]")
|
||||
}
|
||||
folded := c.FoldedFrom
|
||||
if folded == nil {
|
||||
folded = []string{}
|
||||
}
|
||||
if err := q.InsertWantCandidate(ctx, dbq.InsertWantCandidateParams{
|
||||
WantID: wantID, Key: Key(c), Name: c.Name, InfoHash: c.InfoHash, Indexer: c.Indexer,
|
||||
Source: c.Source.Source, SizeBytes: c.SizeBytes, Seeders: int32(c.Seeders),
|
||||
FilesCount: int32(c.FilesCount), Format: release.Describe(c.Release), Quality: string(c.Tier),
|
||||
Year: int32(c.Year), Verdict: string(c.Verdict), Reason: c.Reason, Reasons: reasons,
|
||||
FoldedFrom: folded, Rank: int32(i),
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
// blocked is every release set aside, by infohash, with the sentence a
|
||||
// candidate shows. A failed read blocks nothing: offering a release set
|
||||
// aside costs one grab a person can undo, while blocking everything on a
|
||||
// database blip would hide every result.
|
||||
func (m *Memory) blocked(ctx context.Context) map[string]string {
|
||||
if m == nil {
|
||||
return nil
|
||||
}
|
||||
rows, err := dbq.New(m.pool).ListReleaseBlocks(ctx)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
out := make(map[string]string, len(rows))
|
||||
for _, r := range rows {
|
||||
out[indexer.NormalizeHash(r.InfoHash)] = BlockSentence(r.Reason, r.Detail)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// BlockSentence is what a blocked candidate says.
|
||||
func BlockSentence(reason, detail string) string {
|
||||
if reason == BlockFiles {
|
||||
if detail != "" {
|
||||
return "Set aside for what its files hold: " + detail
|
||||
}
|
||||
return "Set aside for what its files hold."
|
||||
}
|
||||
return "You set this release aside."
|
||||
}
|
||||
|
||||
// Block sets a release aside and marks every kept candidate of it blocked.
|
||||
func (m *Memory) Block(ctx context.Context, p dbq.BlockReleaseParams) error {
|
||||
p.InfoHash = indexer.NormalizeHash(p.InfoHash)
|
||||
if p.InfoHash == "" {
|
||||
return errors.New("a release without an infohash cannot be set aside")
|
||||
}
|
||||
return pgx.BeginFunc(ctx, m.pool, func(tx pgx.Tx) error {
|
||||
q := dbq.New(tx)
|
||||
if err := q.BlockRelease(ctx, p); err != nil {
|
||||
return err
|
||||
}
|
||||
return q.MarkCandidatesBlocked(ctx, dbq.MarkCandidatesBlockedParams{
|
||||
InfoHash: p.InfoHash, Reason: BlockSentence(p.Reason, p.Detail),
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
// Unblock lets a release be taken again. Its kept candidates wait for a
|
||||
// person until the next search judges them afresh.
|
||||
func (m *Memory) Unblock(ctx context.Context, hash string) (bool, error) {
|
||||
hash = indexer.NormalizeHash(hash)
|
||||
var n int64
|
||||
err := pgx.BeginFunc(ctx, m.pool, func(tx pgx.Tx) error {
|
||||
q := dbq.New(tx)
|
||||
var err error
|
||||
if n, err = q.UnblockRelease(ctx, hash); err != nil {
|
||||
return err
|
||||
}
|
||||
return q.MarkCandidatesUnblocked(ctx, dbq.MarkCandidatesUnblockedParams{
|
||||
InfoHash: hash, Reason: "Allowed again. The next search judges it afresh.",
|
||||
})
|
||||
})
|
||||
return n > 0, err
|
||||
}
|
||||
|
||||
// LiveGrab is the download running for a want, if there is one.
|
||||
func (m *Memory) LiveGrab(ctx context.Context, wantID pgtype.UUID) (dbq.Grab, bool, error) {
|
||||
g, err := dbq.New(m.pool).GetLiveGrab(ctx, wantID)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return g, false, nil
|
||||
}
|
||||
if err != nil {
|
||||
return g, false, err
|
||||
}
|
||||
return g, true, nil
|
||||
}
|
||||
|
||||
// Grabbed records a download started, ending the grab it replaced first so
|
||||
// the one-live-grab index holds.
|
||||
func (m *Memory) Grabbed(ctx context.Context, p dbq.InsertGrabParams, replaced pgtype.UUID, why string) (dbq.Grab, error) {
|
||||
var g dbq.Grab
|
||||
err := pgx.BeginFunc(ctx, m.pool, func(tx pgx.Tx) error {
|
||||
q := dbq.New(tx)
|
||||
if replaced.Valid {
|
||||
if err := q.EndGrab(ctx, dbq.EndGrabParams{ID: replaced, EndReason: "superseded", EndDetail: why}); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
var err error
|
||||
g, err = q.InsertGrab(ctx, p)
|
||||
return err
|
||||
})
|
||||
if err != nil {
|
||||
return g, fmt.Errorf("record grab: %w", err)
|
||||
}
|
||||
return g, nil
|
||||
}
|
||||
@@ -0,0 +1,145 @@
|
||||
package acquire
|
||||
|
||||
import (
|
||||
"context"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"git.fabledsword.com/bvandeusen/minstrel/internal/indexer"
|
||||
)
|
||||
|
||||
// SearchPace is the one Pacer every search shares, ported from Fabled
|
||||
// Librarian (#4448). Process-wide because a Fanout is built fresh for each
|
||||
// search; a Pacer per Fanout would start empty and pace nothing.
|
||||
var SearchPace = &Pacer{}
|
||||
|
||||
// holdWithoutRetryAfter is how long a source that answered 429 without saying
|
||||
// for how long is left alone: one schedule tick.
|
||||
const holdWithoutRetryAfter = 10 * time.Minute
|
||||
|
||||
// Pacer spaces the searches sent to each source by its searches_per_hour: a
|
||||
// spacing, not a burst bucket, because Prowlarr answers 429 to a few requests
|
||||
// in quick succession (FL #4304). Every search spends, the schedule's and a
|
||||
// person's alike; only the schedule waits.
|
||||
type Pacer struct {
|
||||
// Now and Sleep are the clock, replaceable in tests.
|
||||
Now func() time.Time
|
||||
Sleep func(context.Context, time.Duration) error
|
||||
|
||||
mu sync.Mutex
|
||||
next map[string]time.Time
|
||||
}
|
||||
|
||||
func (p *Pacer) now() time.Time {
|
||||
if p.Now != nil {
|
||||
return p.Now()
|
||||
}
|
||||
return time.Now()
|
||||
}
|
||||
|
||||
func (p *Pacer) sleep(ctx context.Context, d time.Duration) error {
|
||||
if p.Sleep != nil {
|
||||
return p.Sleep(ctx, d)
|
||||
}
|
||||
t := time.NewTimer(d)
|
||||
defer t.Stop()
|
||||
select {
|
||||
case <-t.C:
|
||||
return nil
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
}
|
||||
}
|
||||
|
||||
// gap is the spacing a source's number asks for; zero is unpaced.
|
||||
func gap(perHour int) time.Duration {
|
||||
if perHour <= 0 {
|
||||
return 0
|
||||
}
|
||||
return time.Hour / time.Duration(perHour)
|
||||
}
|
||||
|
||||
// readyAt is when every source may next be asked.
|
||||
func (p *Pacer) readyAt(sources []Source) time.Time {
|
||||
p.mu.Lock()
|
||||
defer p.mu.Unlock()
|
||||
var at time.Time
|
||||
for _, s := range sources {
|
||||
if s.Search == nil {
|
||||
continue
|
||||
}
|
||||
if n := p.next[s.Code]; n.After(at) {
|
||||
at = n
|
||||
}
|
||||
}
|
||||
return at
|
||||
}
|
||||
|
||||
// Wait holds until every source may be asked, and answers false, without
|
||||
// waiting, when that is after the deadline or ctx ends first. All of them,
|
||||
// not any: a search while one is held back would be narrowed, and the want
|
||||
// recorded as looked for.
|
||||
func (p *Pacer) Wait(ctx context.Context, sources []Source, deadline time.Time) bool {
|
||||
if p == nil {
|
||||
return true
|
||||
}
|
||||
at := p.readyAt(sources)
|
||||
if !deadline.IsZero() && at.After(deadline) {
|
||||
return false
|
||||
}
|
||||
if d := at.Sub(p.now()); d > 0 {
|
||||
return p.sleep(ctx, d) == nil
|
||||
}
|
||||
return ctx.Err() == nil
|
||||
}
|
||||
|
||||
// spend takes one search's slot from a source.
|
||||
func (p *Pacer) spend(s Source) {
|
||||
g := gap(s.PerHour)
|
||||
if p == nil || g == 0 {
|
||||
return
|
||||
}
|
||||
p.mu.Lock()
|
||||
defer p.mu.Unlock()
|
||||
if p.next == nil {
|
||||
p.next = map[string]time.Time{}
|
||||
}
|
||||
at := p.next[s.Code]
|
||||
if t := p.now(); at.Before(t) {
|
||||
at = t
|
||||
}
|
||||
p.next[s.Code] = at.Add(g)
|
||||
}
|
||||
|
||||
// hold leaves a source alone after it answered 429, for as long as it asked,
|
||||
// whatever its searches_per_hour says.
|
||||
func (p *Pacer) hold(code string, rl *indexer.RateLimited) {
|
||||
if p == nil {
|
||||
return
|
||||
}
|
||||
d := rl.RetryAfter
|
||||
if d <= 0 {
|
||||
d = holdWithoutRetryAfter
|
||||
}
|
||||
p.mu.Lock()
|
||||
defer p.mu.Unlock()
|
||||
if p.next == nil {
|
||||
p.next = map[string]time.Time{}
|
||||
}
|
||||
if until := p.now().Add(d); until.After(p.next[code]) {
|
||||
p.next[code] = until
|
||||
}
|
||||
}
|
||||
|
||||
// Ready holds until every source this service searches may be asked again,
|
||||
// and answers false when that is past the deadline. The schedule calls it
|
||||
// before each want; a person's search never does.
|
||||
func (s *Service) Ready(ctx context.Context, deadline time.Time) bool {
|
||||
if s == nil {
|
||||
return ctx.Err() == nil
|
||||
}
|
||||
if f, ok := s.Search.(*Fanout); ok && f != nil {
|
||||
return f.Pace.Wait(ctx, f.Sources, deadline)
|
||||
}
|
||||
return ctx.Err() == nil
|
||||
}
|
||||
@@ -0,0 +1,390 @@
|
||||
package acquire
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"github.com/jackc/pgx/v5/pgtype"
|
||||
|
||||
"git.fabledsword.com/bvandeusen/minstrel/internal/indexer"
|
||||
"git.fabledsword.com/bvandeusen/minstrel/internal/release"
|
||||
)
|
||||
|
||||
// One search for one want, and what could be done about each result (M502
|
||||
// #5521), ported from Fabled Librarian's internal/acquire. internal/release
|
||||
// says whether a name is the album and what quality it is; this package
|
||||
// asks in the right order and says why in words a person can act on.
|
||||
|
||||
// Verdict is what can be done about one result.
|
||||
type Verdict string
|
||||
|
||||
const (
|
||||
// Grab: the album, in a quality the profile takes.
|
||||
Grab Verdict = "grab"
|
||||
// Review: maybe the album; a person decides.
|
||||
Review Verdict = "review"
|
||||
// Refused: the album, in a quality the profile does not take. The only
|
||||
// verdict that names a setting.
|
||||
Refused Verdict = "refused"
|
||||
// Rejected: not the album.
|
||||
Rejected Verdict = "rejected"
|
||||
// Blocked: a release set aside, by a person or by its files.
|
||||
Blocked Verdict = "blocked"
|
||||
)
|
||||
|
||||
// order sorts verdicts: what can be taken first, what was given up on last.
|
||||
var order = map[Verdict]int{Grab: 0, Review: 1, Refused: 2, Rejected: 3, Blocked: 4}
|
||||
|
||||
// Reason kinds: a hard reason is a fact about the bytes, a soft one about the
|
||||
// moment or a setting.
|
||||
const (
|
||||
Hard = "hard"
|
||||
Soft = "soft"
|
||||
)
|
||||
|
||||
// Reason codes beyond internal/release's vetoes.
|
||||
const (
|
||||
CodeBlocked = "blocked"
|
||||
CodeProfile = "profile"
|
||||
CodeNoSeeders = "no_seeders"
|
||||
CodeDoubt = "doubt"
|
||||
)
|
||||
|
||||
// Reason is one thing a verdict rests on.
|
||||
type Reason struct {
|
||||
Kind string `json:"kind"`
|
||||
Code string `json:"code"`
|
||||
Text string `json:"text"`
|
||||
}
|
||||
|
||||
// Want is the album searched for: its wants row and how the catalogue
|
||||
// describes it.
|
||||
type Want struct {
|
||||
ID pgtype.UUID
|
||||
release.Want
|
||||
}
|
||||
|
||||
// Candidate is one result with everything decided about it.
|
||||
type Candidate struct {
|
||||
release.Release
|
||||
// Source is the indexer's own result. It never leaves the server: its
|
||||
// DownloadURL can carry the indexer's key.
|
||||
Source indexer.Result
|
||||
InfoHash string
|
||||
Indexer string
|
||||
SizeBytes int64
|
||||
Seeders int
|
||||
SwarmKnown bool
|
||||
Category int
|
||||
FilesCount int
|
||||
// Files is the torrent's file list once its .torrent has been read.
|
||||
Files []release.File
|
||||
FoldedFrom []string
|
||||
|
||||
Tier release.Tier
|
||||
YearAgrees bool
|
||||
Verdict Verdict
|
||||
Reason string
|
||||
Reasons []Reason
|
||||
}
|
||||
|
||||
// Key is how a candidate is named across a search and a grab: its infohash,
|
||||
// or its name when the source gave none.
|
||||
func Key(c Candidate) string {
|
||||
if c.InfoHash != "" {
|
||||
return c.InfoHash
|
||||
}
|
||||
return c.Name
|
||||
}
|
||||
|
||||
// Result is one search.
|
||||
type Result struct {
|
||||
Query string
|
||||
// Unconfigured: no search source is set up.
|
||||
Unconfigured bool
|
||||
// Error is a search that failed outright.
|
||||
Error string
|
||||
// Narrowed names sources that did not answer on a search that otherwise
|
||||
// did.
|
||||
Narrowed []Narrowing
|
||||
Considered int
|
||||
Folded int
|
||||
// Looked is how many .torrent files were read for their file lists.
|
||||
Looked int
|
||||
Candidates []Candidate
|
||||
// KeepError: the candidates could not be stored.
|
||||
KeepError string
|
||||
}
|
||||
|
||||
// Grabbable is what may be taken, best first.
|
||||
func (r Result) Grabbable() []Candidate {
|
||||
var out []Candidate
|
||||
for _, c := range r.Candidates {
|
||||
if c.Verdict == Grab {
|
||||
out = append(out, c)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// ForAPerson says whether anything found is worth a person's look: a
|
||||
// release that may be the album, whatever its quality.
|
||||
func (r Result) ForAPerson() bool {
|
||||
for _, c := range r.Candidates {
|
||||
if c.Verdict == Grab || c.Verdict == Review || c.Verdict == Refused {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// Service searches and grabs.
|
||||
type Service struct {
|
||||
// Search is nil when no source is configured.
|
||||
Search Searcher
|
||||
// Grabber is nil when no download client is configured.
|
||||
Grabber *Downloader
|
||||
Profile release.Profile
|
||||
// Memory stores candidates, grabs and blocks. Nil keeps nothing.
|
||||
Memory *Memory
|
||||
}
|
||||
|
||||
// Query is the text sent to the indexers: the artist and the title, without
|
||||
// a trailing bracketed part the release names usually leave off.
|
||||
func Query(w release.Want) string {
|
||||
title := w.Title
|
||||
if i := strings.IndexAny(title, "(["); i > 0 {
|
||||
title = title[:i]
|
||||
}
|
||||
return strings.Join(strings.Fields(strings.TrimSpace(w.Artist)+" "+strings.TrimSpace(title)), " ")
|
||||
}
|
||||
|
||||
// For searches for one want and judges everything that comes back. It never
|
||||
// returns an error: a search that could not be made is a Result saying so.
|
||||
func (s *Service) For(ctx context.Context, want Want) Result {
|
||||
r := Result{Query: Query(want.Want)}
|
||||
if s == nil || s.Search == nil {
|
||||
r.Unconfigured = true
|
||||
return r
|
||||
}
|
||||
found, err := s.Search.Search(ctx, indexer.Query{Text: r.Query, Categories: indexer.MusicCategories})
|
||||
// A partial search arrives as results and an error together.
|
||||
if narrowed, answered := narrowings(err); narrowed != nil && answered {
|
||||
r.Narrowed, err = narrowed, nil
|
||||
}
|
||||
switch {
|
||||
case errors.Is(err, indexer.ErrUnconfigured):
|
||||
r.Unconfigured = true
|
||||
return r
|
||||
case err != nil:
|
||||
r.Error = indexer.FirstLine(err.Error())
|
||||
return r
|
||||
}
|
||||
r.Considered = len(found)
|
||||
cands, folded := fold(found)
|
||||
r.Folded = folded
|
||||
|
||||
blocked := s.Memory.blocked(ctx)
|
||||
for i := range cands {
|
||||
decide(&cands[i], want.Want, s.Profile, blocked)
|
||||
}
|
||||
sortCandidates(cands)
|
||||
for _, i := range s.peek(ctx, cands) {
|
||||
decide(&cands[i], want.Want, s.Profile, blocked)
|
||||
r.Looked++
|
||||
}
|
||||
sortCandidates(cands)
|
||||
r.Candidates = cands
|
||||
if s.Memory != nil && want.ID.Valid {
|
||||
if err := s.Memory.Keep(ctx, want.ID, cands); err != nil {
|
||||
r.KeepError = indexer.FirstLine(err.Error())
|
||||
}
|
||||
}
|
||||
return r
|
||||
}
|
||||
|
||||
// decide works out one candidate's verdict, in order: given up on, then
|
||||
// whether it is the album, then whether the profile takes its quality, then
|
||||
// whether anything leaves it to a person. A quality refusal said about the
|
||||
// wrong album would be noise, so identity comes first.
|
||||
func decide(c *Candidate, w release.Want, profile release.Profile, blocked map[string]string) {
|
||||
c.Verdict, c.Reason, c.Reasons = "", "", nil
|
||||
if why, ok := blocked[c.InfoHash]; ok && c.InfoHash != "" {
|
||||
set(c, Blocked, Hard, CodeBlocked, why)
|
||||
return
|
||||
}
|
||||
m := release.Judge(w, c.Release)
|
||||
c.YearAgrees = m.YearAgrees
|
||||
if m.Veto != "" {
|
||||
set(c, Rejected, Hard, m.Veto, m.Reason)
|
||||
return
|
||||
}
|
||||
doubt := m.Doubt
|
||||
if len(c.Files) > 0 {
|
||||
if in := release.Inspect(c.Files); in.Format != "" {
|
||||
c.Format = in.Format
|
||||
}
|
||||
veto, reason, fileDoubt := release.JudgeFiles(w, c.Files)
|
||||
if veto != "" {
|
||||
set(c, Rejected, Hard, veto, reason)
|
||||
return
|
||||
}
|
||||
if doubt == "" {
|
||||
doubt = fileDoubt
|
||||
}
|
||||
}
|
||||
c.Tier = release.TierOf(c.Release, c.Category)
|
||||
if !profile.Allows(c.Tier) {
|
||||
set(c, Refused, Soft, CodeProfile, "Your quality settings don’t take "+strings.ToLower(release.Label(c.Tier))+".")
|
||||
return
|
||||
}
|
||||
if doubt != "" {
|
||||
set(c, Review, Soft, CodeDoubt, doubt)
|
||||
return
|
||||
}
|
||||
if c.Seeders == 0 {
|
||||
set(c, Review, Soft, CodeNoSeeders, "Nobody is sharing it right now.")
|
||||
return
|
||||
}
|
||||
c.Verdict = Grab
|
||||
c.Reason = "The album, " + strings.ToLower(release.Label(c.Tier)) + "."
|
||||
if d := release.Describe(c.Release); d != "" {
|
||||
c.Reason = "The album, " + d + "."
|
||||
}
|
||||
}
|
||||
|
||||
func set(c *Candidate, v Verdict, kind, code, text string) {
|
||||
c.Verdict, c.Reason = v, text
|
||||
c.Reasons = append(c.Reasons, Reason{Kind: kind, Code: code, Text: text})
|
||||
}
|
||||
|
||||
// sortCandidates puts the best first, stably so the same search proposes the
|
||||
// same grab twice.
|
||||
func sortCandidates(cands []Candidate) {
|
||||
sort.SliceStable(cands, func(a, b int) bool { return better(cands[a], cands[b]) })
|
||||
}
|
||||
|
||||
// better says whether x ranks above y: verdict, then quality, then how many
|
||||
// share it, then whether its year agrees.
|
||||
func better(x, y Candidate) bool {
|
||||
if order[x.Verdict] != order[y.Verdict] {
|
||||
return order[x.Verdict] < order[y.Verdict]
|
||||
}
|
||||
if rx, ry := release.Rank(x.Tier), release.Rank(y.Tier); rx != ry {
|
||||
return rx > ry
|
||||
}
|
||||
if x.Seeders != y.Seeders {
|
||||
return x.Seeders > y.Seeders
|
||||
}
|
||||
return x.YearAgrees && !y.YearAgrees
|
||||
}
|
||||
|
||||
// fold merges results that are the same torrent, by infohash. Prowlarr
|
||||
// aggregates, so one release arrives once per tracker that lists it. A
|
||||
// result without a hash is never folded: an absent hash is not a shared one.
|
||||
func fold(found []indexer.Result) ([]Candidate, int) {
|
||||
out := make([]Candidate, 0, len(found))
|
||||
at := make(map[string]int, len(found))
|
||||
folded := 0
|
||||
for _, f := range found {
|
||||
if strings.TrimSpace(f.Title) == "" {
|
||||
continue
|
||||
}
|
||||
hash := indexer.NormalizeHash(f.InfoHash)
|
||||
if hash == "" && f.Magnet != "" {
|
||||
hash = indexer.MagnetHash(f.Magnet)
|
||||
}
|
||||
if hash != "" {
|
||||
if i, seen := at[hash]; seen {
|
||||
folded++
|
||||
if f.Indexer != "" {
|
||||
out[i].FoldedFrom = append(out[i].FoldedFrom, f.Indexer)
|
||||
}
|
||||
// A measured swarm beats a reported one; among like, the
|
||||
// highest count.
|
||||
switch {
|
||||
case f.SwarmKnown && !out[i].SwarmKnown:
|
||||
out[i].Seeders, out[i].SwarmKnown = f.Seeders, true
|
||||
case f.SwarmKnown == out[i].SwarmKnown && f.Seeders > out[i].Seeders:
|
||||
out[i].Seeders = f.Seeders
|
||||
}
|
||||
if f.FilesCount > 0 && out[i].FilesCount == 0 {
|
||||
out[i].FilesCount = f.FilesCount
|
||||
}
|
||||
continue
|
||||
}
|
||||
at[hash] = len(out)
|
||||
}
|
||||
out = append(out, Candidate{
|
||||
Release: release.Parse(f.Title), Source: f, InfoHash: hash, Indexer: f.Indexer,
|
||||
SizeBytes: f.SizeBytes, Seeders: f.Seeders, SwarmKnown: f.SwarmKnown,
|
||||
Category: f.Category, FilesCount: f.FilesCount,
|
||||
})
|
||||
}
|
||||
return out, folded
|
||||
}
|
||||
|
||||
// maxPeeks is how many .torrent files one search reads. Few: each goes
|
||||
// through Prowlarr to a tracker, and Prowlarr answers 429 to a burst.
|
||||
const maxPeeks = 3
|
||||
|
||||
// peek reads the .torrent of candidates that are the album but whose name
|
||||
// states no format, so their files can say what they are; it returns which
|
||||
// now carry a file list. A failed fetch leaves the candidate as it was.
|
||||
func (s *Service) peek(ctx context.Context, cands []Candidate) []int {
|
||||
f, ok := s.Search.(Fetcher)
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
var looked []int
|
||||
for i := range cands {
|
||||
if len(looked) >= maxPeeks || ctx.Err() != nil {
|
||||
break
|
||||
}
|
||||
c := &cands[i]
|
||||
if c.Tier != release.TierUnknown || (c.Verdict != Grab && c.Verdict != Refused) ||
|
||||
c.Files != nil || c.Source.DownloadURL == "" || c.Source.Magnet != "" {
|
||||
continue
|
||||
}
|
||||
body, err := f.Download(ctx, c.Source)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
meta, err := release.ParseTorrent(body)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
c.Files = meta.Files
|
||||
if c.InfoHash == "" {
|
||||
c.InfoHash = meta.InfoHash
|
||||
}
|
||||
if c.FilesCount == 0 {
|
||||
c.FilesCount = len(meta.Files)
|
||||
}
|
||||
looked = append(looked, i)
|
||||
}
|
||||
return looked
|
||||
}
|
||||
|
||||
// pick finds a candidate by key: the infohash first, the exact name for a
|
||||
// result that has none.
|
||||
func pick(cands []Candidate, key string) *Candidate {
|
||||
key = strings.TrimSpace(key)
|
||||
if key == "" {
|
||||
return nil
|
||||
}
|
||||
hash := indexer.NormalizeHash(key)
|
||||
for i := range cands {
|
||||
if cands[i].InfoHash != "" && cands[i].InfoHash == hash {
|
||||
return &cands[i]
|
||||
}
|
||||
}
|
||||
for i := range cands {
|
||||
if cands[i].InfoHash == "" && cands[i].Name == key {
|
||||
return &cands[i]
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,241 @@
|
||||
package acquire
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.fabledsword.com/bvandeusen/minstrel/internal/downloader"
|
||||
"git.fabledsword.com/bvandeusen/minstrel/internal/indexer"
|
||||
"git.fabledsword.com/bvandeusen/minstrel/internal/release"
|
||||
)
|
||||
|
||||
type fakeSearch struct {
|
||||
results []indexer.Result
|
||||
err error
|
||||
asked []indexer.Query
|
||||
}
|
||||
|
||||
func (f *fakeSearch) Search(_ context.Context, q indexer.Query) ([]indexer.Result, error) {
|
||||
f.asked = append(f.asked, q)
|
||||
return f.results, f.err
|
||||
}
|
||||
|
||||
var ram = Want{Want: release.Want{
|
||||
Artist: "Daft Punk", Title: "Random Access Memories", Year: 2013, PrimaryType: "Album",
|
||||
MinTracks: 13, MaxTracks: 15,
|
||||
}}
|
||||
|
||||
var anyQuality = release.NewProfile([]string{
|
||||
"lossless_hires", "lossless", "lossy_high", "lossy_standard", "lossy_low", "lossy_poor", "lossy_trash",
|
||||
})
|
||||
|
||||
func result(title, hash string, seeders int) indexer.Result {
|
||||
return indexer.Result{Title: title, InfoHash: hash, Seeders: seeders, Indexer: "Knaben", Source: "prowlarr",
|
||||
Magnet: "magnet:?xt=urn:btih:" + hash}
|
||||
}
|
||||
|
||||
func TestFor_JudgesSortsAndFolds(t *testing.T) {
|
||||
search := &fakeSearch{results: []indexer.Result{
|
||||
result("Daft Punk - Random Access Memories (2013) [MP3 320]", "aaaa", 40),
|
||||
result("Daft Punk Random Access Memories (Drumless Edition 2023) [Flac 24-88]", "bbbb", 90),
|
||||
result("Daft Punk - Random Access Memories (2013) [FLAC]", "cccc", 12),
|
||||
result("Daft Punk - Random Access Memories (2013) [FLAC]", "CCCC", 30), // the same torrent from another tracker
|
||||
result("Daft Punk - Discovery (2001) [FLAC]", "dddd", 50),
|
||||
}}
|
||||
s := &Service{Search: search, Profile: anyQuality}
|
||||
r := s.For(context.Background(), ram)
|
||||
|
||||
if len(search.asked) != 1 || search.asked[0].Text != "Daft Punk Random Access Memories" {
|
||||
t.Fatalf("asked %+v", search.asked)
|
||||
}
|
||||
if r.Considered != 5 || r.Folded != 1 || len(r.Candidates) != 4 {
|
||||
t.Fatalf("considered %d folded %d candidates %d", r.Considered, r.Folded, len(r.Candidates))
|
||||
}
|
||||
// Lossless before high-quality lossy among what can be taken; the
|
||||
// folded copy keeps the higher seeder count.
|
||||
first, second := r.Candidates[0], r.Candidates[1]
|
||||
if first.InfoHash != "cccc" || first.Verdict != Grab || first.Seeders != 30 || first.Tier != release.TierLossless {
|
||||
t.Errorf("first = %s %s %d %s", first.Name, first.Verdict, first.Seeders, first.Tier)
|
||||
}
|
||||
if second.InfoHash != "aaaa" || second.Verdict != Grab {
|
||||
t.Errorf("second = %s %s", second.Name, second.Verdict)
|
||||
}
|
||||
for _, c := range r.Candidates[2:] {
|
||||
if c.Verdict != Rejected || len(c.Reasons) != 1 || c.Reasons[0].Kind != Hard {
|
||||
t.Errorf("%s: %s %+v", c.Name, c.Verdict, c.Reasons)
|
||||
}
|
||||
}
|
||||
if len(r.Grabbable()) != 2 || !r.ForAPerson() {
|
||||
t.Errorf("grabbable %d", len(r.Grabbable()))
|
||||
}
|
||||
}
|
||||
|
||||
func TestFor_ProfileSeedersAndDoubt(t *testing.T) {
|
||||
search := &fakeSearch{results: []indexer.Result{
|
||||
result("Daft Punk - Random Access Memories [MP3 128]", "aaaa", 5),
|
||||
result("Daft Punk - Random Access Memories [FLAC]", "bbbb", 0),
|
||||
result("Daft Punk - Random Access Memories", "cccc", 9),
|
||||
}}
|
||||
s := &Service{Search: search, Profile: release.NewProfile([]string{"lossless", "lossy_high"})}
|
||||
r := s.For(context.Background(), ram)
|
||||
verdicts := map[string]Verdict{}
|
||||
for _, c := range r.Candidates {
|
||||
verdicts[c.InfoHash] = c.Verdict
|
||||
}
|
||||
if verdicts["aaaa"] != Refused || verdicts["bbbb"] != Review || verdicts["cccc"] != Refused {
|
||||
t.Errorf("verdicts = %v", verdicts)
|
||||
}
|
||||
if len(r.Grabbable()) != 0 || !r.ForAPerson() {
|
||||
t.Error("nothing grabbable, but something for a person")
|
||||
}
|
||||
}
|
||||
|
||||
func TestFor_BlockedReleasesAreSetAside(t *testing.T) {
|
||||
c := Candidate{Release: release.Parse("Daft Punk - Random Access Memories [FLAC]"), InfoHash: "aaaa", Seeders: 3}
|
||||
decide(&c, ram.Want, anyQuality, map[string]string{"aaaa": "You set this release aside."})
|
||||
if c.Verdict != Blocked || c.Reason != "You set this release aside." {
|
||||
t.Errorf("verdict %s %q", c.Verdict, c.Reason)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFor_UnconfiguredAndFailedSearches(t *testing.T) {
|
||||
if r := (&Service{}).For(context.Background(), ram); !r.Unconfigured {
|
||||
t.Error("no source is unconfigured")
|
||||
}
|
||||
r := (&Service{Search: &fakeSearch{err: indexer.ErrUnconfigured}}).For(context.Background(), ram)
|
||||
if !r.Unconfigured {
|
||||
t.Error("ErrUnconfigured is unconfigured")
|
||||
}
|
||||
r = (&Service{Search: &fakeSearch{err: errors.New("dial tcp: refused")}}).For(context.Background(), ram)
|
||||
if r.Error == "" {
|
||||
t.Error("a failed search says so")
|
||||
}
|
||||
}
|
||||
|
||||
func TestFanout_ANarrowedSearchKeepsWhatAnswered(t *testing.T) {
|
||||
up := &fakeSearch{results: []indexer.Result{result("Daft Punk - Random Access Memories [FLAC]", "aaaa", 3)}}
|
||||
down := &fakeSearch{err: indexer.ErrUnreachable}
|
||||
f := &Fanout{Sources: []Source{{Code: "prowlarr", Label: "Prowlarr", Search: up}, {Code: "torznab", Label: "Feed", Search: down}}}
|
||||
r := (&Service{Search: f, Profile: anyQuality}).For(context.Background(), ram)
|
||||
if r.Error != "" || len(r.Candidates) != 1 || len(r.Narrowed) != 1 || r.Narrowed[0].Source != "torznab" {
|
||||
t.Errorf("narrowed search: error %q, %d candidates, narrowed %+v", r.Error, len(r.Candidates), r.Narrowed)
|
||||
}
|
||||
f.Sources = f.Sources[1:]
|
||||
if r := (&Service{Search: f}).For(context.Background(), ram); r.Error == "" {
|
||||
t.Error("every source down is a failed search")
|
||||
}
|
||||
}
|
||||
|
||||
func TestPacer_SpacesSearchesAndHoldsAfterA429(t *testing.T) {
|
||||
now := time.Unix(0, 0)
|
||||
var slept time.Duration
|
||||
p := &Pacer{Now: func() time.Time { return now }, Sleep: func(_ context.Context, d time.Duration) error { slept += d; return nil }}
|
||||
src := []Source{{Code: "prowlarr", PerHour: 120, Search: &fakeSearch{}}}
|
||||
p.spend(src[0])
|
||||
if !p.Wait(context.Background(), src, now.Add(time.Hour)) || slept != 30*time.Second {
|
||||
t.Errorf("slept %s, want 30s", slept)
|
||||
}
|
||||
if p.Wait(context.Background(), src, now.Add(time.Second)) {
|
||||
t.Error("waited past the deadline")
|
||||
}
|
||||
p.hold("prowlarr", &indexer.RateLimited{RetryAfter: 5 * time.Minute})
|
||||
if at := p.readyAt(src); at.Sub(now) != 5*time.Minute {
|
||||
t.Errorf("held until %s", at.Sub(now))
|
||||
}
|
||||
}
|
||||
|
||||
type fakeClient struct {
|
||||
downloader.Driver
|
||||
added []downloader.Add
|
||||
addErr error
|
||||
deleted []string
|
||||
progress float64
|
||||
}
|
||||
|
||||
func (c *fakeClient) AddTorrent(_ context.Context, a downloader.Add) error {
|
||||
c.added = append(c.added, a)
|
||||
return c.addErr
|
||||
}
|
||||
|
||||
func (c *fakeClient) Torrents(_ context.Context, hashes ...string) ([]downloader.Torrent, error) {
|
||||
return []downloader.Torrent{{Hash: hashes[0], Progress: c.progress}}, nil
|
||||
}
|
||||
|
||||
func (c *fakeClient) Delete(_ context.Context, _ bool, hashes ...string) error {
|
||||
c.deleted = append(c.deleted, hashes...)
|
||||
return nil
|
||||
}
|
||||
|
||||
func TestGrab_AddsAMagnetToItsOwnFolder(t *testing.T) {
|
||||
client := &fakeClient{}
|
||||
s := &Service{Search: &fakeSearch{results: []indexer.Result{
|
||||
result("Daft Punk - Random Access Memories (2013) [FLAC]", "ABCD", 8),
|
||||
}}, Profile: anyQuality, Grabber: &Downloader{Client: client, Code: "qbittorrent", Landing: "/downloads"}}
|
||||
r := s.For(context.Background(), ram)
|
||||
took := s.Grab(context.Background(), ram, &r.Candidates[0], r.Candidates, false, Origin{By: OriginSchedule})
|
||||
if !took.Added || took.SavePath != "/downloads/abcd" {
|
||||
t.Fatalf("took %+v", took)
|
||||
}
|
||||
if len(client.added) != 1 || !strings.Contains(strings.ToLower(client.added[0].Magnet), "abcd") || client.added[0].Tags[0] != Tag {
|
||||
t.Errorf("added %+v", client.added)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGrab_RefusesWhatTheVerdictWouldNotTake(t *testing.T) {
|
||||
client := &fakeClient{}
|
||||
s := &Service{Grabber: &Downloader{Client: client, Landing: "/downloads"}}
|
||||
review := Candidate{Verdict: Review, InfoHash: "aaaa", Source: indexer.Result{Magnet: "magnet:?xt=urn:btih:aaaa"}}
|
||||
if took := s.Grab(context.Background(), ram, &review, nil, false, Origin{By: OriginSchedule}); !took.Refused {
|
||||
t.Error("a review was taken unforced")
|
||||
}
|
||||
if took := s.Grab(context.Background(), ram, &review, nil, true, Origin{By: OriginOperator}); !took.Added {
|
||||
t.Errorf("a forced review was not taken: %+v", took)
|
||||
}
|
||||
blocked := Candidate{Verdict: Blocked, InfoHash: "bbbb"}
|
||||
if took := s.Grab(context.Background(), ram, &blocked, nil, true, Origin{By: OriginOperator}); !took.Refused {
|
||||
t.Error("forcing took a release set aside")
|
||||
}
|
||||
if took := (&Service{}).Grab(context.Background(), ram, &review, nil, true, Origin{}); !took.Unconfigured || took.Missing != MissingClient {
|
||||
t.Errorf("no client: %+v", took)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGrab_AClientThatAlreadyHasItCounts(t *testing.T) {
|
||||
client := &fakeClient{addErr: downloader.ErrDuplicate}
|
||||
s := &Service{Grabber: &Downloader{Client: client, Landing: "/downloads"}}
|
||||
c := Candidate{Verdict: Grab, InfoHash: "aaaa", Source: indexer.Result{Magnet: "magnet:?xt=urn:btih:aaaa"}}
|
||||
if took := s.Grab(context.Background(), ram, &c, nil, false, Origin{By: OriginSchedule}); !took.Added {
|
||||
t.Errorf("a duplicate: %+v", took)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBetter_QualityThenSeedersThenYear(t *testing.T) {
|
||||
flac := Candidate{Verdict: Grab, Tier: release.TierLossless, Seeders: 2}
|
||||
mp3 := Candidate{Verdict: Grab, Tier: release.TierLossyHigh, Seeders: 90}
|
||||
if !better(flac, mp3) || better(mp3, flac) {
|
||||
t.Error("quality first among grabs")
|
||||
}
|
||||
busy := Candidate{Verdict: Grab, Tier: release.TierLossless, Seeders: 20}
|
||||
if !better(busy, flac) {
|
||||
t.Error("then seeders")
|
||||
}
|
||||
dated := Candidate{Verdict: Grab, Tier: release.TierLossless, Seeders: 2, YearAgrees: true}
|
||||
if !better(dated, flac) {
|
||||
t.Error("then the year")
|
||||
}
|
||||
if better(Candidate{Verdict: Review}, flac) || !better(flac, Candidate{Verdict: Review}) {
|
||||
t.Error("a grab ranks above a review")
|
||||
}
|
||||
}
|
||||
|
||||
func TestLandingFor_OneFolderPerTorrent(t *testing.T) {
|
||||
if got := landingFor("/downloads", "abcd", "x"); got != "/downloads/abcd" {
|
||||
t.Error(got)
|
||||
}
|
||||
if got := landingFor("/downloads", "", "Daft Punk: R.A.M. [FLAC]"); got != "/downloads/daft-punk-r-a-m--flac" {
|
||||
t.Error(got)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user