feat: search and grab — music release names, a quality ladder, a paced search schedule and one live grab per want (M502 #5521)
release / govulncheck (push) Successful in 21s
release / go (push) Failing after 33s
release / web (push) Successful in 56s
release / integration (push) Failing after 4m48s
release / android (push) Successful in 5m6s
release / Build signed APK (releases and dev) (push) Successful in 5m16s
release / Attach APK to the Release (tag releases only) (push) Skipped
release / Build + push container image (push) Skipped
release / Verify release artifacts (tag releases only) (push) Skipped
release / govulncheck (push) Successful in 21s
release / go (push) Failing after 33s
release / web (push) Successful in 56s
release / integration (push) Failing after 4m48s
release / android (push) Successful in 5m6s
release / Build signed APK (releases and dev) (push) Successful in 5m16s
release / Attach APK to the Release (tag releases only) (push) Skipped
release / Build + push container image (push) Skipped
release / Verify release artifacts (tag releases only) (push) Skipped
- internal/release: reads a release name (format, bitrate, bit depth and sample rate, source, year, video, discography) and judges it against the album wanted: the artist and title must be in it; video rips, discographies, other versions (drumless, instrumental, karaoke, remix, live unless the album is live…) and singles or EPs for an album are refused. Quality tiers on Lidarr's ladder; a .torrent's file list is read for format, video files and a track count short of the album. - internal/acquire: fan-out across every search source with a narrowed search kept, a process-wide pacer per source, every candidate judged and kept with its reasons, and the grab: one live grab per want, tie-broken (a person's grab beats the schedule's; a better release replaces a worse one unless it is nearly done). Each download lands in its own folder under the client's downloads folder. - internal/wanted: the search schedule, every ten minutes, never-searched first, then requests, missing files and followed artists, taking the best release when auto-grab is on (the default). - Migration 0082: acquisition_settings (auto-grab, interval, allowed qualities seeded from Lidarr's "Any"), want search columns, want_candidates, grabs, release_blocks. - Web: the Wanted page shows each want's last search, a Search button and every release found with its verdict (Grab, Grab anyway, Never this one, Allow again). Integrations gains "Searching and grabbing". Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,145 @@
|
||||
package acquire
|
||||
|
||||
import (
|
||||
"context"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"git.fabledsword.com/bvandeusen/minstrel/internal/indexer"
|
||||
)
|
||||
|
||||
// SearchPace is the one Pacer every search shares, ported from Fabled
|
||||
// Librarian (#4448). Process-wide because a Fanout is built fresh for each
|
||||
// search; a Pacer per Fanout would start empty and pace nothing.
|
||||
var SearchPace = &Pacer{}
|
||||
|
||||
// holdWithoutRetryAfter is how long a source that answered 429 without saying
|
||||
// for how long is left alone: one schedule tick.
|
||||
const holdWithoutRetryAfter = 10 * time.Minute
|
||||
|
||||
// Pacer spaces the searches sent to each source by its searches_per_hour: a
|
||||
// spacing, not a burst bucket, because Prowlarr answers 429 to a few requests
|
||||
// in quick succession (FL #4304). Every search spends, the schedule's and a
|
||||
// person's alike; only the schedule waits.
|
||||
type Pacer struct {
|
||||
// Now and Sleep are the clock, replaceable in tests.
|
||||
Now func() time.Time
|
||||
Sleep func(context.Context, time.Duration) error
|
||||
|
||||
mu sync.Mutex
|
||||
next map[string]time.Time
|
||||
}
|
||||
|
||||
func (p *Pacer) now() time.Time {
|
||||
if p.Now != nil {
|
||||
return p.Now()
|
||||
}
|
||||
return time.Now()
|
||||
}
|
||||
|
||||
func (p *Pacer) sleep(ctx context.Context, d time.Duration) error {
|
||||
if p.Sleep != nil {
|
||||
return p.Sleep(ctx, d)
|
||||
}
|
||||
t := time.NewTimer(d)
|
||||
defer t.Stop()
|
||||
select {
|
||||
case <-t.C:
|
||||
return nil
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
}
|
||||
}
|
||||
|
||||
// gap is the spacing a source's number asks for; zero is unpaced.
|
||||
func gap(perHour int) time.Duration {
|
||||
if perHour <= 0 {
|
||||
return 0
|
||||
}
|
||||
return time.Hour / time.Duration(perHour)
|
||||
}
|
||||
|
||||
// readyAt is when every source may next be asked.
|
||||
func (p *Pacer) readyAt(sources []Source) time.Time {
|
||||
p.mu.Lock()
|
||||
defer p.mu.Unlock()
|
||||
var at time.Time
|
||||
for _, s := range sources {
|
||||
if s.Search == nil {
|
||||
continue
|
||||
}
|
||||
if n := p.next[s.Code]; n.After(at) {
|
||||
at = n
|
||||
}
|
||||
}
|
||||
return at
|
||||
}
|
||||
|
||||
// Wait holds until every source may be asked, and answers false, without
|
||||
// waiting, when that is after the deadline or ctx ends first. All of them,
|
||||
// not any: a search while one is held back would be narrowed, and the want
|
||||
// recorded as looked for.
|
||||
func (p *Pacer) Wait(ctx context.Context, sources []Source, deadline time.Time) bool {
|
||||
if p == nil {
|
||||
return true
|
||||
}
|
||||
at := p.readyAt(sources)
|
||||
if !deadline.IsZero() && at.After(deadline) {
|
||||
return false
|
||||
}
|
||||
if d := at.Sub(p.now()); d > 0 {
|
||||
return p.sleep(ctx, d) == nil
|
||||
}
|
||||
return ctx.Err() == nil
|
||||
}
|
||||
|
||||
// spend takes one search's slot from a source.
|
||||
func (p *Pacer) spend(s Source) {
|
||||
g := gap(s.PerHour)
|
||||
if p == nil || g == 0 {
|
||||
return
|
||||
}
|
||||
p.mu.Lock()
|
||||
defer p.mu.Unlock()
|
||||
if p.next == nil {
|
||||
p.next = map[string]time.Time{}
|
||||
}
|
||||
at := p.next[s.Code]
|
||||
if t := p.now(); at.Before(t) {
|
||||
at = t
|
||||
}
|
||||
p.next[s.Code] = at.Add(g)
|
||||
}
|
||||
|
||||
// hold leaves a source alone after it answered 429, for as long as it asked,
|
||||
// whatever its searches_per_hour says.
|
||||
func (p *Pacer) hold(code string, rl *indexer.RateLimited) {
|
||||
if p == nil {
|
||||
return
|
||||
}
|
||||
d := rl.RetryAfter
|
||||
if d <= 0 {
|
||||
d = holdWithoutRetryAfter
|
||||
}
|
||||
p.mu.Lock()
|
||||
defer p.mu.Unlock()
|
||||
if p.next == nil {
|
||||
p.next = map[string]time.Time{}
|
||||
}
|
||||
if until := p.now().Add(d); until.After(p.next[code]) {
|
||||
p.next[code] = until
|
||||
}
|
||||
}
|
||||
|
||||
// Ready holds until every source this service searches may be asked again,
|
||||
// and answers false when that is past the deadline. The schedule calls it
|
||||
// before each want; a person's search never does.
|
||||
func (s *Service) Ready(ctx context.Context, deadline time.Time) bool {
|
||||
if s == nil {
|
||||
return ctx.Err() == nil
|
||||
}
|
||||
if f, ok := s.Search.(*Fanout); ok && f != nil {
|
||||
return f.Pace.Wait(ctx, f.Sources, deadline)
|
||||
}
|
||||
return ctx.Err() == nil
|
||||
}
|
||||
Reference in New Issue
Block a user