feat: search and grab — music release names, a quality ladder, a paced search schedule and one live grab per want (M502 #5521)
release / govulncheck (push) Successful in 21s
release / go (push) Failing after 33s
release / web (push) Successful in 56s
release / integration (push) Failing after 4m48s
release / android (push) Successful in 5m6s
release / Build signed APK (releases and dev) (push) Successful in 5m16s
release / Attach APK to the Release (tag releases only) (push) Skipped
release / Build + push container image (push) Skipped
release / Verify release artifacts (tag releases only) (push) Skipped
release / govulncheck (push) Successful in 21s
release / go (push) Failing after 33s
release / web (push) Successful in 56s
release / integration (push) Failing after 4m48s
release / android (push) Successful in 5m6s
release / Build signed APK (releases and dev) (push) Successful in 5m16s
release / Attach APK to the Release (tag releases only) (push) Skipped
release / Build + push container image (push) Skipped
release / Verify release artifacts (tag releases only) (push) Skipped
- internal/release: reads a release name (format, bitrate, bit depth and sample rate, source, year, video, discography) and judges it against the album wanted: the artist and title must be in it; video rips, discographies, other versions (drumless, instrumental, karaoke, remix, live unless the album is live…) and singles or EPs for an album are refused. Quality tiers on Lidarr's ladder; a .torrent's file list is read for format, video files and a track count short of the album. - internal/acquire: fan-out across every search source with a narrowed search kept, a process-wide pacer per source, every candidate judged and kept with its reasons, and the grab: one live grab per want, tie-broken (a person's grab beats the schedule's; a better release replaces a worse one unless it is nearly done). Each download lands in its own folder under the client's downloads folder. - internal/wanted: the search schedule, every ten minutes, never-searched first, then requests, missing files and followed artists, taking the best release when auto-grab is on (the default). - Migration 0082: acquisition_settings (auto-grab, interval, allowed qualities seeded from Lidarr's "Any"), want search columns, want_candidates, grabs, release_blocks. - Web: the Wanted page shows each want's last search, a Search button and every release found with its verdict (Grab, Grab anyway, Never this one, Allow again). Integrations gains "Searching and grabbing". Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,307 @@
|
||||
package acquire
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"path"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
"github.com/jackc/pgx/v5/pgtype"
|
||||
|
||||
"git.fabledsword.com/bvandeusen/minstrel/internal/db/dbq"
|
||||
"git.fabledsword.com/bvandeusen/minstrel/internal/downloader"
|
||||
"git.fabledsword.com/bvandeusen/minstrel/internal/indexer"
|
||||
"git.fabledsword.com/bvandeusen/minstrel/internal/release"
|
||||
)
|
||||
|
||||
// Taking a release (M502 #5521), ported from Fabled Librarian's grab.go and
|
||||
// rivals.go. One live grab per want: a second is tie-broken against the
|
||||
// download already running (FL #4470).
|
||||
|
||||
// Tag goes on every torrent Minstrel adds, so the operator can tell them
|
||||
// from their own in the client.
|
||||
const Tag = "minstrel"
|
||||
|
||||
// MissingClient is Taken.Missing when no download client is set up.
|
||||
const MissingClient = "download_client"
|
||||
|
||||
// nearlyDone is how far along a running download stays whatever would
|
||||
// replace it.
|
||||
const nearlyDone = 0.9
|
||||
|
||||
// Downloader is the grab half of a Service.
|
||||
type Downloader struct {
|
||||
Client downloader.Driver
|
||||
// Code is the client's acquisition_services row.
|
||||
Code string
|
||||
// Landing is the client's downloads folder as Minstrel sees it; each
|
||||
// grab lands in a folder of its own under it.
|
||||
Landing string
|
||||
// Fetch gets a result's .torrent from the source that offered it.
|
||||
Fetch Fetcher
|
||||
}
|
||||
|
||||
// Origin says who took a release.
|
||||
type Origin struct {
|
||||
By string // OriginSchedule or OriginOperator
|
||||
User pgtype.UUID
|
||||
}
|
||||
|
||||
// Contest is how a grab fared against the download already running.
|
||||
type Contest struct {
|
||||
// Kept: the running one stays and this grab was not made. Replaced:
|
||||
// this grab took its place.
|
||||
Kept, Replaced bool
|
||||
Reason string
|
||||
Rival dbq.Grab
|
||||
}
|
||||
|
||||
// Taken is what came of a grab.
|
||||
type Taken struct {
|
||||
Chosen *Candidate
|
||||
// Added is the only field that means a download started.
|
||||
Added bool
|
||||
SavePath string
|
||||
Grab dbq.Grab
|
||||
// Gone: the search no longer offers the release.
|
||||
Gone bool
|
||||
// Refused: not taken; the reason is on Chosen.
|
||||
Refused bool
|
||||
// Unconfigured: Missing names what is not set up ("" for search).
|
||||
Unconfigured bool
|
||||
Missing string
|
||||
Contest *Contest
|
||||
Error string
|
||||
}
|
||||
|
||||
// Take searches again and grabs the candidate named by key: a person's
|
||||
// grab. Searching again keeps the indexer's download link (which can carry
|
||||
// its key) on the server, and finds a release gone since it was shown.
|
||||
// force takes what the verdict would not.
|
||||
func (s *Service) Take(ctx context.Context, want Want, key string, force bool, origin Origin) Taken {
|
||||
if s.Grabber == nil {
|
||||
return Taken{Unconfigured: true, Missing: MissingClient}
|
||||
}
|
||||
found := s.For(ctx, want)
|
||||
switch {
|
||||
case found.Unconfigured:
|
||||
return Taken{Unconfigured: true}
|
||||
case found.Error != "":
|
||||
return Taken{Error: found.Error}
|
||||
}
|
||||
chosen := pick(found.Candidates, key)
|
||||
if chosen == nil {
|
||||
return Taken{Gone: true}
|
||||
}
|
||||
return s.Grab(ctx, want, chosen, found.Candidates, force, origin)
|
||||
}
|
||||
|
||||
// Grab hands a judged candidate to the download client. The schedule calls
|
||||
// it with the candidate it has just judged; Take calls it after searching
|
||||
// again.
|
||||
func (s *Service) Grab(ctx context.Context, want Want, chosen *Candidate, all []Candidate, force bool, origin Origin) Taken {
|
||||
out := Taken{Chosen: chosen}
|
||||
if s.Grabber == nil || s.Grabber.Client == nil {
|
||||
out.Unconfigured, out.Missing = true, MissingClient
|
||||
return out
|
||||
}
|
||||
if chosen.Verdict == Blocked || (chosen.Verdict != Grab && !force) {
|
||||
out.Refused = true
|
||||
return out
|
||||
}
|
||||
|
||||
unlock := lockWant(want.ID)
|
||||
defer unlock()
|
||||
contest := s.contest(ctx, want, chosen, all, origin)
|
||||
if contest != nil && contest.Kept {
|
||||
out.Contest = contest
|
||||
return out
|
||||
}
|
||||
|
||||
add := downloader.Add{Magnet: chosen.Source.Magnet, Tags: []string{Tag}}
|
||||
hash := chosen.InfoHash
|
||||
if add.Magnet == "" {
|
||||
if s.Grabber.Fetch == nil {
|
||||
out.Error = "This release is a .torrent link, and nothing is set up to fetch it."
|
||||
return out
|
||||
}
|
||||
body, err := s.Grabber.Fetch.Download(ctx, chosen.Source)
|
||||
if magnet, ok := indexer.AsMagnet(err); ok {
|
||||
add.Magnet, err = magnet, nil
|
||||
}
|
||||
if err != nil {
|
||||
out.Error = indexer.FirstLine(err.Error())
|
||||
return out
|
||||
}
|
||||
if add.Magnet == "" {
|
||||
add.Torrent = body
|
||||
if meta, err := release.ParseTorrent(body); err == nil {
|
||||
if hash == "" {
|
||||
hash = meta.InfoHash
|
||||
}
|
||||
if refused := s.judgeFiles(ctx, want, chosen, hash, meta.Files, force); refused {
|
||||
out.Refused = true
|
||||
return out
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if hash == "" && add.Magnet != "" {
|
||||
hash = indexer.MagnetHash(add.Magnet)
|
||||
}
|
||||
add.SavePath = landingFor(s.Grabber.Landing, hash, chosen.Name)
|
||||
|
||||
if err := s.Grabber.Client.AddTorrent(ctx, add); err != nil && !errors.Is(err, downloader.ErrDuplicate) {
|
||||
if errors.Is(err, downloader.ErrUnconfigured) {
|
||||
out.Unconfigured, out.Missing = true, MissingClient
|
||||
} else {
|
||||
out.Error = indexer.FirstLine(err.Error())
|
||||
}
|
||||
return out
|
||||
}
|
||||
out.Added, out.SavePath = true, add.SavePath
|
||||
if s.Memory == nil {
|
||||
return out
|
||||
}
|
||||
|
||||
var replaced pgtype.UUID
|
||||
why := ""
|
||||
if contest != nil && contest.Replaced {
|
||||
out.Contest = contest
|
||||
replaced, why = contest.Rival.ID, "Replaced by "+chosen.Name+": "+contest.Reason+"."
|
||||
}
|
||||
g, err := s.Memory.Grabbed(ctx, dbq.InsertGrabParams{
|
||||
WantID: want.ID, InfoHash: hash, Name: chosen.Name, Key: Key(*chosen), Indexer: chosen.Indexer,
|
||||
Source: chosen.Source.Source, Client: s.Grabber.Code, SavePath: add.SavePath,
|
||||
Quality: string(chosen.Tier), Origin: origin.By, Forced: force, GrabbedBy: origin.User,
|
||||
}, replaced, why)
|
||||
if err != nil {
|
||||
out.Error = "The download started, but could not be recorded: " + indexer.FirstLine(err.Error())
|
||||
return out
|
||||
}
|
||||
out.Grab = g
|
||||
// Removed only after the new download was added and recorded, so a
|
||||
// failure never leaves the want with nothing running.
|
||||
if replaced.Valid && contest.Rival.InfoHash != "" {
|
||||
if err := s.Grabber.Client.Delete(ctx, true, contest.Rival.InfoHash); err != nil {
|
||||
contest.Reason += ". The download it replaced could not be removed from the client: " + indexer.FirstLine(err.Error())
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// judgeFiles judges a chosen release again on its .torrent's file list, and
|
||||
// sets it aside when the files say it is not the album, so no later search
|
||||
// offers it again. force still takes it.
|
||||
func (s *Service) judgeFiles(ctx context.Context, want Want, chosen *Candidate, hash string, files []release.File, force bool) bool {
|
||||
chosen.Files = files
|
||||
if chosen.FilesCount == 0 {
|
||||
chosen.FilesCount = len(files)
|
||||
}
|
||||
veto, reason, _ := release.JudgeFiles(want.Want, files)
|
||||
if veto == "" || force {
|
||||
return false
|
||||
}
|
||||
set(chosen, Rejected, Hard, veto, reason)
|
||||
if s.Memory != nil && hash != "" {
|
||||
_ = s.Memory.Block(ctx, dbq.BlockReleaseParams{
|
||||
InfoHash: hash, Name: chosen.Name, Reason: BlockFiles, Detail: reason, WantID: want.ID,
|
||||
})
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
var wantLocks sync.Map // want id → *sync.Mutex
|
||||
|
||||
// lockWant settles two grabs for one want one after the other. The
|
||||
// one-live-grab index is the backstop for anything outside this process.
|
||||
func lockWant(id pgtype.UUID) func() {
|
||||
if !id.Valid {
|
||||
return func() {}
|
||||
}
|
||||
m, _ := wantLocks.LoadOrStore(id.Bytes, &sync.Mutex{})
|
||||
mu := m.(*sync.Mutex)
|
||||
mu.Lock()
|
||||
return mu.Unlock
|
||||
}
|
||||
|
||||
// contest settles a grab against the download running for its want: a
|
||||
// person's grab beats the schedule's; between two of a kind the better
|
||||
// candidate wins; a tie, a download nearly done, or one that cannot be
|
||||
// removed stays. Nil when nothing is running.
|
||||
func (s *Service) contest(ctx context.Context, want Want, chosen *Candidate, all []Candidate, origin Origin) *Contest {
|
||||
if s.Memory == nil || !want.ID.Valid {
|
||||
return nil
|
||||
}
|
||||
running, ok, err := s.Memory.LiveGrab(ctx, want.ID)
|
||||
if err != nil || !ok {
|
||||
return nil
|
||||
}
|
||||
c := &Contest{Rival: running}
|
||||
if running.InfoHash != "" && running.InfoHash == chosen.InfoHash {
|
||||
c.Kept, c.Reason = true, "That release is already downloading for this album."
|
||||
return c
|
||||
}
|
||||
switch {
|
||||
case origin.By == OriginOperator && running.Origin == OriginSchedule:
|
||||
c.Reason = "you chose it over the one the schedule started"
|
||||
case origin.By == OriginSchedule && running.Origin == OriginOperator:
|
||||
c.Kept, c.Reason = true, "A download you started for this album is already running."
|
||||
return c
|
||||
default:
|
||||
var theirs *Candidate
|
||||
for i := range all {
|
||||
if (running.InfoHash != "" && all[i].InfoHash == running.InfoHash) || Key(all[i]) == running.Key {
|
||||
theirs = &all[i]
|
||||
break
|
||||
}
|
||||
}
|
||||
if theirs == nil || !better(*chosen, *theirs) {
|
||||
c.Kept, c.Reason = true, "A download for this album is already running, and this release is no better."
|
||||
return c
|
||||
}
|
||||
c.Reason = "it ranks above the one that was running"
|
||||
}
|
||||
if running.InfoHash == "" {
|
||||
c.Kept, c.Reason = true, "A download for this album is already running, and it cannot be identified to replace it."
|
||||
return c
|
||||
}
|
||||
if ts, err := s.Grabber.Client.Torrents(ctx, running.InfoHash); err == nil && len(ts) > 0 && ts[0].Progress >= nearlyDone {
|
||||
c.Kept, c.Reason = true, "A download for this album is already running and nearly done."
|
||||
return c
|
||||
}
|
||||
c.Replaced = true
|
||||
return c
|
||||
}
|
||||
|
||||
// landingFor is the folder a download lands in: one per torrent, named by
|
||||
// its infohash, so two grabs never unpack into the same place.
|
||||
func landingFor(root, hash, name string) string {
|
||||
dir := hash
|
||||
if dir == "" {
|
||||
dir = slug(name)
|
||||
}
|
||||
if dir == "" {
|
||||
dir = "unnamed"
|
||||
}
|
||||
return path.Join(root, dir)
|
||||
}
|
||||
|
||||
// slug keeps a folder name to characters every filesystem in the chain
|
||||
// takes.
|
||||
func slug(s string) string {
|
||||
var b strings.Builder
|
||||
for _, r := range strings.ToLower(strings.TrimSpace(s)) {
|
||||
if b.Len() >= 80 {
|
||||
break
|
||||
}
|
||||
switch {
|
||||
case r >= 'a' && r <= 'z', r >= '0' && r <= '9':
|
||||
b.WriteRune(r)
|
||||
case r == '-', r == '_', r == ' ', r == '.':
|
||||
b.WriteRune('-')
|
||||
}
|
||||
}
|
||||
return strings.Trim(b.String(), "-")
|
||||
}
|
||||
Reference in New Issue
Block a user