feat: search and grab — music release names, a quality ladder, a paced search schedule and one live grab per want (M502 #5521)
release / govulncheck (push) Successful in 21s
release / go (push) Failing after 33s
release / web (push) Successful in 56s
release / integration (push) Failing after 4m48s
release / android (push) Successful in 5m6s
release / Build signed APK (releases and dev) (push) Successful in 5m16s
release / Attach APK to the Release (tag releases only) (push) Skipped
release / Build + push container image (push) Skipped
release / Verify release artifacts (tag releases only) (push) Skipped
release / govulncheck (push) Successful in 21s
release / go (push) Failing after 33s
release / web (push) Successful in 56s
release / integration (push) Failing after 4m48s
release / android (push) Successful in 5m6s
release / Build signed APK (releases and dev) (push) Successful in 5m16s
release / Attach APK to the Release (tag releases only) (push) Skipped
release / Build + push container image (push) Skipped
release / Verify release artifacts (tag releases only) (push) Skipped
- internal/release: reads a release name (format, bitrate, bit depth and sample rate, source, year, video, discography) and judges it against the album wanted: the artist and title must be in it; video rips, discographies, other versions (drumless, instrumental, karaoke, remix, live unless the album is live…) and singles or EPs for an album are refused. Quality tiers on Lidarr's ladder; a .torrent's file list is read for format, video files and a track count short of the album. - internal/acquire: fan-out across every search source with a narrowed search kept, a process-wide pacer per source, every candidate judged and kept with its reasons, and the grab: one live grab per want, tie-broken (a person's grab beats the schedule's; a better release replaces a worse one unless it is nearly done). Each download lands in its own folder under the client's downloads folder. - internal/wanted: the search schedule, every ten minutes, never-searched first, then requests, missing files and followed artists, taking the best release when auto-grab is on (the default). - Migration 0082: acquisition_settings (auto-grab, interval, allowed qualities seeded from Lidarr's "Any"), want search columns, want_candidates, grabs, release_blocks. - Web: the Wanted page shows each want's last search, a Search button and every release found with its verdict (Grab, Grab anyway, Never this one, Allow again). Integrations gains "Searching and grabbing". Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,195 @@
|
||||
package wanted
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"io"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
|
||||
"git.fabledsword.com/bvandeusen/minstrel/internal/acquire"
|
||||
"git.fabledsword.com/bvandeusen/minstrel/internal/db/dbq"
|
||||
)
|
||||
|
||||
const ramHash = "0123456789abcdef0123456789abcdef01234567"
|
||||
|
||||
const ramFeed = `<?xml version="1.0"?>
|
||||
<rss xmlns:torznab="http://torznab.com/schemas/2015/feed"><channel>
|
||||
<item><title>Daft Punk - Random Access Memories (2013) [FLAC]</title>
|
||||
<link>magnet:?xt=urn:btih:` + ramHash + `</link><size>400000000</size>
|
||||
<torznab:attr name="seeders" value="12"/><torznab:attr name="indexer" value="Knaben"/></item>
|
||||
<item><title>Daft Punk Random Access Memories (Drumless Edition 2023) [Flac 24-88]</title>
|
||||
<link>magnet:?xt=urn:btih:fedcba9876543210fedcba9876543210fedcba98</link><size>900000000</size>
|
||||
<torznab:attr name="seeders" value="40"/></item>
|
||||
</channel></rss>`
|
||||
|
||||
// searchStack is a fake indexer feed and a fake FabledExchange download
|
||||
// client, configured the way the operator would in Integrations.
|
||||
func searchStack(t *testing.T, ctx context.Context, store *acquire.Store) *atomic.Int32 {
|
||||
t.Helper()
|
||||
var added atomic.Int32
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
switch {
|
||||
case r.URL.Path == "/feed":
|
||||
if q := r.URL.Query().Get("q"); q != "Daft Punk Random Access Memories" {
|
||||
t.Errorf("searched for %q", q)
|
||||
}
|
||||
_, _ = io.WriteString(w, ramFeed)
|
||||
case r.URL.Path == "/api/downloads" && r.Method == http.MethodPost:
|
||||
var body struct {
|
||||
Magnet string `json:"magnet"`
|
||||
SavePath string `json:"save_path"`
|
||||
}
|
||||
_ = json.NewDecoder(r.Body).Decode(&body)
|
||||
if !strings.Contains(body.Magnet, ramHash) || body.SavePath != "/downloads/"+ramHash {
|
||||
t.Errorf("added %+v", body)
|
||||
}
|
||||
added.Add(1)
|
||||
_, _ = io.WriteString(w, "{}")
|
||||
default:
|
||||
http.NotFound(w, r)
|
||||
}
|
||||
}))
|
||||
t.Cleanup(srv.Close)
|
||||
|
||||
feed, err := store.Create(ctx, acquire.ProtocolTorznab)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := store.Save(ctx, feed.Code, acquire.Update{
|
||||
Label: "Feed", Enabled: true, BaseURL: srv.URL + "/feed", TimeoutMs: 5000, LocalPath: "/downloads",
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ex, err := store.Create(ctx, acquire.ProtocolExchange)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := store.Save(ctx, ex.Code, acquire.Update{
|
||||
Label: "Exchange", Enabled: true, BaseURL: srv.URL, TimeoutMs: 5000, ServesDownloads: true, LocalPath: "/downloads",
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return &added
|
||||
}
|
||||
|
||||
func TestSearcher_FindsAndTakesAWant_Integration(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
pool := newPool(t)
|
||||
q := dbq.New(pool)
|
||||
added := searchStack(t, ctx, acquire.NewStore(pool))
|
||||
w, err := WantWith(ctx, q, WantParams{
|
||||
GroupMBID: "rg-ram", ArtistMBID: "mb-daft-punk", ArtistName: "Daft Punk", Title: "Random Access Memories",
|
||||
PrimaryType: "Album", FirstReleaseDate: "2013-05-17", Reason: ReasonRequest,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
s := NewSearcher(pool, nil, slog.New(slog.NewTextHandler(io.Discard, nil)))
|
||||
|
||||
rep := s.Pass(ctx)
|
||||
if rep.Looked != 1 || rep.Grabbed != 1 || !rep.AutoGrab || added.Load() != 1 {
|
||||
t.Fatalf("pass = %+v, added %d", rep, added.Load())
|
||||
}
|
||||
got, err := q.GetWantByReleaseGroup(ctx, "rg-ram")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got.SearchOutcome != OutcomeGrabbed || got.SearchConsidered != 2 || !strings.Contains(got.BestRelease, "(2013) [FLAC]") {
|
||||
t.Errorf("want after the pass: %s %d %q", got.SearchOutcome, got.SearchConsidered, got.BestRelease)
|
||||
}
|
||||
cands, err := q.ListWantCandidates(ctx, w.ID)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(cands) != 2 || cands[0].Verdict != "grab" || cands[1].Verdict != "rejected" || !strings.Contains(cands[1].Reason, "drumless") {
|
||||
t.Errorf("candidates = %+v", cands)
|
||||
}
|
||||
g, err := q.GetLiveGrab(ctx, w.ID)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if g.InfoHash != ramHash || g.Client != "exchange" || g.Origin != acquire.OriginSchedule || g.Quality != "lossless" {
|
||||
t.Errorf("grab = %+v", g)
|
||||
}
|
||||
settings, _ := q.GetAcquisitionSettings(ctx)
|
||||
if !settings.LastPassAt.Valid || !strings.Contains(string(settings.LastPass), `"grabbed":1`) {
|
||||
t.Errorf("last pass = %s", settings.LastPass)
|
||||
}
|
||||
|
||||
// A want with a download running is not searched again.
|
||||
if rep := s.Pass(ctx); rep.Looked != 0 {
|
||||
t.Errorf("searched a want already downloading: %+v", rep)
|
||||
}
|
||||
|
||||
// Setting the drumless edition aside marks it on every kept candidate;
|
||||
// allowing it again leaves it for a person until the next search.
|
||||
mem := acquire.NewMemory(pool)
|
||||
if err := mem.Block(ctx, dbq.BlockReleaseParams{
|
||||
InfoHash: cands[1].InfoHash, Name: cands[1].Name, Reason: acquire.BlockOperator, WantID: w.ID,
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
c, _ := q.GetWantCandidate(ctx, dbq.GetWantCandidateParams{WantID: w.ID, Key: cands[1].Key})
|
||||
if c.Verdict != "blocked" {
|
||||
t.Errorf("blocked candidate = %s", c.Verdict)
|
||||
}
|
||||
if ok, err := mem.Unblock(ctx, cands[1].InfoHash); err != nil || !ok {
|
||||
t.Fatalf("unblock: %v %v", ok, err)
|
||||
}
|
||||
c, _ = q.GetWantCandidate(ctx, dbq.GetWantCandidateParams{WantID: w.ID, Key: cands[1].Key})
|
||||
if c.Verdict != "review" {
|
||||
t.Errorf("unblocked candidate = %s", c.Verdict)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSearcher_OffAndUnconfigured_Integration(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
pool := newPool(t)
|
||||
s := NewSearcher(pool, nil, slog.New(slog.NewTextHandler(io.Discard, nil)))
|
||||
if rep := s.Pass(ctx); !rep.Unconfigured {
|
||||
t.Errorf("no indexer: %+v", rep)
|
||||
}
|
||||
if _, err := dbq.New(pool).UpdateAcquisitionSettings(ctx, dbq.UpdateAcquisitionSettingsParams{
|
||||
AutoGrab: true, SearchIntervalHours: 0, AllowedQualities: []string{"lossless"},
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if rep := s.Pass(ctx); !rep.Off {
|
||||
t.Errorf("interval 0: %+v", rep)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGrabs_OneLivePerWant_Integration(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
pool := newPool(t)
|
||||
q := dbq.New(pool)
|
||||
w, err := WantWith(ctx, q, WantParams{GroupMBID: "rg-x", ArtistMBID: "mb-x", ArtistName: "X", Title: "Y", Reason: ReasonFollow})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
mem := acquire.NewMemory(pool)
|
||||
grab := func(hash string) dbq.InsertGrabParams {
|
||||
return dbq.InsertGrabParams{WantID: w.ID, InfoHash: hash, Name: hash, Key: hash, Client: "qbittorrent",
|
||||
SavePath: "/downloads/" + hash, Origin: acquire.OriginSchedule}
|
||||
}
|
||||
first, err := mem.Grabbed(ctx, grab("aaaa"), dbq.Grab{}.ID, "")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := mem.Grabbed(ctx, grab("bbbb"), dbq.Grab{}.ID, ""); err == nil {
|
||||
t.Error("a second live grab for one want was recorded")
|
||||
}
|
||||
second, err := mem.Grabbed(ctx, grab("bbbb"), first.ID, "Replaced by bbbb.")
|
||||
if err != nil {
|
||||
t.Fatalf("replacing: %v", err)
|
||||
}
|
||||
live, ok, err := mem.LiveGrab(ctx, w.ID)
|
||||
if err != nil || !ok || live.ID != second.ID {
|
||||
t.Errorf("live = %+v %v %v", live, ok, err)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user