feat: indexers and download clients you add and configure (M502 #5518)
release / govulncheck (push) Successful in 34s
release / web (push) Successful in 1m13s
release / go (push) Successful in 2m12s
release / integration (push) Successful in 4m6s
release / Build signed APK (releases and dev) (push) Successful in 8m41s
release / android (push) Successful in 9m47s
release / Attach APK to the Release (tag releases only) (push) Skipped
release / Build + push container image (push) Successful in 59s
release / Verify release artifacts (tag releases only) (push) Skipped
release / govulncheck (push) Successful in 34s
release / web (push) Successful in 1m13s
release / go (push) Successful in 2m12s
release / integration (push) Successful in 4m6s
release / Build signed APK (releases and dev) (push) Successful in 8m41s
release / android (push) Successful in 9m47s
release / Attach APK to the Release (tag releases only) (push) Skipped
release / Build + push container image (push) Successful in 59s
release / Verify release artifacts (tag releases only) (push) Skipped
Admin → Integrations gains "Indexers and download clients": two lists of
connections the operator adds, Fabled Librarian's model. Nothing is seeded
or copied from Lidarr.
- Indexers: Prowlarr (its native /api/v1/search, which asks every indexer
at once; Prowlarr has no aggregate Torznab feed) and any Torznab feed.
- Download clients: qBittorrent (WebUI v2, with re-login on an expired
session) and FabledExchange, one entry with a switch per job:
downloads (/api/downloads), search (/api/search) and release details
(/api/torrents/{infohash}). All three start on; its key is optional.
- Each entry: enable, name, address, a write-only key or password, Test.
A download client's Test lists its torrents and checks the downloads
folder is visible and writable from Minstrel; an Exchange's Test
reports each job switched on.
- Download clients carry a path mapping (Minstrel's /downloads vs the
client's own name for it); search sources carry searches per hour.
Migration 0079 adds acquisition_services with its CHECKs. Ported from
Fabled Librarian: internal/indexer (Torznab, 429 Retry-After, magnet
redirects), internal/prowlarr, internal/exchange, internal/downloader,
and internal/acquire's one builder per kind, which Test already uses
so a passing Test vouches for the path a grab will take.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,157 @@
|
||||
// Package indexer is what every search source answers with, and the Torznab
|
||||
// client (M502 #5518). Ported from Fabled Librarian's internal/indexer.
|
||||
//
|
||||
// A client here holds ONE endpoint and knows nothing about the others. The
|
||||
// fan-out across sources, the merge and what a source being down means belong
|
||||
// to internal/acquire, because they are decisions about a search rather than
|
||||
// about one source.
|
||||
//
|
||||
// Prowlarr is not served from here: it has no aggregate Torznab feed (`all`
|
||||
// 404s like any id it does not have, measured by Fabled Librarian, #4304), so
|
||||
// asking it everything at once means its native API, which internal/prowlarr
|
||||
// speaks.
|
||||
package indexer
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// ErrUnconfigured is a source with no address (or, for Prowlarr, no key): not
|
||||
// a failure, an instance that does not search there.
|
||||
var ErrUnconfigured = errors.New("indexer is not configured")
|
||||
|
||||
// ErrUnreachable is a search that could not be made: no network, a timeout, a
|
||||
// malformed answer, a status outside 2xx.
|
||||
var ErrUnreachable = errors.New("indexer unreachable")
|
||||
|
||||
// ErrAuth is a key the source refused. Separate from ErrUnreachable because
|
||||
// it needs a different sentence and a different action: a refused key is a
|
||||
// field in Settings, an outage is something to wait out.
|
||||
var ErrAuth = errors.New("the API key was refused")
|
||||
|
||||
// UserAgent names Minstrel in the operator's own Prowlarr and indexer logs.
|
||||
const UserAgent = "Minstrel (self-hosted music server; release search)"
|
||||
|
||||
// MusicCategories are the Newznab categories a music search asks for: 3000
|
||||
// Audio, with its MP3 (3010) and Lossless (3040) children alongside, because
|
||||
// an indexer that files music under a child answers only for the child.
|
||||
// 3030 (Audiobook) is deliberately absent.
|
||||
var MusicCategories = []int{3000, 3010, 3040}
|
||||
|
||||
// Query is one search: the words, and which categories to ask in. Empty
|
||||
// Categories asks every category.
|
||||
type Query struct {
|
||||
Text string
|
||||
Categories []int
|
||||
}
|
||||
|
||||
// Result is one search result, as the source described it.
|
||||
type Result struct {
|
||||
Title string
|
||||
// InfoHash is the torrent's identity, lowercased; empty when the source
|
||||
// gave none.
|
||||
InfoHash string
|
||||
// Magnet is a magnet URI when the source published one.
|
||||
Magnet string
|
||||
// DownloadURL is a .torrent URL when it did not. It must be fetched
|
||||
// through the source that offered it (it holds the tracker credentials),
|
||||
// never handed to the download client.
|
||||
DownloadURL string
|
||||
SizeBytes int64
|
||||
Category int
|
||||
// Indexer is the tracker BEHIND the source, as the source named it.
|
||||
Indexer string
|
||||
// Source is the acquisition_services code that was asked: where a
|
||||
// .torrent fetch must be routed back to.
|
||||
Source string
|
||||
Published time.Time
|
||||
// Seeders and Peers as reported. A feed that has not sampled a swarm
|
||||
// reports a nominal 1, so these are not health unless SwarmKnown.
|
||||
Seeders int
|
||||
Peers int
|
||||
// SwarmKnown says Seeders was measured, which only FabledExchange's own
|
||||
// search can say.
|
||||
SwarmKnown bool
|
||||
// FilesCount is how many files the torrent holds, 0 when not said.
|
||||
FilesCount int
|
||||
}
|
||||
|
||||
// Send sends req through c, stopping at a redirect to a magnet (returned as a
|
||||
// *MagnetLink), and reads at most 8 MB of the answer. A call that never got
|
||||
// an answer is ErrUnreachable; the status is the caller's to read.
|
||||
func Send(c *http.Client, req *http.Request) (*http.Response, []byte, error) {
|
||||
resp, err := StopAtMagnet(c).Do(req)
|
||||
if m, ok := AsMagnet(err); ok {
|
||||
return nil, nil, &MagnetLink{URI: m}
|
||||
}
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("%w: %v", ErrUnreachable, err)
|
||||
}
|
||||
defer func() { _ = resp.Body.Close() }()
|
||||
body, err := io.ReadAll(io.LimitReader(resp.Body, 8<<20))
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("%w: read body: %v", ErrUnreachable, err)
|
||||
}
|
||||
return resp, body, nil
|
||||
}
|
||||
|
||||
// StatusError is what a non-2xx status means for a search source: a 429 is
|
||||
// RateLimited naming who, anything else ErrUnreachable. Nil for a 2xx.
|
||||
func StatusError(resp *http.Response, who string) error {
|
||||
switch {
|
||||
case resp.StatusCode == http.StatusTooManyRequests:
|
||||
return &RateLimited{Who: who, RetryAfter: RetryAfter(resp.Header, time.Now())}
|
||||
case resp.StatusCode < 200 || resp.StatusCode >= 300:
|
||||
return fmt.Errorf("%w: HTTP %d", ErrUnreachable, resp.StatusCode)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// NormalizeHash is a torrent's identity in its one canonical spelling. Every
|
||||
// comparison of infohashes goes through it, so a hash differing only in case
|
||||
// is never two releases.
|
||||
func NormalizeHash(s string) string {
|
||||
return strings.ToLower(strings.TrimSpace(s))
|
||||
}
|
||||
|
||||
// IsInfoHash says whether s is shaped like a v1 (40) or v2 (64) hex infohash.
|
||||
func IsInfoHash(s string) bool {
|
||||
s = NormalizeHash(s)
|
||||
if len(s) != 40 && len(s) != 64 {
|
||||
return false
|
||||
}
|
||||
for _, r := range s {
|
||||
if (r < '0' || r > '9') && (r < 'a' || r > 'f') {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// MagnetHash pulls the infohash out of a magnet link, or returns "". It does
|
||||
// not validate it.
|
||||
func MagnetHash(magnet string) string {
|
||||
const prefix = "urn:btih:"
|
||||
at := strings.Index(magnet, prefix)
|
||||
if at < 0 {
|
||||
return ""
|
||||
}
|
||||
hash := magnet[at+len(prefix):]
|
||||
if cut := strings.IndexAny(hash, "&/?"); cut >= 0 {
|
||||
hash = hash[:cut]
|
||||
}
|
||||
return strings.ToLower(hash)
|
||||
}
|
||||
|
||||
// FirstLine is an error's first line, for a sentence on the settings page.
|
||||
func FirstLine(s string) string {
|
||||
if i := strings.IndexByte(s, '\n'); i >= 0 {
|
||||
return s[:i]
|
||||
}
|
||||
return s
|
||||
}
|
||||
@@ -0,0 +1,58 @@
|
||||
package indexer
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"errors"
|
||||
"net/http"
|
||||
)
|
||||
|
||||
// MagnetLink is a download link that turned out to be a magnet. Returned as
|
||||
// an error by a .torrent fetch so the grab hands the magnet to the download
|
||||
// client instead. Public trackers do this routinely: Prowlarr's download link
|
||||
// answers with a redirect to `magnet:?xt=…`, which Go's client would otherwise
|
||||
// fail on as an unsupported scheme.
|
||||
type MagnetLink struct{ URI string }
|
||||
|
||||
func (m *MagnetLink) Error() string { return "the download link is a magnet link" }
|
||||
|
||||
// AsMagnet says whether err is a magnet link, and which.
|
||||
func AsMagnet(err error) (string, bool) {
|
||||
var m *MagnetLink
|
||||
if errors.As(err, &m) && m.URI != "" {
|
||||
return m.URI, true
|
||||
}
|
||||
return "", false
|
||||
}
|
||||
|
||||
// StopAtMagnet returns a copy of c that stops at a redirect to a magnet URI
|
||||
// and reports it as a *MagnetLink. Every other redirect behaves as c would.
|
||||
func StopAtMagnet(c *http.Client) *http.Client {
|
||||
if c == nil {
|
||||
c = http.DefaultClient
|
||||
}
|
||||
out := *c
|
||||
next := c.CheckRedirect
|
||||
out.CheckRedirect = func(req *http.Request, via []*http.Request) error {
|
||||
if req.URL != nil && req.URL.Scheme == "magnet" {
|
||||
return &MagnetLink{URI: req.URL.String()}
|
||||
}
|
||||
if next != nil {
|
||||
return next(req, via)
|
||||
}
|
||||
if len(via) >= 10 {
|
||||
return errors.New("stopped after 10 redirects")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
return &out
|
||||
}
|
||||
|
||||
// MagnetBody is a response whose whole body is a magnet URI, which some
|
||||
// indexers send with a 200 instead of a redirect. Nil otherwise.
|
||||
func MagnetBody(body []byte) error {
|
||||
trimmed := bytes.TrimSpace(body)
|
||||
if bytes.HasPrefix(trimmed, []byte("magnet:")) && !bytes.ContainsAny(trimmed, " \n<") {
|
||||
return &MagnetLink{URI: string(trimmed)}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,50 @@
|
||||
package indexer
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"time"
|
||||
)
|
||||
|
||||
// RateLimited is a source answering 429. Named so the search pacing can hold
|
||||
// that source back for as long as it asked; still an ErrUnreachable to
|
||||
// everything else, because this search did not reach it.
|
||||
type RateLimited struct {
|
||||
// RetryAfter is what the source asked for; zero when it did not say.
|
||||
RetryAfter time.Duration
|
||||
// Who names the source in the sentence, e.g. "Prowlarr".
|
||||
Who string
|
||||
}
|
||||
|
||||
func (e *RateLimited) Error() string {
|
||||
return fmt.Sprintf("%v: HTTP 429, %s is rate-limiting this search", ErrUnreachable, e.Who)
|
||||
}
|
||||
|
||||
func (e *RateLimited) Unwrap() error { return ErrUnreachable }
|
||||
|
||||
// AsRateLimited finds a RateLimited anywhere in err's chain.
|
||||
func AsRateLimited(err error) (*RateLimited, bool) {
|
||||
var rl *RateLimited
|
||||
if errors.As(err, &rl) {
|
||||
return rl, true
|
||||
}
|
||||
return nil, false
|
||||
}
|
||||
|
||||
// RetryAfter reads a Retry-After header: seconds, or an HTTP date. Zero when
|
||||
// absent or unreadable.
|
||||
func RetryAfter(h http.Header, now time.Time) time.Duration {
|
||||
v := h.Get("Retry-After")
|
||||
if v == "" {
|
||||
return 0
|
||||
}
|
||||
if s, err := strconv.Atoi(v); err == nil && s > 0 {
|
||||
return time.Duration(s) * time.Second
|
||||
}
|
||||
if t, err := http.ParseTime(v); err == nil && t.After(now) {
|
||||
return t.Sub(now)
|
||||
}
|
||||
return 0
|
||||
}
|
||||
@@ -0,0 +1,254 @@
|
||||
package indexer
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/xml"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// DefaultTimeout is what an unset Timeout means. Longer than a metadata
|
||||
// call: a feed may itself fan out to several trackers before answering.
|
||||
const DefaultTimeout = 30 * time.Second
|
||||
|
||||
// TorznabConfig is one Torznab feed.
|
||||
type TorznabConfig struct {
|
||||
// Code is the acquisition_services row this client was built from,
|
||||
// stamped onto every Result.
|
||||
Code string
|
||||
// FeedURL is the feed's whole address up to and including /api, as
|
||||
// Jackett or a Prowlarr indexer shows it — e.g.
|
||||
// http://prowlarr:9696/12/api. Query parameters are appended to it.
|
||||
// One field rather than Fabled Librarian's base + path + indexer id: a
|
||||
// pasted URL is more reliable than Minstrel being right about a third
|
||||
// party's routes.
|
||||
FeedURL string
|
||||
// APIKey is optional: Torznab makes it so. A feed that needs one says so
|
||||
// with Newznab error 100, which is ErrAuth.
|
||||
APIKey string
|
||||
Timeout time.Duration
|
||||
}
|
||||
|
||||
// Torznab searches one Torznab feed.
|
||||
type Torznab struct {
|
||||
Config TorznabConfig
|
||||
// HTTP is injectable for tests. Nil takes http.DefaultClient.
|
||||
HTTP *http.Client
|
||||
}
|
||||
|
||||
// Search asks the feed for q. `t=search` is free text, which every indexer
|
||||
// supports; `t=music` takes artist and album separately, and a release name
|
||||
// does not reliably say which of its parts is which.
|
||||
func (c *Torznab) Search(ctx context.Context, q Query) ([]Result, error) {
|
||||
params := url.Values{"t": {"search"}, "q": {q.Text}}
|
||||
for _, cat := range q.Categories {
|
||||
params.Add("cat", strconv.Itoa(cat))
|
||||
}
|
||||
body, err := c.get(ctx, c.endpoint(params))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out, err := parseFeed(body)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for i := range out {
|
||||
out[i].Source = c.Config.Code
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// Download fetches a result's .torrent bytes through the feed, which holds
|
||||
// the tracker's credentials. A result with a magnet needs none of this.
|
||||
func (c *Torznab) Download(ctx context.Context, r Result) ([]byte, error) {
|
||||
if r.DownloadURL == "" {
|
||||
return nil, fmt.Errorf("%w: %q has no download URL", ErrUnreachable, r.Title)
|
||||
}
|
||||
body, err := c.get(ctx, r.DownloadURL)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return torrentBody(r.Title, body)
|
||||
}
|
||||
|
||||
// torrentBody checks a fetched .torrent. A torrent is a bencoded dictionary
|
||||
// and starts with 'd'; an expired session gets an HTML login page with HTTP
|
||||
// 200, which the download client would otherwise reject as an unsupported
|
||||
// file from a component that did nothing wrong.
|
||||
func torrentBody(title string, body []byte) ([]byte, error) {
|
||||
if err := MagnetBody(body); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(body) == 0 || body[0] != 'd' {
|
||||
return nil, fmt.Errorf("%w: %q returned %d bytes that are not a torrent", ErrUnreachable, title, len(body))
|
||||
}
|
||||
return body, nil
|
||||
}
|
||||
|
||||
// TorrentBody is torrentBody for the other search clients' fetches.
|
||||
func TorrentBody(title string, body []byte) ([]byte, error) { return torrentBody(title, body) }
|
||||
|
||||
func (c *Torznab) endpoint(params url.Values) string {
|
||||
if c.Config.APIKey != "" {
|
||||
params.Set("apikey", c.Config.APIKey)
|
||||
}
|
||||
feed := strings.TrimSpace(c.Config.FeedURL)
|
||||
sep := "?"
|
||||
if strings.Contains(feed, "?") {
|
||||
sep = "&"
|
||||
}
|
||||
return feed + sep + params.Encode()
|
||||
}
|
||||
|
||||
func (c *Torznab) get(ctx context.Context, endpoint string) ([]byte, error) {
|
||||
if strings.TrimSpace(c.Config.FeedURL) == "" {
|
||||
return nil, ErrUnconfigured
|
||||
}
|
||||
timeout := c.Config.Timeout
|
||||
if timeout <= 0 {
|
||||
timeout = DefaultTimeout
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(ctx, timeout)
|
||||
defer cancel()
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, endpoint, nil)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%w: build request: %v", ErrUnreachable, err)
|
||||
}
|
||||
req.Header.Set("User-Agent", UserAgent)
|
||||
resp, body, err := Send(c.HTTP, req)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := StatusError(resp, "the indexer"); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// Newznab reports errors as HTTP 200 with an <error> document, so a
|
||||
// client checking only the status reads a refused key as an empty result
|
||||
// set: search quietly stops finding things.
|
||||
if err := errorDocument(body); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return body, nil
|
||||
}
|
||||
|
||||
// errorDocument reads an <error code=… description=…/> answer. Newznab's 1xx
|
||||
// codes (100 wrong key, 101 suspended, 102 insufficient privileges) are all
|
||||
// about who is asking, so they are ErrAuth.
|
||||
func errorDocument(body []byte) error {
|
||||
head := body
|
||||
if len(head) > 512 {
|
||||
head = head[:512]
|
||||
}
|
||||
if !strings.Contains(string(head), "<error") {
|
||||
return nil
|
||||
}
|
||||
var doc struct {
|
||||
XMLName xml.Name `xml:"error"`
|
||||
Code string `xml:"code,attr"`
|
||||
Description string `xml:"description,attr"`
|
||||
}
|
||||
if err := xml.Unmarshal(body, &doc); err != nil {
|
||||
return nil
|
||||
}
|
||||
if doc.Code == "" && doc.Description == "" {
|
||||
return nil
|
||||
}
|
||||
if strings.HasPrefix(doc.Code, "1") && len(doc.Code) == 3 {
|
||||
return fmt.Errorf("%w: indexer error %s: %s", ErrAuth, doc.Code, doc.Description)
|
||||
}
|
||||
return fmt.Errorf("%w: indexer error %s: %s", ErrUnreachable, doc.Code, doc.Description)
|
||||
}
|
||||
|
||||
// The Torznab feed: RSS 2.0 with torznab:attr for what RSS has no field for.
|
||||
type feed struct {
|
||||
Channel struct {
|
||||
Items []item `xml:"item"`
|
||||
} `xml:"channel"`
|
||||
}
|
||||
|
||||
type item struct {
|
||||
Title string `xml:"title"`
|
||||
GUID string `xml:"guid"`
|
||||
Link string `xml:"link"`
|
||||
PubDate string `xml:"pubDate"`
|
||||
Size int64 `xml:"size"`
|
||||
Enclosure struct {
|
||||
URL string `xml:"url,attr"`
|
||||
Length int64 `xml:"length,attr"`
|
||||
} `xml:"enclosure"`
|
||||
Attrs []struct {
|
||||
Name string `xml:"name,attr"`
|
||||
Value string `xml:"value,attr"`
|
||||
} `xml:"attr"`
|
||||
}
|
||||
|
||||
func (i item) attr(name string) string {
|
||||
for _, a := range i.Attrs {
|
||||
if strings.EqualFold(a.Name, name) {
|
||||
return a.Value
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func parseFeed(body []byte) ([]Result, error) {
|
||||
var f feed
|
||||
if err := xml.Unmarshal(body, &f); err != nil {
|
||||
return nil, fmt.Errorf("%w: parse feed: %v", ErrUnreachable, err)
|
||||
}
|
||||
out := make([]Result, 0, len(f.Channel.Items))
|
||||
for _, it := range f.Channel.Items {
|
||||
r := Result{
|
||||
Title: strings.TrimSpace(it.Title),
|
||||
SizeBytes: it.Size,
|
||||
Indexer: it.attr("indexer"),
|
||||
Seeders: atoi(it.attr("seeders")),
|
||||
Peers: atoi(it.attr("peers")),
|
||||
Category: atoi(it.attr("category")),
|
||||
}
|
||||
if r.Title == "" {
|
||||
continue
|
||||
}
|
||||
if r.SizeBytes == 0 {
|
||||
r.SizeBytes = it.Enclosure.Length
|
||||
}
|
||||
// The link is a magnet or a .torrent URL; Prowlarr puts it in <link>
|
||||
// and repeats it in <enclosure url>, either populated by version.
|
||||
for _, candidate := range []string{it.Link, it.Enclosure.URL} {
|
||||
if strings.HasPrefix(candidate, "magnet:") && r.Magnet == "" {
|
||||
r.Magnet = candidate
|
||||
} else if candidate != "" && r.DownloadURL == "" {
|
||||
r.DownloadURL = candidate
|
||||
}
|
||||
}
|
||||
// The infohash from the explicit attr, then the magnet, then the GUID
|
||||
// only when it is shaped like a hash: Prowlarr puts a URL there as
|
||||
// often as a hash.
|
||||
r.InfoHash = NormalizeHash(it.attr("infohash"))
|
||||
if r.InfoHash == "" {
|
||||
r.InfoHash = MagnetHash(r.Magnet)
|
||||
}
|
||||
if r.InfoHash == "" && IsInfoHash(it.GUID) {
|
||||
r.InfoHash = NormalizeHash(it.GUID)
|
||||
}
|
||||
if t, err := time.Parse(time.RFC1123Z, strings.TrimSpace(it.PubDate)); err == nil {
|
||||
r.Published = t
|
||||
} else if t, err := time.Parse(time.RFC1123, strings.TrimSpace(it.PubDate)); err == nil {
|
||||
r.Published = t
|
||||
}
|
||||
out = append(out, r)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func atoi(s string) int {
|
||||
n, err := strconv.Atoi(strings.TrimSpace(s))
|
||||
if err != nil {
|
||||
return 0
|
||||
}
|
||||
return n
|
||||
}
|
||||
@@ -0,0 +1,188 @@
|
||||
package indexer
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
const sampleFeed = `<?xml version="1.0" encoding="UTF-8"?>
|
||||
<rss version="2.0" xmlns:torznab="http://torznab.com/schemas/2015/feed">
|
||||
<channel>
|
||||
<item>
|
||||
<title>Radiohead - OK Computer (1997) [FLAC]</title>
|
||||
<guid>https://tracker.example/details/1</guid>
|
||||
<link>https://prowlarr.example/1/download?link=abc</link>
|
||||
<size>512000000</size>
|
||||
<pubDate>Tue, 07 Oct 2026 10:00:00 +0000</pubDate>
|
||||
<torznab:attr name="seeders" value="12"/>
|
||||
<torznab:attr name="peers" value="15"/>
|
||||
<torznab:attr name="category" value="3040"/>
|
||||
<torznab:attr name="indexer" value="RED"/>
|
||||
<torznab:attr name="infohash" value="AABBCCDDEEFF00112233445566778899AABBCCDD"/>
|
||||
</item>
|
||||
<item>
|
||||
<title>Radiohead - Kid A (2000) [MP3 320]</title>
|
||||
<guid>0123456789abcdef0123456789abcdef01234567</guid>
|
||||
<link>magnet:?xt=urn:btih:FFEEDDCCBBAA99887766554433221100FFEEDDCC&dn=kid+a</link>
|
||||
</item>
|
||||
<item>
|
||||
<title> </title>
|
||||
</item>
|
||||
</channel>
|
||||
</rss>`
|
||||
|
||||
func TestTorznabSearch_ParsesTheFeedAndStampsTheSource(t *testing.T) {
|
||||
reqs := make(chan *http.Request, 1)
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
reqs <- r
|
||||
_, _ = w.Write([]byte(sampleFeed))
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
c := &Torznab{Config: TorznabConfig{Code: "torznab-2", FeedURL: srv.URL + "/12/api", APIKey: "k"}}
|
||||
results, err := c.Search(context.Background(), Query{Text: "radiohead", Categories: MusicCategories})
|
||||
if err != nil {
|
||||
t.Fatalf("search: %v", err)
|
||||
}
|
||||
got := <-reqs
|
||||
if got.URL.Path != "/12/api" || got.URL.Query().Get("t") != "search" || got.URL.Query().Get("apikey") != "k" {
|
||||
t.Errorf("request = %s, want the feed URL with t=search and the key", got.URL)
|
||||
}
|
||||
if cats := got.URL.Query()["cat"]; len(cats) != 3 || cats[0] != "3000" {
|
||||
t.Errorf("categories = %v, want the music categories", cats)
|
||||
}
|
||||
if len(results) != 2 {
|
||||
t.Fatalf("results = %d, want 2 (the untitled item dropped)", len(results))
|
||||
}
|
||||
first := results[0]
|
||||
if first.InfoHash != "aabbccddeeff00112233445566778899aabbccdd" || first.Seeders != 12 ||
|
||||
first.Category != 3040 || first.Indexer != "RED" || first.DownloadURL == "" || first.Source != "torznab-2" {
|
||||
t.Errorf("first = %+v", first)
|
||||
}
|
||||
if first.Published.IsZero() {
|
||||
t.Error("pubDate was not parsed")
|
||||
}
|
||||
// The magnet's hash wins over a GUID shaped like another hash.
|
||||
if results[1].Magnet == "" || results[1].InfoHash != "ffeeddccbbaa99887766554433221100ffeeddcc" {
|
||||
t.Errorf("second = %+v, want the magnet's hash", results[1])
|
||||
}
|
||||
}
|
||||
|
||||
func TestTorznabSearch_KeepsAQueryAlreadyInTheFeedURL(t *testing.T) {
|
||||
reqs := make(chan *http.Request, 1)
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
reqs <- r
|
||||
_, _ = w.Write([]byte(`<rss><channel></channel></rss>`))
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
c := &Torznab{Config: TorznabConfig{FeedURL: srv.URL + "/api?passkey=p"}}
|
||||
if _, err := c.Search(context.Background(), Query{Text: "x"}); err != nil {
|
||||
t.Fatalf("search: %v", err)
|
||||
}
|
||||
got := <-reqs
|
||||
if got.URL.Query().Get("passkey") != "p" || got.URL.Query().Get("q") != "x" {
|
||||
t.Errorf("request = %s, want the pasted query kept and ours added", got.URL)
|
||||
}
|
||||
}
|
||||
|
||||
// Newznab reports a refused key as HTTP 200 with an <error> document. Read
|
||||
// as a feed, that is an empty result set: search would quietly find nothing.
|
||||
func TestTorznabSearch_ReadsAnErrorDocument(t *testing.T) {
|
||||
for name, tc := range map[string]struct {
|
||||
doc string
|
||||
want error
|
||||
}{
|
||||
"a wrong key": {`<error code="100" description="Incorrect user credentials"/>`, ErrAuth},
|
||||
"anything else": {`<error code="300" description="Indexer down"/>`, ErrUnreachable},
|
||||
} {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
_, _ = w.Write([]byte(`<?xml version="1.0"?>` + tc.doc))
|
||||
}))
|
||||
c := &Torznab{Config: TorznabConfig{FeedURL: srv.URL + "/api"}}
|
||||
_, err := c.Search(context.Background(), Query{Text: "x"})
|
||||
srv.Close()
|
||||
if !errors.Is(err, tc.want) {
|
||||
t.Errorf("%s: err = %v, want %v", name, err, tc.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestTorznabSearch_A429IsRateLimitedWithItsRetryAfter(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
w.Header().Set("Retry-After", "90")
|
||||
w.WriteHeader(http.StatusTooManyRequests)
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
c := &Torznab{Config: TorznabConfig{FeedURL: srv.URL + "/api"}}
|
||||
_, err := c.Search(context.Background(), Query{Text: "x"})
|
||||
rl, ok := AsRateLimited(err)
|
||||
if !ok || rl.RetryAfter != 90*time.Second {
|
||||
t.Fatalf("err = %v, want RateLimited for 90s", err)
|
||||
}
|
||||
if !errors.Is(err, ErrUnreachable) {
|
||||
t.Error("a 429 should still read as unreachable to everything else")
|
||||
}
|
||||
}
|
||||
|
||||
func TestTorznabSearch_NoAddressIsUnconfigured(t *testing.T) {
|
||||
_, err := (&Torznab{}).Search(context.Background(), Query{Text: "x"})
|
||||
if !errors.Is(err, ErrUnconfigured) {
|
||||
t.Fatalf("err = %v, want ErrUnconfigured", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTorznabDownload_RefusesWhatIsNotATorrent(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
switch r.URL.Path {
|
||||
case "/login":
|
||||
_, _ = w.Write([]byte(`<html>please log in</html>`))
|
||||
case "/magnet":
|
||||
_, _ = w.Write([]byte("magnet:?xt=urn:btih:abc"))
|
||||
case "/redirect":
|
||||
http.Redirect(w, r, "magnet:?xt=urn:btih:def", http.StatusFound)
|
||||
default:
|
||||
_, _ = w.Write([]byte("d8:announce0:e"))
|
||||
}
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
c := &Torznab{Config: TorznabConfig{FeedURL: srv.URL + "/api"}}
|
||||
ctx := context.Background()
|
||||
if _, err := c.Download(ctx, Result{Title: "t", DownloadURL: srv.URL + "/login"}); !errors.Is(err, ErrUnreachable) {
|
||||
t.Errorf("an HTML page: err = %v, want ErrUnreachable", err)
|
||||
}
|
||||
for _, path := range []string{"/magnet", "/redirect"} {
|
||||
_, err := c.Download(ctx, Result{Title: "t", DownloadURL: srv.URL + path})
|
||||
if _, ok := AsMagnet(err); !ok {
|
||||
t.Errorf("%s: err = %v, want a MagnetLink", path, err)
|
||||
}
|
||||
}
|
||||
body, err := c.Download(ctx, Result{Title: "t", DownloadURL: srv.URL + "/file"})
|
||||
if err != nil || string(body) != "d8:announce0:e" {
|
||||
t.Errorf("a torrent: body %q err %v", body, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRetryAfter(t *testing.T) {
|
||||
now := time.Date(2026, 10, 10, 0, 0, 0, 0, time.UTC)
|
||||
for v, want := range map[string]time.Duration{
|
||||
"": 0,
|
||||
"30": 30 * time.Second,
|
||||
"soon": 0,
|
||||
"Sat, 10 Oct 2026 00:02:00 GMT": 2 * time.Minute,
|
||||
} {
|
||||
h := http.Header{}
|
||||
if v != "" {
|
||||
h.Set("Retry-After", v)
|
||||
}
|
||||
if got := RetryAfter(h, now); got != want {
|
||||
t.Errorf("RetryAfter(%q) = %v, want %v", v, got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user