feat: indexers and download clients you add and configure (M502 #5518)
release / govulncheck (push) Successful in 34s
release / web (push) Successful in 1m13s
release / go (push) Successful in 2m12s
release / integration (push) Successful in 4m6s
release / Build signed APK (releases and dev) (push) Successful in 8m41s
release / android (push) Successful in 9m47s
release / Attach APK to the Release (tag releases only) (push) Skipped
release / Build + push container image (push) Successful in 59s
release / Verify release artifacts (tag releases only) (push) Skipped
release / govulncheck (push) Successful in 34s
release / web (push) Successful in 1m13s
release / go (push) Successful in 2m12s
release / integration (push) Successful in 4m6s
release / Build signed APK (releases and dev) (push) Successful in 8m41s
release / android (push) Successful in 9m47s
release / Attach APK to the Release (tag releases only) (push) Skipped
release / Build + push container image (push) Successful in 59s
release / Verify release artifacts (tag releases only) (push) Skipped
Admin → Integrations gains "Indexers and download clients": two lists of
connections the operator adds, Fabled Librarian's model. Nothing is seeded
or copied from Lidarr.
- Indexers: Prowlarr (its native /api/v1/search, which asks every indexer
at once; Prowlarr has no aggregate Torznab feed) and any Torznab feed.
- Download clients: qBittorrent (WebUI v2, with re-login on an expired
session) and FabledExchange, one entry with a switch per job:
downloads (/api/downloads), search (/api/search) and release details
(/api/torrents/{infohash}). All three start on; its key is optional.
- Each entry: enable, name, address, a write-only key or password, Test.
A download client's Test lists its torrents and checks the downloads
folder is visible and writable from Minstrel; an Exchange's Test
reports each job switched on.
- Download clients carry a path mapping (Minstrel's /downloads vs the
client's own name for it); search sources carry searches per hour.
Migration 0079 adds acquisition_services with its CHECKs. Ported from
Fabled Librarian: internal/indexer (Torznab, 429 Retry-After, magnet
redirects), internal/prowlarr, internal/exchange, internal/downloader,
and internal/acquire's one builder per kind, which Test already uses
so a passing Test vouches for the path a grab will take.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,157 @@
|
||||
// Package indexer is what every search source answers with, and the Torznab
|
||||
// client (M502 #5518). Ported from Fabled Librarian's internal/indexer.
|
||||
//
|
||||
// A client here holds ONE endpoint and knows nothing about the others. The
|
||||
// fan-out across sources, the merge and what a source being down means belong
|
||||
// to internal/acquire, because they are decisions about a search rather than
|
||||
// about one source.
|
||||
//
|
||||
// Prowlarr is not served from here: it has no aggregate Torznab feed (`all`
|
||||
// 404s like any id it does not have, measured by Fabled Librarian, #4304), so
|
||||
// asking it everything at once means its native API, which internal/prowlarr
|
||||
// speaks.
|
||||
package indexer
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// ErrUnconfigured is a source with no address (or, for Prowlarr, no key): not
|
||||
// a failure, an instance that does not search there.
|
||||
var ErrUnconfigured = errors.New("indexer is not configured")
|
||||
|
||||
// ErrUnreachable is a search that could not be made: no network, a timeout, a
|
||||
// malformed answer, a status outside 2xx.
|
||||
var ErrUnreachable = errors.New("indexer unreachable")
|
||||
|
||||
// ErrAuth is a key the source refused. Separate from ErrUnreachable because
|
||||
// it needs a different sentence and a different action: a refused key is a
|
||||
// field in Settings, an outage is something to wait out.
|
||||
var ErrAuth = errors.New("the API key was refused")
|
||||
|
||||
// UserAgent names Minstrel in the operator's own Prowlarr and indexer logs.
|
||||
const UserAgent = "Minstrel (self-hosted music server; release search)"
|
||||
|
||||
// MusicCategories are the Newznab categories a music search asks for: 3000
|
||||
// Audio, with its MP3 (3010) and Lossless (3040) children alongside, because
|
||||
// an indexer that files music under a child answers only for the child.
|
||||
// 3030 (Audiobook) is deliberately absent.
|
||||
var MusicCategories = []int{3000, 3010, 3040}
|
||||
|
||||
// Query is one search: the words, and which categories to ask in. Empty
|
||||
// Categories asks every category.
|
||||
type Query struct {
|
||||
Text string
|
||||
Categories []int
|
||||
}
|
||||
|
||||
// Result is one search result, as the source described it.
|
||||
type Result struct {
|
||||
Title string
|
||||
// InfoHash is the torrent's identity, lowercased; empty when the source
|
||||
// gave none.
|
||||
InfoHash string
|
||||
// Magnet is a magnet URI when the source published one.
|
||||
Magnet string
|
||||
// DownloadURL is a .torrent URL when it did not. It must be fetched
|
||||
// through the source that offered it (it holds the tracker credentials),
|
||||
// never handed to the download client.
|
||||
DownloadURL string
|
||||
SizeBytes int64
|
||||
Category int
|
||||
// Indexer is the tracker BEHIND the source, as the source named it.
|
||||
Indexer string
|
||||
// Source is the acquisition_services code that was asked: where a
|
||||
// .torrent fetch must be routed back to.
|
||||
Source string
|
||||
Published time.Time
|
||||
// Seeders and Peers as reported. A feed that has not sampled a swarm
|
||||
// reports a nominal 1, so these are not health unless SwarmKnown.
|
||||
Seeders int
|
||||
Peers int
|
||||
// SwarmKnown says Seeders was measured, which only FabledExchange's own
|
||||
// search can say.
|
||||
SwarmKnown bool
|
||||
// FilesCount is how many files the torrent holds, 0 when not said.
|
||||
FilesCount int
|
||||
}
|
||||
|
||||
// Send sends req through c, stopping at a redirect to a magnet (returned as a
|
||||
// *MagnetLink), and reads at most 8 MB of the answer. A call that never got
|
||||
// an answer is ErrUnreachable; the status is the caller's to read.
|
||||
func Send(c *http.Client, req *http.Request) (*http.Response, []byte, error) {
|
||||
resp, err := StopAtMagnet(c).Do(req)
|
||||
if m, ok := AsMagnet(err); ok {
|
||||
return nil, nil, &MagnetLink{URI: m}
|
||||
}
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("%w: %v", ErrUnreachable, err)
|
||||
}
|
||||
defer func() { _ = resp.Body.Close() }()
|
||||
body, err := io.ReadAll(io.LimitReader(resp.Body, 8<<20))
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("%w: read body: %v", ErrUnreachable, err)
|
||||
}
|
||||
return resp, body, nil
|
||||
}
|
||||
|
||||
// StatusError is what a non-2xx status means for a search source: a 429 is
|
||||
// RateLimited naming who, anything else ErrUnreachable. Nil for a 2xx.
|
||||
func StatusError(resp *http.Response, who string) error {
|
||||
switch {
|
||||
case resp.StatusCode == http.StatusTooManyRequests:
|
||||
return &RateLimited{Who: who, RetryAfter: RetryAfter(resp.Header, time.Now())}
|
||||
case resp.StatusCode < 200 || resp.StatusCode >= 300:
|
||||
return fmt.Errorf("%w: HTTP %d", ErrUnreachable, resp.StatusCode)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// NormalizeHash is a torrent's identity in its one canonical spelling. Every
|
||||
// comparison of infohashes goes through it, so a hash differing only in case
|
||||
// is never two releases.
|
||||
func NormalizeHash(s string) string {
|
||||
return strings.ToLower(strings.TrimSpace(s))
|
||||
}
|
||||
|
||||
// IsInfoHash says whether s is shaped like a v1 (40) or v2 (64) hex infohash.
|
||||
func IsInfoHash(s string) bool {
|
||||
s = NormalizeHash(s)
|
||||
if len(s) != 40 && len(s) != 64 {
|
||||
return false
|
||||
}
|
||||
for _, r := range s {
|
||||
if (r < '0' || r > '9') && (r < 'a' || r > 'f') {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// MagnetHash pulls the infohash out of a magnet link, or returns "". It does
|
||||
// not validate it.
|
||||
func MagnetHash(magnet string) string {
|
||||
const prefix = "urn:btih:"
|
||||
at := strings.Index(magnet, prefix)
|
||||
if at < 0 {
|
||||
return ""
|
||||
}
|
||||
hash := magnet[at+len(prefix):]
|
||||
if cut := strings.IndexAny(hash, "&/?"); cut >= 0 {
|
||||
hash = hash[:cut]
|
||||
}
|
||||
return strings.ToLower(hash)
|
||||
}
|
||||
|
||||
// FirstLine is an error's first line, for a sentence on the settings page.
|
||||
func FirstLine(s string) string {
|
||||
if i := strings.IndexByte(s, '\n'); i >= 0 {
|
||||
return s[:i]
|
||||
}
|
||||
return s
|
||||
}
|
||||
Reference in New Issue
Block a user