feat: indexers and download clients you add and configure (M502 #5518)
release / govulncheck (push) Successful in 34s
release / web (push) Successful in 1m13s
release / go (push) Successful in 2m12s
release / integration (push) Successful in 4m6s
release / Build signed APK (releases and dev) (push) Successful in 8m41s
release / android (push) Successful in 9m47s
release / Attach APK to the Release (tag releases only) (push) Skipped
release / Build + push container image (push) Successful in 59s
release / Verify release artifacts (tag releases only) (push) Skipped
release / govulncheck (push) Successful in 34s
release / web (push) Successful in 1m13s
release / go (push) Successful in 2m12s
release / integration (push) Successful in 4m6s
release / Build signed APK (releases and dev) (push) Successful in 8m41s
release / android (push) Successful in 9m47s
release / Attach APK to the Release (tag releases only) (push) Skipped
release / Build + push container image (push) Successful in 59s
release / Verify release artifacts (tag releases only) (push) Skipped
Admin → Integrations gains "Indexers and download clients": two lists of
connections the operator adds, Fabled Librarian's model. Nothing is seeded
or copied from Lidarr.
- Indexers: Prowlarr (its native /api/v1/search, which asks every indexer
at once; Prowlarr has no aggregate Torznab feed) and any Torznab feed.
- Download clients: qBittorrent (WebUI v2, with re-login on an expired
session) and FabledExchange, one entry with a switch per job:
downloads (/api/downloads), search (/api/search) and release details
(/api/torrents/{infohash}). All three start on; its key is optional.
- Each entry: enable, name, address, a write-only key or password, Test.
A download client's Test lists its torrents and checks the downloads
folder is visible and writable from Minstrel; an Exchange's Test
reports each job switched on.
- Download clients carry a path mapping (Minstrel's /downloads vs the
client's own name for it); search sources carry searches per hour.
Migration 0079 adds acquisition_services with its CHECKs. Ported from
Fabled Librarian: internal/indexer (Torznab, 429 Retry-After, magnet
redirects), internal/prowlarr, internal/exchange, internal/downloader,
and internal/acquire's one builder per kind, which Test already uses
so a passing Test vouches for the path a grab will take.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,169 @@
|
||||
package exchange
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/url"
|
||||
"strings"
|
||||
|
||||
"git.fabledsword.com/bvandeusen/minstrel/internal/indexer"
|
||||
)
|
||||
|
||||
// SearchLimit is one page of /api/search; SearchDepth is the most read across
|
||||
// pages. A query matching more than SearchDepth is too broad to judge anyway.
|
||||
const (
|
||||
SearchLimit = 100
|
||||
SearchDepth = 300
|
||||
)
|
||||
|
||||
type searchResult struct {
|
||||
InfoHash string `json:"infohash"`
|
||||
Name string `json:"name"`
|
||||
Size int64 `json:"size"`
|
||||
FilesCount int `json:"files_count"`
|
||||
ContentType string `json:"content_type"`
|
||||
Seeders int `json:"seeders"`
|
||||
SwarmKnown bool `json:"swarm_known"`
|
||||
Leechers int `json:"leechers"`
|
||||
LeechersKnown bool `json:"leechers_known"`
|
||||
}
|
||||
|
||||
type searchPage struct {
|
||||
Total int `json:"total"`
|
||||
Results []searchResult `json:"results"`
|
||||
}
|
||||
|
||||
// Search asks Exchange's DHT index. Every result is a magnet, so there is no
|
||||
// .torrent to fetch through it. A later page that fails keeps the earlier
|
||||
// ones; a first page that fails is the search failing.
|
||||
func (c *Client) Search(ctx context.Context, q indexer.Query) ([]indexer.Result, error) {
|
||||
if strings.TrimSpace(c.Config.BaseURL) == "" {
|
||||
return nil, indexer.ErrUnconfigured
|
||||
}
|
||||
params := url.Values{"q": {q.Text}, "limit": {fmt.Sprint(SearchLimit)}}
|
||||
if t := contentType(q.Categories); t != "" {
|
||||
params.Set("type", t)
|
||||
}
|
||||
var (
|
||||
out []indexer.Result
|
||||
seen = map[string]bool{}
|
||||
)
|
||||
for offset := 0; offset < SearchDepth; offset += SearchLimit {
|
||||
params.Set("offset", fmt.Sprint(offset))
|
||||
page, err := c.searchPage(ctx, params)
|
||||
if err != nil {
|
||||
if offset == 0 {
|
||||
return nil, err
|
||||
}
|
||||
c.log().Warn("exchange: a later page of a search could not be read; keeping the pages before it",
|
||||
"q", q.Text, "offset", offset, "err", err)
|
||||
break
|
||||
}
|
||||
for _, r := range page.Results {
|
||||
res, ok := c.result(r)
|
||||
if !ok || seen[res.InfoHash] {
|
||||
continue
|
||||
}
|
||||
seen[res.InfoHash] = true
|
||||
out = append(out, res)
|
||||
}
|
||||
if len(page.Results) < SearchLimit || offset+SearchLimit >= page.Total {
|
||||
break
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (c *Client) searchPage(ctx context.Context, params url.Values) (searchPage, error) {
|
||||
var page searchPage
|
||||
body, err := c.get(ctx, "/api/search?"+params.Encode())
|
||||
if err != nil {
|
||||
return page, searchError(err)
|
||||
}
|
||||
if err := json.Unmarshal(body, &page); err != nil {
|
||||
return page, fmt.Errorf("%w: parse search results: %v", indexer.ErrUnreachable, err)
|
||||
}
|
||||
return page, nil
|
||||
}
|
||||
|
||||
func (c *Client) result(r searchResult) (indexer.Result, bool) {
|
||||
hash := indexer.NormalizeHash(r.InfoHash)
|
||||
if hash == "" || strings.TrimSpace(r.Name) == "" {
|
||||
return indexer.Result{}, false
|
||||
}
|
||||
res := indexer.Result{
|
||||
Title: r.Name,
|
||||
InfoHash: hash,
|
||||
Magnet: magnet(hash, r.Name),
|
||||
SizeBytes: r.Size,
|
||||
Category: category(r.ContentType),
|
||||
Source: c.Code,
|
||||
FilesCount: r.FilesCount,
|
||||
SwarmKnown: r.SwarmKnown,
|
||||
}
|
||||
// Seeders only when measured: an unmeasured count is not health.
|
||||
if r.SwarmKnown {
|
||||
res.Seeders, res.Peers = r.Seeders, r.Seeders
|
||||
if r.LeechersKnown {
|
||||
res.Peers = r.Seeders + r.Leechers
|
||||
}
|
||||
}
|
||||
return res, true
|
||||
}
|
||||
|
||||
// searchError puts Exchange's errors in the vocabulary every search source
|
||||
// shares, so a refused key reads the same whichever source refused it.
|
||||
func searchError(err error) error {
|
||||
switch {
|
||||
case errors.Is(err, ErrAuth):
|
||||
return fmt.Errorf("%w: %v", indexer.ErrAuth, err)
|
||||
case errors.Is(err, ErrUnknown):
|
||||
return fmt.Errorf("%w: this FabledExchange has no search API", indexer.ErrUnreachable)
|
||||
}
|
||||
return fmt.Errorf("%w: %v", indexer.ErrUnreachable, err)
|
||||
}
|
||||
|
||||
// contentType narrows Exchange's search to music when the query asks only
|
||||
// for music categories (3000s, not 3030 audiobooks).
|
||||
func contentType(cats []int) string {
|
||||
music := false
|
||||
for _, c := range cats {
|
||||
if c < 3000 || c >= 4000 || c == 3030 {
|
||||
return ""
|
||||
}
|
||||
music = true
|
||||
}
|
||||
if music {
|
||||
return "music"
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func category(contentType string) int {
|
||||
switch contentType {
|
||||
case "music":
|
||||
return 3000
|
||||
case "audiobook":
|
||||
return 3030
|
||||
case "ebook":
|
||||
return 7020
|
||||
case "comic":
|
||||
return 7030
|
||||
case "movie":
|
||||
return 2000
|
||||
case "tv":
|
||||
return 5000
|
||||
}
|
||||
return 8000
|
||||
}
|
||||
|
||||
// magnet is a trackerless magnet for a hash Exchange found on the DHT.
|
||||
func magnet(hash, name string) string {
|
||||
m := "magnet:?xt=urn:btih:" + hash
|
||||
if name != "" {
|
||||
m += "&dn=" + url.QueryEscape(name)
|
||||
}
|
||||
return m
|
||||
}
|
||||
Reference in New Issue
Block a user