feat: indexers and download clients you add and configure (M502 #5518)

Admin → Integrations gains "Indexers and download clients": two lists of connections the operator adds, Fabled Librarian's model. Nothing is seeded or copied from Lidarr. - Indexers: Prowlarr (its native /api/v1/search, which asks every indexer at once; Prowlarr has no aggregate Torznab feed) and any Torznab feed. - Download clients: qBittorrent (WebUI v2, with re-login on an expired session) and FabledExchange, one entry with a switch per job: downloads (/api/downloads), search (/api/search) and release details (/api/torrents/{infohash}). All three start on; its key is optional. - Each entry: enable, name, address, a write-only key or password, Test. A download client's Test lists its torrents and checks the downloads folder is visible and writable from Minstrel; an Exchange's Test reports each job switched on. - Download clients carry a path mapping (Minstrel's /downloads vs the client's own name for it); search sources carry searches per hour. Migration 0079 adds acquisition_services with its CHECKs. Ported from Fabled Librarian: internal/indexer (Torznab, 429 Retry-After, magnet redirects), internal/prowlarr, internal/exchange, internal/downloader, and internal/acquire's one builder per kind, which Test already uses so a passing Test vouches for the path a grab will take. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-09 20:34:51 -04:00
parent 5306a261c1
commit a2e1890d4c
35 changed files with 4966 additions and 2 deletions
@@ -0,0 +1,182 @@
+// Package exchange talks to FabledExchange's native API for the two jobs that
+// are not downloading (M502 #5518): searching its DHT index (/api/search) and
+// describing one torrent (/api/torrents/{infohash}). Downloading is
+// internal/downloader's. Ported from Fabled Librarian's internal/exchange.
+//
+// The native search rather than Exchange's Torznab feed, because it reports
+// whether a swarm was measured and how many files a torrent holds, which the
+// feed cannot say.
+package exchange
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"log/slog"
+	"net/http"
+	"strings"
+	"time"
+
+	"git.fabledsword.com/bvandeusen/minstrel/internal/indexer"
+)
+
+// ErrUnconfigured is an Exchange with no address.
+var ErrUnconfigured = errors.New("exchange is not configured")
+
+// ErrUnreachable is a call that could not be made or was not understood.
+var ErrUnreachable = errors.New("exchange unreachable")
+
+// ErrAuth is an API key Exchange refused, or a secured Exchange asked with
+// none.
+var ErrAuth = errors.New("exchange refused the API key")
+
+// ErrUnknown is a torrent Exchange has not crawled: a JSON 404, which is an
+// answer and not a fault.
+var ErrUnknown = errors.New("exchange has not crawled this torrent")
+
+// UserAgent names Minstrel in Exchange's logs.
+const UserAgent = "Minstrel (self-hosted music server; release search)"
+
+// DefaultTimeout is what an unset Timeout means. Exchange answers from its
+// own database.
+const DefaultTimeout = 20 * time.Second
+
+// Config is one Exchange.
+type Config struct {
+	BaseURL string
+	// APIKey is optional and sent only when set: an empty bearer makes a
+	// secured Exchange answer 401.
+	APIKey  string
+	Timeout time.Duration
+}
+
+// Client asks one Exchange.
+type Client struct {
+	Config Config
+	// HTTP is injectable for tests. Nil takes http.DefaultClient.
+	HTTP *http.Client
+	// Code is the acquisition_services row, stamped onto search results.
+	Code   string
+	Logger *slog.Logger
+}
+
+func (c *Client) log() *slog.Logger {
+	if c == nil || c.Logger == nil {
+		return slog.Default()
+	}
+	return c.Logger
+}
+
+// Detail is what Exchange knows about one torrent.
+type Detail struct {
+	InfoHash      string       `json:"infohash"`
+	Name          string       `json:"name"`
+	Size          int64        `json:"size"`
+	FilesCount    int          `json:"files_count"`
+	Seeders       int          `json:"seeders"`
+	SwarmKnown    bool         `json:"swarm_known"`
+	Leechers      int          `json:"leechers"`
+	LeechersKnown bool         `json:"leechers_known"`
+	ContentType   string       `json:"content_type"`
+	Files         []DetailFile `json:"files"`
+}
+
+// DetailFile is one file inside a torrent.
+type DetailFile struct {
+	Index int    `json:"index"`
+	Path  string `json:"path"`
+	Size  int64  `json:"size"`
+}
+
+// Torrent asks Exchange about one v1 infohash.
+func (c *Client) Torrent(ctx context.Context, infohash string) (Detail, error) {
+	if strings.TrimSpace(c.Config.BaseURL) == "" {
+		return Detail{}, ErrUnconfigured
+	}
+	hash := indexer.NormalizeHash(infohash)
+	if len(hash) != 40 || !indexer.IsInfoHash(hash) {
+		return Detail{}, fmt.Errorf("%w: %q is not a 40-character infohash", ErrUnreachable, infohash)
+	}
+	body, err := c.get(ctx, "/api/torrents/"+hash)
+	if err != nil {
+		return Detail{}, err
+	}
+	var out Detail
+	if err := json.Unmarshal(body, &out); err != nil {
+		return Detail{}, fmt.Errorf("%w: parse torrent detail: %v", ErrUnreachable, err)
+	}
+	if out.InfoHash == "" {
+		return Detail{}, ErrUnknown
+	}
+	return out, nil
+}
+
+func (c *Client) get(ctx context.Context, path string) ([]byte, error) {
+	timeout := c.Config.Timeout
+	if timeout <= 0 {
+		timeout = DefaultTimeout
+	}
+	ctx, cancel := context.WithTimeout(ctx, timeout)
+	defer cancel()
+
+	target := strings.TrimRight(c.Config.BaseURL, "/") + path
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, target, nil)
+	if err != nil {
+		return nil, fmt.Errorf("%w: %v", ErrUnreachable, err)
+	}
+	req.Header.Set("User-Agent", UserAgent)
+	req.Header.Set("Accept", "application/json")
+	if key := strings.TrimSpace(c.Config.APIKey); key != "" {
+		req.Header.Set("Authorization", "Bearer "+key)
+	}
+	client := c.HTTP
+	if client == nil {
+		client = http.DefaultClient
+	}
+	res, err := client.Do(req)
+	if err != nil {
+		return nil, fmt.Errorf("%w: %v", ErrUnreachable, err)
+	}
+	defer func() { _ = res.Body.Close() }()
+	body, err := io.ReadAll(io.LimitReader(res.Body, 1<<20))
+	if err != nil {
+		return nil, fmt.Errorf("%w: read answer: %v", ErrUnreachable, err)
+	}
+	switch {
+	case res.StatusCode == http.StatusNotFound:
+		// A 404 that is not JSON came from something else at this address,
+		// which must not look like a healthy Exchange that lacks a torrent.
+		if !looksLikeJSON(body) {
+			return nil, fmt.Errorf("%w: something at this address answered 404, "+
+				"but it was not FabledExchange", ErrUnreachable)
+		}
+		return nil, ErrUnknown
+	case res.StatusCode == http.StatusUnauthorized || res.StatusCode == http.StatusForbidden:
+		return nil, fmt.Errorf("%w: %s", ErrAuth,
+			message(body, "set this FabledExchange's API key in Minstrel's settings"))
+	case res.StatusCode == http.StatusServiceUnavailable:
+		return nil, fmt.Errorf("%w: %s", ErrUnreachable, message(body, "no index is configured"))
+	case res.StatusCode < 200 || res.StatusCode >= 300:
+		return nil, fmt.Errorf("%w: %s", ErrUnreachable,
+			message(body, fmt.Sprintf("returned %d", res.StatusCode)))
+	}
+	return body, nil
+}
+
+// message is Exchange's own {"error": "..."} text, or fallback.
+func message(body []byte, fallback string) string {
+	var env struct {
+		Error string `json:"error"`
+	}
+	if err := json.Unmarshal(body, &env); err == nil && strings.TrimSpace(env.Error) != "" {
+		return strings.TrimSpace(env.Error)
+	}
+	return fallback
+}
+
+func looksLikeJSON(body []byte) bool {
+	t := strings.TrimSpace(string(body))
+	return strings.HasPrefix(t, "{") || strings.HasPrefix(t, "[")
+}
@@ -0,0 +1,108 @@
+package exchange
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"testing"
+
+	"git.fabledsword.com/bvandeusen/minstrel/internal/indexer"
+)
+
+func TestTorrent_SendsTheKeyOnlyWhenSet(t *testing.T) {
+	auth := make(chan string, 2)
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		auth <- r.Header.Get("Authorization")
+		_, _ = w.Write([]byte(`{"infohash": "` + strings.Repeat("a", 40) + `", "files_count": 12}`))
+	}))
+	defer srv.Close()
+
+	for _, key := range []string{"", "k"} {
+		c := &Client{Config: Config{BaseURL: srv.URL, APIKey: key}}
+		d, err := c.Torrent(context.Background(), strings.Repeat("A", 40))
+		if err != nil || d.FilesCount != 12 {
+			t.Fatalf("key %q: detail %+v err %v", key, d, err)
+		}
+	}
+	if without, with := <-auth, <-auth; without != "" || with != "Bearer k" {
+		t.Errorf("Authorization = %q then %q, want none without a key and a bearer with one", without, with)
+	}
+}
+
+func TestTorrent_TellsAnUncrawledTorrentFromAnotherService(t *testing.T) {
+	for name, tc := range map[string]struct {
+		status int
+		body   string
+		want   error
+	}{
+		"Exchange has not crawled it": {http.StatusNotFound, `{"error": "unknown torrent"}`, ErrUnknown},
+		"not FabledExchange":          {http.StatusNotFound, `<html>404</html>`, ErrUnreachable},
+		"a refused key":               {http.StatusUnauthorized, `{"error": "bad key"}`, ErrAuth},
+	} {
+		srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+			w.WriteHeader(tc.status)
+			_, _ = w.Write([]byte(tc.body))
+		}))
+		_, err := (&Client{Config: Config{BaseURL: srv.URL}}).Torrent(context.Background(), strings.Repeat("0", 40))
+		srv.Close()
+		if !errors.Is(err, tc.want) {
+			t.Errorf("%s: err = %v, want %v", name, err, tc.want)
+		}
+	}
+}
+
+func TestSearch_PagesAndKeepsMeasuredSwarmsOnly(t *testing.T) {
+	queries := make(chan string, 4)
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		queries <- r.URL.RawQuery
+		offset := r.URL.Query().Get("offset")
+		var rows []string
+		if offset == "0" {
+			for i := 0; i < SearchLimit; i++ {
+				rows = append(rows, fmt.Sprintf(`{"infohash": "%040x", "name": "r%d", "content_type": "music", "seeders": 5, "swarm_known": %t}`, i, i, i == 0))
+			}
+		} else {
+			// A repeat of page one's first hash, which must not be listed twice.
+			rows = append(rows, fmt.Sprintf(`{"infohash": "%040x", "name": "again"}`, 0))
+		}
+		_, _ = fmt.Fprintf(w, `{"total": %d, "results": [%s]}`, SearchLimit+1, strings.Join(rows, ","))
+	}))
+	defer srv.Close()
+
+	c := &Client{Code: "exchange", Config: Config{BaseURL: srv.URL}}
+	results, err := c.Search(context.Background(), indexer.Query{Text: "x", Categories: indexer.MusicCategories})
+	if err != nil {
+		t.Fatalf("search: %v", err)
+	}
+	close(queries)
+	var asked []string
+	for q := range queries {
+		asked = append(asked, q)
+	}
+	if len(asked) != 2 || !strings.Contains(asked[0], "type=music") {
+		t.Errorf("queries = %v, want two pages asking for music", asked)
+	}
+	if len(results) != SearchLimit {
+		t.Errorf("results = %d, want %d (the repeat dropped)", len(results), SearchLimit)
+	}
+	if r := results[0]; !r.SwarmKnown || r.Seeders != 5 || r.Category != 3000 || r.Source != "exchange" || !strings.HasPrefix(r.Magnet, "magnet:") {
+		t.Errorf("first = %+v", r)
+	}
+	if r := results[1]; r.SwarmKnown || r.Seeders != 0 {
+		t.Errorf("an unmeasured swarm reported seeders: %+v", r)
+	}
+}
+
+func TestSearch_ARefusedKeyReadsAsAuth(t *testing.T) {
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+		w.WriteHeader(http.StatusForbidden)
+	}))
+	defer srv.Close()
+	_, err := (&Client{Config: Config{BaseURL: srv.URL}}).Search(context.Background(), indexer.Query{Text: "x"})
+	if !errors.Is(err, indexer.ErrAuth) {
+		t.Fatalf("err = %v, want indexer.ErrAuth", err)
+	}
+}
@@ -0,0 +1,169 @@
+package exchange
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"net/url"
+	"strings"
+
+	"git.fabledsword.com/bvandeusen/minstrel/internal/indexer"
+)
+
+// SearchLimit is one page of /api/search; SearchDepth is the most read across
+// pages. A query matching more than SearchDepth is too broad to judge anyway.
+const (
+	SearchLimit = 100
+	SearchDepth = 300
+)
+
+type searchResult struct {
+	InfoHash      string `json:"infohash"`
+	Name          string `json:"name"`
+	Size          int64  `json:"size"`
+	FilesCount    int    `json:"files_count"`
+	ContentType   string `json:"content_type"`
+	Seeders       int    `json:"seeders"`
+	SwarmKnown    bool   `json:"swarm_known"`
+	Leechers      int    `json:"leechers"`
+	LeechersKnown bool   `json:"leechers_known"`
+}
+
+type searchPage struct {
+	Total   int            `json:"total"`
+	Results []searchResult `json:"results"`
+}
+
+// Search asks Exchange's DHT index. Every result is a magnet, so there is no
+// .torrent to fetch through it. A later page that fails keeps the earlier
+// ones; a first page that fails is the search failing.
+func (c *Client) Search(ctx context.Context, q indexer.Query) ([]indexer.Result, error) {
+	if strings.TrimSpace(c.Config.BaseURL) == "" {
+		return nil, indexer.ErrUnconfigured
+	}
+	params := url.Values{"q": {q.Text}, "limit": {fmt.Sprint(SearchLimit)}}
+	if t := contentType(q.Categories); t != "" {
+		params.Set("type", t)
+	}
+	var (
+		out  []indexer.Result
+		seen = map[string]bool{}
+	)
+	for offset := 0; offset < SearchDepth; offset += SearchLimit {
+		params.Set("offset", fmt.Sprint(offset))
+		page, err := c.searchPage(ctx, params)
+		if err != nil {
+			if offset == 0 {
+				return nil, err
+			}
+			c.log().Warn("exchange: a later page of a search could not be read; keeping the pages before it",
+				"q", q.Text, "offset", offset, "err", err)
+			break
+		}
+		for _, r := range page.Results {
+			res, ok := c.result(r)
+			if !ok || seen[res.InfoHash] {
+				continue
+			}
+			seen[res.InfoHash] = true
+			out = append(out, res)
+		}
+		if len(page.Results) < SearchLimit || offset+SearchLimit >= page.Total {
+			break
+		}
+	}
+	return out, nil
+}
+
+func (c *Client) searchPage(ctx context.Context, params url.Values) (searchPage, error) {
+	var page searchPage
+	body, err := c.get(ctx, "/api/search?"+params.Encode())
+	if err != nil {
+		return page, searchError(err)
+	}
+	if err := json.Unmarshal(body, &page); err != nil {
+		return page, fmt.Errorf("%w: parse search results: %v", indexer.ErrUnreachable, err)
+	}
+	return page, nil
+}
+
+func (c *Client) result(r searchResult) (indexer.Result, bool) {
+	hash := indexer.NormalizeHash(r.InfoHash)
+	if hash == "" || strings.TrimSpace(r.Name) == "" {
+		return indexer.Result{}, false
+	}
+	res := indexer.Result{
+		Title:      r.Name,
+		InfoHash:   hash,
+		Magnet:     magnet(hash, r.Name),
+		SizeBytes:  r.Size,
+		Category:   category(r.ContentType),
+		Source:     c.Code,
+		FilesCount: r.FilesCount,
+		SwarmKnown: r.SwarmKnown,
+	}
+	// Seeders only when measured: an unmeasured count is not health.
+	if r.SwarmKnown {
+		res.Seeders, res.Peers = r.Seeders, r.Seeders
+		if r.LeechersKnown {
+			res.Peers = r.Seeders + r.Leechers
+		}
+	}
+	return res, true
+}
+
+// searchError puts Exchange's errors in the vocabulary every search source
+// shares, so a refused key reads the same whichever source refused it.
+func searchError(err error) error {
+	switch {
+	case errors.Is(err, ErrAuth):
+		return fmt.Errorf("%w: %v", indexer.ErrAuth, err)
+	case errors.Is(err, ErrUnknown):
+		return fmt.Errorf("%w: this FabledExchange has no search API", indexer.ErrUnreachable)
+	}
+	return fmt.Errorf("%w: %v", indexer.ErrUnreachable, err)
+}
+
+// contentType narrows Exchange's search to music when the query asks only
+// for music categories (3000s, not 3030 audiobooks).
+func contentType(cats []int) string {
+	music := false
+	for _, c := range cats {
+		if c < 3000 || c >= 4000 || c == 3030 {
+			return ""
+		}
+		music = true
+	}
+	if music {
+		return "music"
+	}
+	return ""
+}
+
+func category(contentType string) int {
+	switch contentType {
+	case "music":
+		return 3000
+	case "audiobook":
+		return 3030
+	case "ebook":
+		return 7020
+	case "comic":
+		return 7030
+	case "movie":
+		return 2000
+	case "tv":
+		return 5000
+	}
+	return 8000
+}
+
+// magnet is a trackerless magnet for a hash Exchange found on the DHT.
+func magnet(hash, name string) string {
+	m := "magnet:?xt=urn:btih:" + hash
+	if name != "" {
+		m += "&dn=" + url.QueryEscape(name)
+	}
+	return m
+}