From a2e1890d4c4c2fa48426c4285864ad91a8bcafcd Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Fri, 9 Oct 2026 20:34:51 -0400 Subject: [PATCH] feat: indexers and download clients you add and configure (M502 #5518) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Admin → Integrations gains "Indexers and download clients": two lists of connections the operator adds, Fabled Librarian's model. Nothing is seeded or copied from Lidarr. - Indexers: Prowlarr (its native /api/v1/search, which asks every indexer at once; Prowlarr has no aggregate Torznab feed) and any Torznab feed. - Download clients: qBittorrent (WebUI v2, with re-login on an expired session) and FabledExchange, one entry with a switch per job: downloads (/api/downloads), search (/api/search) and release details (/api/torrents/{infohash}). All three start on; its key is optional. - Each entry: enable, name, address, a write-only key or password, Test. A download client's Test lists its torrents and checks the downloads folder is visible and writable from Minstrel; an Exchange's Test reports each job switched on. - Download clients carry a path mapping (Minstrel's /downloads vs the client's own name for it); search sources carry searches per hour. Migration 0079 adds acquisition_services with its CHECKs. Ported from Fabled Librarian: internal/indexer (Torznab, 429 Retry-After, magnet redirects), internal/prowlarr, internal/exchange, internal/downloader, and internal/acquire's one builder per kind, which Test already uses so a passing Test vouches for the path a grab will take. Co-Authored-By: Claude Opus 5.5 --- internal/acquire/acquire_test.go | 180 ++++++++ internal/acquire/settings.go | 170 ++++++++ internal/acquire/store.go | 191 +++++++++ internal/acquire/store_test.go | 108 +++++ internal/acquire/test.go | 134 ++++++ internal/api/admin_acquisition.go | 209 ++++++++++ internal/api/api.go | 13 + internal/db/dbq/acquisition.sql.go | 362 ++++++++++++++++ internal/db/dbq/models.go | 22 + .../0079_acquisition_services.down.sql | 1 + .../0079_acquisition_services.up.sql | 63 +++ internal/db/queries/acquisition.sql | 76 ++++ internal/dbtest/reset.go | 1 + internal/downloader/downloader_test.go | 156 +++++++ internal/downloader/driver.go | 198 +++++++++ internal/downloader/exchange.go | 340 +++++++++++++++ internal/downloader/pathmap.go | 107 +++++ internal/downloader/qbittorrent.go | 367 ++++++++++++++++ internal/exchange/exchange.go | 182 ++++++++ internal/exchange/exchange_test.go | 108 +++++ internal/exchange/search.go | 169 ++++++++ internal/indexer/indexer.go | 157 +++++++ internal/indexer/magnet.go | 58 +++ internal/indexer/ratelimit.go | 50 +++ internal/indexer/torznab.go | 254 +++++++++++ internal/indexer/torznab_test.go | 188 +++++++++ internal/prowlarr/prowlarr.go | 177 ++++++++ internal/prowlarr/prowlarr_test.go | 78 ++++ web/src/lib/api/admin.ts | 79 ++++ web/src/lib/components/AcquisitionCard.svelte | 196 +++++++++ .../lib/components/AcquisitionCard.test.ts | 168 ++++++++ .../components/AcquisitionServiceRow.svelte | 394 ++++++++++++++++++ web/src/lib/styles/error-copy.json | 4 +- .../routes/admin/integrations/+page.svelte | 3 + .../admin/integrations/integrations.test.ts | 5 +- 35 files changed, 4966 insertions(+), 2 deletions(-) create mode 100644 internal/acquire/acquire_test.go create mode 100644 internal/acquire/settings.go create mode 100644 internal/acquire/store.go create mode 100644 internal/acquire/store_test.go create mode 100644 internal/acquire/test.go create mode 100644 internal/api/admin_acquisition.go create mode 100644 internal/db/dbq/acquisition.sql.go create mode 100644 internal/db/migrations/0079_acquisition_services.down.sql create mode 100644 internal/db/migrations/0079_acquisition_services.up.sql create mode 100644 internal/db/queries/acquisition.sql create mode 100644 internal/downloader/downloader_test.go create mode 100644 internal/downloader/driver.go create mode 100644 internal/downloader/exchange.go create mode 100644 internal/downloader/pathmap.go create mode 100644 internal/downloader/qbittorrent.go create mode 100644 internal/exchange/exchange.go create mode 100644 internal/exchange/exchange_test.go create mode 100644 internal/exchange/search.go create mode 100644 internal/indexer/indexer.go create mode 100644 internal/indexer/magnet.go create mode 100644 internal/indexer/ratelimit.go create mode 100644 internal/indexer/torznab.go create mode 100644 internal/indexer/torznab_test.go create mode 100644 internal/prowlarr/prowlarr.go create mode 100644 internal/prowlarr/prowlarr_test.go create mode 100644 web/src/lib/components/AcquisitionCard.svelte create mode 100644 web/src/lib/components/AcquisitionCard.test.ts create mode 100644 web/src/lib/components/AcquisitionServiceRow.svelte diff --git a/internal/acquire/acquire_test.go b/internal/acquire/acquire_test.go new file mode 100644 index 00000000..ac0ebf16 --- /dev/null +++ b/internal/acquire/acquire_test.go @@ -0,0 +1,180 @@ +package acquire + +import ( + "context" + "errors" + "net/http" + "net/http/httptest" + "path/filepath" + "strings" + "testing" + + "git.fabledsword.com/bvandeusen/minstrel/internal/db/dbq" + "git.fabledsword.com/bvandeusen/minstrel/internal/downloader" + "git.fabledsword.com/bvandeusen/minstrel/internal/exchange" + "git.fabledsword.com/bvandeusen/minstrel/internal/indexer" + "git.fabledsword.com/bvandeusen/minstrel/internal/prowlarr" +) + +func validUpdate() Update { + return Update{Label: "Prowlarr", BaseURL: "http://prowlarr:9696", TimeoutMs: 30000, LocalPath: "/downloads", SearchesPerHour: 120} +} + +func TestUpdateValidate(t *testing.T) { + if err := validUpdate().validate(); err != nil { + t.Fatalf("a valid form: %v", err) + } + for name, tc := range map[string]struct { + edit func(*Update) + field string + }{ + "no name": {func(u *Update) { u.Label = "" }, "label"}, + "switched on with no address": {func(u *Update) { u.Enabled, u.BaseURL = true, "" }, "base_url"}, + "an address with no scheme": {func(u *Update) { u.BaseURL = "prowlarr:9696" }, "base_url"}, + "a timeout too short": {func(u *Update) { u.TimeoutMs = 10 }, "timeout_ms"}, + "a relative downloads folder": {func(u *Update) { u.LocalPath = "downloads" }, "local_path"}, + "a relative client folder": {func(u *Update) { u.RemotePath = "data" }, "remote_path"}, + "too many searches": {func(u *Update) { u.SearchesPerHour = 4000 }, "searches_per_hour"}, + } { + u := validUpdate() + tc.edit(&u) + var invalid *InvalidError + if err := u.validate(); !errors.As(err, &invalid) || invalid.Field != tc.field { + t.Errorf("%s: err = %v, want an InvalidError on %s", name, err, tc.field) + } + } + // Switched off, an empty address is a row still being set up. + u := validUpdate() + u.BaseURL = "" + if err := u.validate(); err != nil { + t.Errorf("switched off with no address: %v", err) + } +} + +func TestSourceFor_BuildsTheClientTheProtocolNames(t *testing.T) { + for protocol, check := range map[string]func(Source) bool{ + ProtocolProwlarr: func(s Source) bool { _, ok := s.Search.(*prowlarr.Client); return ok && s.Fetch != nil }, + ProtocolTorznab: func(s Source) bool { _, ok := s.Search.(*indexer.Torznab); return ok && s.Fetch != nil }, + // Exchange's results are all magnets: nothing to fetch through it. + ProtocolExchange: func(s Source) bool { _, ok := s.Search.(*exchange.Client); return ok && s.Fetch == nil }, + } { + src := SourceFor(dbq.AcquisitionService{Code: protocol, Protocol: protocol, SearchesPerHour: 7}) + if !check(src) || src.Code != protocol || src.PerHour != 7 { + t.Errorf("%s: source %+v", protocol, src) + } + } +} + +func TestJobs_AnExchangeDoesWhatItsSwitchesAllow(t *testing.T) { + ex := dbq.AcquisitionService{Kind: KindDownload, Protocol: ProtocolExchange, ServesSearch: true} + if !Searches(ex) || Fetches(ex) { + t.Error("an exchange with only search on should search and not fetch") + } + // A qBittorrent's switches mean nothing: it fetches, whatever they say. + qb := dbq.AcquisitionService{Kind: KindDownload, Protocol: ProtocolQBittorrent, ServesSearch: true} + if Searches(qb) || !Fetches(qb) { + t.Error("a qBittorrent fetches and never searches") + } +} + +func TestTest_NoAddressIsNothingToReach(t *testing.T) { + r := Test(context.Background(), dbq.AcquisitionService{Protocol: ProtocolProwlarr, Kind: KindSearch}) + if r.OK || !strings.Contains(r.Detail, "No address") { + t.Fatalf("result = %+v", r) + } +} + +func TestTest_SearchReportsResultsAndARefusedKey(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Header.Get("X-Api-Key") != "good" { + w.WriteHeader(http.StatusUnauthorized) + return + } + _, _ = w.Write([]byte(`[{"title": "Radiohead - OK Computer"}]`)) + })) + defer srv.Close() + + row := dbq.AcquisitionService{Kind: KindSearch, Protocol: ProtocolProwlarr, BaseUrl: srv.URL, ApiKey: "good", TimeoutMs: 5000} + if r := Test(context.Background(), row); !r.OK || !strings.Contains(r.Detail, "1 results") { + t.Errorf("a working key: %+v", r) + } + row.ApiKey = "bad" + if r := Test(context.Background(), row); r.OK || !strings.Contains(r.Detail, "refused") { + t.Errorf("a refused key: %+v", r) + } +} + +// fakeExchange answers the three native APIs. +func fakeExchange(t *testing.T) *httptest.Server { + t.Helper() + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch { + case r.URL.Path == "/api/downloads": + _, _ = w.Write([]byte(`{"enabled": true, "running": true, "inbound_conns": 3, "downloads": []}`)) + case r.URL.Path == "/api/search": + _, _ = w.Write([]byte(`{"total": 0, "results": []}`)) + case strings.HasPrefix(r.URL.Path, "/api/torrents/"): + w.WriteHeader(http.StatusNotFound) + _, _ = w.Write([]byte(`{"error": "unknown torrent"}`)) + default: + w.WriteHeader(http.StatusNotFound) + } + })) + t.Cleanup(srv.Close) + return srv +} + +func TestTest_AnExchangeReportsEachJobSwitchedOn(t *testing.T) { + srv := fakeExchange(t) + row := dbq.AcquisitionService{ + Kind: KindDownload, Protocol: ProtocolExchange, BaseUrl: srv.URL, TimeoutMs: 5000, + ServesDownloads: true, ServesSearch: true, ServesEnrichment: true, LocalPath: t.TempDir(), + } + r := Test(context.Background(), row) + if !r.OK { + t.Fatalf("all three jobs: %+v", r) + } + for _, job := range []string{"Downloads:", "Search:", "Release details:"} { + if !strings.Contains(r.Detail, job) { + t.Errorf("detail %q does not report %s", r.Detail, job) + } + } + row.ServesDownloads, row.ServesEnrichment = false, false + if r := Test(context.Background(), row); strings.Contains(r.Detail, "Downloads:") || !strings.Contains(r.Detail, "Search:") { + t.Errorf("only search on: %+v", r) + } + row.ServesSearch = false + if r := Test(context.Background(), row); r.OK || !strings.Contains(r.Detail, "switched off") { + t.Errorf("every job off: %+v", r) + } +} + +func TestTest_ADownloadClientNeedsAFolderMinstrelCanSee(t *testing.T) { + srv := fakeExchange(t) + row := dbq.AcquisitionService{ + Kind: KindDownload, Protocol: ProtocolExchange, BaseUrl: srv.URL, TimeoutMs: 5000, + ServesDownloads: true, Category: DefaultCategory, + LocalPath: filepath.Join(t.TempDir(), "not-mounted"), + } + r := Test(context.Background(), row) + if r.OK || !strings.Contains(r.Detail, "cannot see its downloads folder") { + t.Fatalf("a missing folder: %+v", r) + } + row.LocalPath = t.TempDir() + if r := Test(context.Background(), row); !r.OK || !strings.Contains(r.Detail, `"minstrel" category`) { + t.Errorf("a visible, writable folder: %+v", r) + } +} + +func TestDownloadClient_WrapsThePathMapping(t *testing.T) { + d, err := DownloadClient(dbq.AcquisitionService{Protocol: ProtocolQBittorrent, LocalPath: "/downloads", RemotePath: "/data"}) + if err != nil { + t.Fatal(err) + } + if _, plain := d.(*downloader.Client); plain { + t.Error("a row naming two folders should come back wrapped in its mapping") + } + if _, err := DownloadClient(dbq.AcquisitionService{Protocol: ProtocolProwlarr}); err == nil { + t.Error("a search protocol is not a download client") + } +} diff --git a/internal/acquire/settings.go b/internal/acquire/settings.go new file mode 100644 index 00000000..a7a9c9ea --- /dev/null +++ b/internal/acquire/settings.go @@ -0,0 +1,170 @@ +// Package acquire turns the operator's configured indexers and download +// clients into working clients (M502 #5518). Ported from Fabled Librarian's +// internal/acquire. +// +// ONE BUILDER PER KIND. SourceFor makes a search source from a row and +// DownloadClient makes a download client, path mapping included. The grab, +// the importer, the stall watchdog and the settings page's Test all build +// through them, so a Test that passes vouches for the path real traffic takes. +// Clients are rebuilt from the row on every use, so a changed address or a new +// key takes effect on the next call. +package acquire + +import ( + "context" + "time" + + "github.com/jackc/pgx/v5/pgxpool" + + "git.fabledsword.com/bvandeusen/minstrel/internal/db/dbq" + "git.fabledsword.com/bvandeusen/minstrel/internal/downloader" + "git.fabledsword.com/bvandeusen/minstrel/internal/exchange" + "git.fabledsword.com/bvandeusen/minstrel/internal/indexer" + "git.fabledsword.com/bvandeusen/minstrel/internal/prowlarr" +) + +// Kinds: which list a row appears in. +const ( + KindSearch = "search" + KindDownload = "download" +) + +// Jobs a row can do. An exchange row does each its switch allows; every +// other row does the one its kind names. +const ( + JobSearch = "search" + JobDownload = "download" + JobEnrichment = "enrichment" +) + +// Protocols: which client a row builds. +const ( + ProtocolProwlarr = "prowlarr" + ProtocolTorznab = "torznab" + ProtocolQBittorrent = "qbittorrent" + ProtocolExchange = "exchange" +) + +// DefaultCategory files everything Minstrel adds to a download client. +const DefaultCategory = "minstrel" + +// addable is what a new row of a protocol starts as. +type addable struct { + kind string + label string + category string + timeoutMs int32 + perHour int32 +} + +// catalogue is every protocol the operator can add. Exchange starts unpaced +// (it searches its own database); the others at 120 searches an hour, which +// keeps a Prowlarr's indexers under the rate most trackers allow. +var catalogue = map[string]addable{ + ProtocolProwlarr: {kind: KindSearch, label: "Prowlarr", timeoutMs: 60000, perHour: 120}, + ProtocolTorznab: {kind: KindSearch, label: "Torznab feed", timeoutMs: 30000, perHour: 120}, + ProtocolQBittorrent: {kind: KindDownload, label: "qBittorrent", category: DefaultCategory, timeoutMs: 15000, perHour: 0}, + ProtocolExchange: {kind: KindDownload, label: "FabledExchange", category: DefaultCategory, timeoutMs: 30000, perHour: 0}, +} + +// Addable says whether protocol is one the operator can add. +func Addable(protocol string) bool { + _, ok := catalogue[protocol] + return ok +} + +// Searches says whether a row is a search source. +func Searches(row dbq.AcquisitionService) bool { + if row.Protocol == ProtocolExchange { + return row.ServesSearch + } + return row.Kind == KindSearch +} + +// Fetches says whether a row is a download client. +func Fetches(row dbq.AcquisitionService) bool { + if row.Protocol == ProtocolExchange { + return row.ServesDownloads + } + return row.Kind == KindDownload +} + +// Searcher is a search source. +type Searcher interface { + Search(ctx context.Context, q indexer.Query) ([]indexer.Result, error) +} + +// Fetcher fetches a result's .torrent through the source that offered it. +type Fetcher interface { + Download(ctx context.Context, r indexer.Result) ([]byte, error) +} + +// Source is one configured search source. +type Source struct { + Code string + Label string + // PerHour is the most searches the schedule sends it an hour; 0 = unpaced. + PerHour int + Search Searcher + // Fetch is nil for a source whose results are all magnets. + Fetch Fetcher +} + +// SourceFor builds the search source a row describes. +func SourceFor(row dbq.AcquisitionService) Source { + src := Source{Code: row.Code, Label: row.Label, PerHour: int(row.SearchesPerHour)} + timeout := msDuration(row.TimeoutMs) + switch row.Protocol { + case ProtocolProwlarr: + c := &prowlarr.Client{Config: prowlarr.Config{ + Code: row.Code, BaseURL: row.BaseUrl, APIKey: row.ApiKey, Timeout: timeout, + }} + src.Search, src.Fetch = c, c + case ProtocolExchange: + src.Search = ExchangeFor(row) + default: + c := &indexer.Torznab{Config: indexer.TorznabConfig{ + Code: row.Code, FeedURL: row.BaseUrl, APIKey: row.ApiKey, Timeout: timeout, + }} + src.Search, src.Fetch = c, c + } + return src +} + +// ExchangeFor builds the search and enrichment client for an exchange row. +func ExchangeFor(row dbq.AcquisitionService) *exchange.Client { + return &exchange.Client{ + Code: row.Code, + Config: exchange.Config{ + BaseURL: row.BaseUrl, APIKey: row.ApiKey, Timeout: msDuration(row.TimeoutMs), + }, + } +} + +// DownloadClient builds the download client a row describes, wrapped in its +// path mapping. +func DownloadClient(row dbq.AcquisitionService) (downloader.Driver, error) { + d, err := downloader.For(row.Protocol, downloader.Config{ + BaseURL: row.BaseUrl, Username: row.Username, Password: row.Password, + APIKey: row.ApiKey, Category: row.Category, Timeout: msDuration(row.TimeoutMs), + }) + if err != nil { + return nil, err + } + return downloader.Mapped(d, downloader.PathMap{Local: row.LocalPath, Remote: row.RemotePath}), nil +} + +// Sources builds every enabled search source. +func Sources(ctx context.Context, pool *pgxpool.Pool) ([]Source, error) { + rows, err := dbq.New(pool).ListUsableAcquisitionServices(ctx, JobSearch) + if err != nil { + return nil, err + } + out := make([]Source, len(rows)) + for i, r := range rows { + out[i] = SourceFor(r) + } + return out, nil +} + +func msDuration(ms int32) time.Duration { return time.Duration(ms) * time.Millisecond } diff --git a/internal/acquire/store.go b/internal/acquire/store.go new file mode 100644 index 00000000..833435b9 --- /dev/null +++ b/internal/acquire/store.go @@ -0,0 +1,191 @@ +package acquire + +import ( + "context" + "errors" + "fmt" + "net/url" + "strings" + + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgxpool" + + "git.fabledsword.com/bvandeusen/minstrel/internal/db/dbq" +) + +// ErrNotFound is a code with no row. +var ErrNotFound = errors.New("no such service") + +// ErrTooMany is a twenty-first row of one protocol. +var ErrTooMany = errors.New("there are already twenty of these") + +// InvalidError is a setting the operator has to change, in words for the +// settings card. +type InvalidError struct { + Field string + Message string +} + +func (e *InvalidError) Error() string { return e.Message } + +func invalid(field, msg string) error { return &InvalidError{Field: field, Message: msg} } + +// maxPerProtocol bounds the codes tried for a new row. Nobody runs twenty +// Prowlarrs; the bound keeps a loop over a full table finite. +const maxPerProtocol = 20 + +// Store reads and writes acquisition_services. +type Store struct { + pool *pgxpool.Pool +} + +// NewStore returns a Store over pool. +func NewStore(pool *pgxpool.Pool) *Store { return &Store{pool: pool} } + +// List returns every row without its secrets. +func (s *Store) List(ctx context.Context) ([]dbq.ListAcquisitionServicesRow, error) { + return dbq.New(s.pool).ListAcquisitionServices(ctx) +} + +// Get returns one row, secrets included: for building a client only. +func (s *Store) Get(ctx context.Context, code string) (dbq.AcquisitionService, error) { + row, err := dbq.New(s.pool).GetAcquisitionService(ctx, code) + if errors.Is(err, pgx.ErrNoRows) { + return row, ErrNotFound + } + return row, err +} + +// Create adds a row of protocol, switched off and with no address. Its code +// is the protocol for the first, then protocol-2, protocol-3. +func (s *Store) Create(ctx context.Context, protocol string) (dbq.AcquisitionService, error) { + a, ok := catalogue[protocol] + if !ok { + return dbq.AcquisitionService{}, invalid("protocol", "One of prowlarr, torznab, qbittorrent, exchange.") + } + q := dbq.New(s.pool) + for n := 1; n <= maxPerProtocol; n++ { + code, label := protocol, a.label + if n > 1 { + code, label = fmt.Sprintf("%s-%d", protocol, n), fmt.Sprintf("%s %d", a.label, n) + } + row, err := q.CreateAcquisitionService(ctx, dbq.CreateAcquisitionServiceParams{ + Code: code, Kind: a.kind, Protocol: protocol, Label: label, + Category: a.category, TimeoutMs: a.timeoutMs, SearchesPerHour: a.perHour, + }) + if errors.Is(err, pgx.ErrNoRows) { + continue + } + return row, err + } + return dbq.AcquisitionService{}, ErrTooMany +} + +// Update is everything on the settings form except the credentials. +type Update struct { + Label string + Enabled bool + BaseURL string + Username string + Category string + TimeoutMs int32 + ServesSearch bool + ServesDownloads bool + ServesEnrichment bool + LocalPath string + RemotePath string + SearchesPerHour int32 +} + +// Save validates u and writes it to code's row. +func (s *Store) Save(ctx context.Context, code string, u Update) (dbq.AcquisitionService, error) { + u.Label = strings.TrimSpace(u.Label) + u.BaseURL = strings.TrimSpace(u.BaseURL) + u.Username = strings.TrimSpace(u.Username) + u.Category = strings.TrimSpace(u.Category) + u.LocalPath = strings.TrimSpace(u.LocalPath) + u.RemotePath = strings.TrimSpace(u.RemotePath) + if err := u.validate(); err != nil { + return dbq.AcquisitionService{}, err + } + row, err := dbq.New(s.pool).UpdateAcquisitionService(ctx, dbq.UpdateAcquisitionServiceParams{ + Code: code, Label: u.Label, Enabled: u.Enabled, BaseUrl: u.BaseURL, + Username: u.Username, Category: u.Category, TimeoutMs: u.TimeoutMs, + ServesSearch: u.ServesSearch, ServesDownloads: u.ServesDownloads, + ServesEnrichment: u.ServesEnrichment, LocalPath: u.LocalPath, + RemotePath: u.RemotePath, SearchesPerHour: u.SearchesPerHour, + }) + if errors.Is(err, pgx.ErrNoRows) { + return row, ErrNotFound + } + return row, err +} + +// validate mirrors the table's CHECKs, so a bad value is named in the card's +// own terms before Postgres refuses it. +func (u Update) validate() error { + if u.Label == "" { + return invalid("label", "Name: give it a name.") + } + if u.Enabled && u.BaseURL == "" { + return invalid("base_url", "Address: give the address before switching this on.") + } + if u.BaseURL != "" { + parsed, err := url.Parse(u.BaseURL) + if err != nil || (parsed.Scheme != "http" && parsed.Scheme != "https") || parsed.Host == "" { + return invalid("base_url", "Address: start with http:// or https://.") + } + } + if u.TimeoutMs < 1000 || u.TimeoutMs > 300000 { + return invalid("timeout_ms", "Give up after: between 1000 and 300000 milliseconds.") + } + if !strings.HasPrefix(u.LocalPath, "/") { + return invalid("local_path", "Downloads folder as Minstrel sees it: a full path, starting with /.") + } + if u.RemotePath != "" && !strings.HasPrefix(u.RemotePath, "/") { + return invalid("remote_path", "Downloads folder as the client sees it: a full path starting with /, or empty when it is the same path.") + } + if u.SearchesPerHour < 0 || u.SearchesPerHour > 3600 { + return invalid("searches_per_hour", "Searches per hour: between 0 and 3600. 0 means no limit.") + } + return nil +} + +// Delete removes code's row. +func (s *Store) Delete(ctx context.Context, code string) error { + n, err := dbq.New(s.pool).DeleteAcquisitionService(ctx, code) + if err != nil { + return err + } + if n == 0 { + return ErrNotFound + } + return nil +} + +// SetAPIKey stores a key, trimmed; empty clears it. +func (s *Store) SetAPIKey(ctx context.Context, code, key string) error { + n, err := dbq.New(s.pool).SetAcquisitionServiceAPIKey(ctx, dbq.SetAcquisitionServiceAPIKeyParams{ + Code: code, ApiKey: strings.TrimSpace(key), + }) + return affected(n, err) +} + +// SetPassword stores a password as typed (spaces are part of a password); +// empty clears it. +func (s *Store) SetPassword(ctx context.Context, code, password string) error { + n, err := dbq.New(s.pool).SetAcquisitionServicePassword(ctx, dbq.SetAcquisitionServicePasswordParams{ + Code: code, Password: password, + }) + return affected(n, err) +} + +func affected(n int64, err error) error { + if err != nil { + return err + } + if n == 0 { + return ErrNotFound + } + return nil +} diff --git a/internal/acquire/store_test.go b/internal/acquire/store_test.go new file mode 100644 index 00000000..37c369da --- /dev/null +++ b/internal/acquire/store_test.go @@ -0,0 +1,108 @@ +package acquire + +import ( + "context" + "errors" + "io" + "log/slog" + "os" + "testing" + + "github.com/jackc/pgx/v5/pgxpool" + + "git.fabledsword.com/bvandeusen/minstrel/internal/db" + "git.fabledsword.com/bvandeusen/minstrel/internal/dbtest" +) + +func newPool(t *testing.T) *pgxpool.Pool { + t.Helper() + if testing.Short() { + t.Skip("skipping integration test in -short mode") + } + dsn := os.Getenv("MINSTREL_TEST_DATABASE_URL") + if dsn == "" { + t.Skip("MINSTREL_TEST_DATABASE_URL not set") + } + if err := db.Migrate(dsn, slog.New(slog.NewTextHandler(io.Discard, nil))); err != nil { + t.Fatalf("migrate: %v", err) + } + pool, err := pgxpool.New(context.Background(), dsn) + if err != nil { + t.Fatalf("pool: %v", err) + } + t.Cleanup(pool.Close) + dbtest.ResetDB(t, pool) + return pool +} + +func TestStore_AddsConfiguresAndRemoves_Integration(t *testing.T) { + ctx := context.Background() + s := NewStore(newPool(t)) + + first, err := s.Create(ctx, ProtocolExchange) + if err != nil { + t.Fatalf("create: %v", err) + } + second, err := s.Create(ctx, ProtocolExchange) + if err != nil { + t.Fatalf("create a second: %v", err) + } + if first.Code != "exchange" || second.Code != "exchange-2" || second.Label != "FabledExchange 2" { + t.Errorf("codes = %q, %q (%q); want exchange, exchange-2", first.Code, second.Code, second.Label) + } + // A new exchange is off, does every job, files under minstrel and is + // unpaced. + if first.Enabled || first.Kind != KindDownload || !first.ServesSearch || !first.ServesDownloads || + !first.ServesEnrichment || first.Category != DefaultCategory || first.SearchesPerHour != 0 || first.LocalPath != "/downloads" { + t.Errorf("a new exchange = %+v", first) + } + if _, err := s.Create(ctx, "transmission"); !errors.As(err, new(*InvalidError)) { + t.Errorf("an unknown protocol: err = %v", err) + } + + u := Update{ + Label: "Exchange", Enabled: true, BaseURL: "http://exchange:8080", Category: "minstrel", + TimeoutMs: 20000, ServesSearch: true, LocalPath: "/downloads", RemotePath: "/data/torrents", + } + saved, err := s.Save(ctx, first.Code, u) + if err != nil { + t.Fatalf("save: %v", err) + } + if !saved.Enabled || saved.ServesDownloads || saved.RemotePath != "/data/torrents" { + t.Errorf("saved = %+v", saved) + } + if _, err := s.Save(ctx, "nope", u); !errors.Is(err, ErrNotFound) { + t.Errorf("saving a missing row: err = %v", err) + } + + // Secrets are write-only: the list says whether one is set, never what. + if err := s.SetAPIKey(ctx, first.Code, " key "); err != nil { + t.Fatalf("set key: %v", err) + } + full, err := s.Get(ctx, first.Code) + if err != nil || full.ApiKey != "key" { + t.Errorf("stored key = %q (err %v), want it trimmed", full.ApiKey, err) + } + rows, err := s.List(ctx) + if err != nil || len(rows) != 2 { + t.Fatalf("list: %d rows, err %v", len(rows), err) + } + for _, r := range rows { + if r.Code == first.Code && !r.HasApiKey { + t.Error("the list should say a key is set") + } + } + + // Only the exchange with search switched on is a search source. + srcs, err := Sources(ctx, s.pool) + if err != nil || len(srcs) != 1 || srcs[0].Code != first.Code { + t.Errorf("sources = %+v, err %v; want only the enabled exchange", srcs, err) + } + + if err := s.Delete(ctx, second.Code); err != nil { + t.Fatalf("delete: %v", err) + } + if err := s.Delete(ctx, second.Code); !errors.Is(err, ErrNotFound) { + t.Errorf("deleting twice: err = %v", err) + } +} diff --git a/internal/acquire/test.go b/internal/acquire/test.go new file mode 100644 index 00000000..9ec6d07b --- /dev/null +++ b/internal/acquire/test.go @@ -0,0 +1,134 @@ +package acquire + +import ( + "context" + "errors" + "fmt" + "os" + "strings" + + "git.fabledsword.com/bvandeusen/minstrel/internal/db/dbq" + "git.fabledsword.com/bvandeusen/minstrel/internal/exchange" + "git.fabledsword.com/bvandeusen/minstrel/internal/indexer" +) + +// TestResult is what the settings card says after Test. +type TestResult struct { + OK bool `json:"ok"` + Detail string `json:"detail"` +} + +// testQuery is a search every music indexer should answer. +const testQuery = "Radiohead" + +// Test reaches the service a row describes, through the same builders real +// traffic uses. It runs whether or not the row is switched on: paste the +// address, test it, then switch it on. +func Test(ctx context.Context, row dbq.AcquisitionService) TestResult { + if strings.TrimSpace(row.BaseUrl) == "" { + return TestResult{Detail: "No address is set, so there is nothing to reach yet."} + } + if row.Protocol == ProtocolExchange { + return testExchange(ctx, row) + } + if row.Kind == KindSearch { + return testSearch(ctx, row) + } + return testDownload(ctx, row) +} + +// testExchange runs each job the row has switched on and reports them all. +func testExchange(ctx context.Context, row dbq.AcquisitionService) TestResult { + var parts []string + ok := true + run := func(name string, on bool, test func() TestResult) { + if !on { + return + } + r := test() + ok = ok && r.OK + parts = append(parts, name+": "+r.Detail) + } + run("Downloads", row.ServesDownloads, func() TestResult { return testDownload(ctx, row) }) + run("Search", row.ServesSearch, func() TestResult { return testSearch(ctx, row) }) + run("Release details", row.ServesEnrichment, func() TestResult { return testEnrichment(ctx, row) }) + if len(parts) == 0 { + return TestResult{Detail: "Every job of this FabledExchange is switched off, so there is nothing to test."} + } + return TestResult{OK: ok, Detail: strings.Join(parts, " ")} +} + +// testSearch runs a real search. +func testSearch(ctx context.Context, row dbq.AcquisitionService) TestResult { + src := SourceFor(row) + results, err := src.Search.Search(ctx, indexer.Query{Text: testQuery, Categories: indexer.MusicCategories}) + switch { + case errors.Is(err, indexer.ErrUnconfigured): + return TestResult{Detail: "Add this service's address, and the API key it shows on its own settings page."} + case errors.Is(err, indexer.ErrAuth): + return TestResult{Detail: indexer.FirstLine(err.Error()) + ". Check it on the service's own settings page."} + case err != nil: + return TestResult{Detail: indexer.FirstLine(err.Error())} + case len(results) == 0: + return TestResult{OK: true, Detail: "Connected, but a test search for music found nothing. Check that it has an indexer with music categories."} + } + return TestResult{OK: true, Detail: fmt.Sprintf("Connected. %d results for a test search.", len(results))} +} + +// testDownload lists the client's torrents, which proves the session and not +// only the login, then checks the downloads folder from Minstrel's side. +func testDownload(ctx context.Context, row dbq.AcquisitionService) TestResult { + c, err := DownloadClient(row) + if err != nil { + return TestResult{Detail: err.Error()} + } + ts, err := c.Torrents(ctx) + if err != nil { + return TestResult{Detail: indexer.FirstLine(err.Error())} + } + detail := fmt.Sprintf("Connected. %d torrents.", len(ts)) + if row.Category != "" { + detail = fmt.Sprintf("Connected. %d torrents in the %q category.", len(ts), row.Category) + } + if up, err := c.Connected(ctx); err == nil && !up { + detail += " The client reports it is not connected to any peers, so downloads will stall." + } + if problem := landingProblem(row.LocalPath); problem != "" { + return TestResult{Detail: detail + " " + problem} + } + return TestResult{OK: true, Detail: detail} +} + +// landingProblem checks Minstrel can see and write the downloads folder: it +// moves finished downloads out of it. Only the local path can be checked; +// the client's own name for it cannot be seen from here. +func landingProblem(dir string) string { + info, err := os.Stat(dir) + switch { + case err != nil: + return fmt.Sprintf("But Minstrel cannot see its downloads folder at %s. Mount the download client's downloads folder there, or change where Minstrel looks for it.", dir) + case !info.IsDir(): + return fmt.Sprintf("But %s is a file, not the downloads folder.", dir) + } + probe, err := os.CreateTemp(dir, ".minstrel-probe-*") + if err != nil { + return fmt.Sprintf("But Minstrel cannot write to %s, so it could not move a finished download out of it. Check the mount is not read-only and Minstrel's user can write there.", dir) + } + name := probe.Name() + _ = probe.Close() + _ = os.Remove(name) + return "" +} + +// testEnrichment asks about a hash no torrent has. A JSON "not crawled" is a +// healthy answer; a 404 from something that is not Exchange is not. +func testEnrichment(ctx context.Context, row dbq.AcquisitionService) TestResult { + _, err := ExchangeFor(row).Torrent(ctx, strings.Repeat("0", 40)) + switch { + case err == nil, errors.Is(err, exchange.ErrUnknown): + return TestResult{OK: true, Detail: "Connected. The index answered."} + case errors.Is(err, exchange.ErrUnconfigured): + return TestResult{Detail: "No address is set, so there is nothing to reach yet."} + } + return TestResult{Detail: indexer.FirstLine(err.Error())} +} diff --git a/internal/api/admin_acquisition.go b/internal/api/admin_acquisition.go new file mode 100644 index 00000000..e857d24a --- /dev/null +++ b/internal/api/admin_acquisition.go @@ -0,0 +1,209 @@ +package api + +import ( + "encoding/json" + "errors" + "io" + "net/http" + + "github.com/go-chi/chi/v5" + + "git.fabledsword.com/bvandeusen/minstrel/internal/acquire" + "git.fabledsword.com/bvandeusen/minstrel/internal/apierror" + "git.fabledsword.com/bvandeusen/minstrel/internal/db/dbq" +) + +// Indexers and download clients, admin side (M502 #5518). Rows the operator +// adds and configures; nothing is seeded. Secrets are write-only: no response +// carries one, only whether it is set. + +// acquisitionBodyLimit caps a settings body. The largest is a full form of +// short strings. +const acquisitionBodyLimit = 64 << 10 + +type acquisitionServiceResp struct { + Code string `json:"code"` + Kind string `json:"kind"` + Protocol string `json:"protocol"` + Label string `json:"label"` + Enabled bool `json:"enabled"` + BaseURL string `json:"base_url"` + HasAPIKey bool `json:"has_api_key"` + HasPassword bool `json:"has_password"` + Username string `json:"username"` + Category string `json:"category"` + TimeoutMs int32 `json:"timeout_ms"` + ServesSearch bool `json:"serves_search"` + ServesDownloads bool `json:"serves_downloads"` + ServesEnrichment bool `json:"serves_enrichment"` + LocalPath string `json:"local_path"` + RemotePath string `json:"remote_path"` + SearchesPerHour int32 `json:"searches_per_hour"` + // Usable is switched on and addressed: the state the jobs read. + Usable bool `json:"usable"` +} + +func acquisitionRespOf(r dbq.AcquisitionService) acquisitionServiceResp { + return acquisitionServiceResp{ + Code: r.Code, Kind: r.Kind, Protocol: r.Protocol, Label: r.Label, + Enabled: r.Enabled, BaseURL: r.BaseUrl, + HasAPIKey: r.ApiKey != "", HasPassword: r.Password != "", + Username: r.Username, Category: r.Category, TimeoutMs: r.TimeoutMs, + ServesSearch: r.ServesSearch, ServesDownloads: r.ServesDownloads, + ServesEnrichment: r.ServesEnrichment, + LocalPath: r.LocalPath, RemotePath: r.RemotePath, + SearchesPerHour: r.SearchesPerHour, + Usable: r.Enabled && r.BaseUrl != "", + } +} + +// handleListAcquisitionServices implements GET /api/admin/acquisition/services. +func (h *handlers) handleListAcquisitionServices(w http.ResponseWriter, r *http.Request) { + rows, err := h.acquisition.List(r.Context()) + if err != nil { + writeErrWithLog(w, h.logger, "admin: list acquisition services", apierror.Internal(err)) + return + } + out := make([]acquisitionServiceResp, len(rows)) + for i, row := range rows { + out[i] = acquisitionRespOf(dbq.AcquisitionService{ + Code: row.Code, Kind: row.Kind, Protocol: row.Protocol, Label: row.Label, + Enabled: row.Enabled, BaseUrl: row.BaseUrl, Username: row.Username, + Category: row.Category, TimeoutMs: row.TimeoutMs, + ServesSearch: row.ServesSearch, ServesDownloads: row.ServesDownloads, + ServesEnrichment: row.ServesEnrichment, LocalPath: row.LocalPath, + RemotePath: row.RemotePath, SearchesPerHour: row.SearchesPerHour, + }) + out[i].HasAPIKey, out[i].HasPassword = row.HasApiKey, row.HasPassword + } + writeJSON(w, http.StatusOK, map[string]any{"services": out}) +} + +// handleCreateAcquisitionService implements POST /api/admin/acquisition/services +// with {"protocol": "prowlarr"|"torznab"|"qbittorrent"|"exchange"}. The row +// starts switched off, with no address. +func (h *handlers) handleCreateAcquisitionService(w http.ResponseWriter, r *http.Request) { + var body struct { + Protocol string `json:"protocol"` + } + if !decodeAcquisitionBody(w, r, &body) { + return + } + row, err := h.acquisition.Create(r.Context(), body.Protocol) + if err != nil { + h.writeAcquisitionErr(w, "admin: create acquisition service", err) + return + } + writeJSON(w, http.StatusCreated, acquisitionRespOf(row)) +} + +type updateAcquisitionBody struct { + Label string `json:"label"` + Enabled bool `json:"enabled"` + BaseURL string `json:"base_url"` + Username string `json:"username"` + Category string `json:"category"` + TimeoutMs int32 `json:"timeout_ms"` + ServesSearch bool `json:"serves_search"` + ServesDownloads bool `json:"serves_downloads"` + ServesEnrichment bool `json:"serves_enrichment"` + LocalPath string `json:"local_path"` + RemotePath string `json:"remote_path"` + SearchesPerHour int32 `json:"searches_per_hour"` +} + +// handleUpdateAcquisitionService implements PUT +// /api/admin/acquisition/services/{code}: the whole form except credentials. +func (h *handlers) handleUpdateAcquisitionService(w http.ResponseWriter, r *http.Request) { + var body updateAcquisitionBody + if !decodeAcquisitionBody(w, r, &body) { + return + } + row, err := h.acquisition.Save(r.Context(), chi.URLParam(r, "code"), acquire.Update(body)) + if err != nil { + h.writeAcquisitionErr(w, "admin: update acquisition service", err) + return + } + writeJSON(w, http.StatusOK, acquisitionRespOf(row)) +} + +// handleDeleteAcquisitionService implements DELETE +// /api/admin/acquisition/services/{code}. +func (h *handlers) handleDeleteAcquisitionService(w http.ResponseWriter, r *http.Request) { + if err := h.acquisition.Delete(r.Context(), chi.URLParam(r, "code")); err != nil { + h.writeAcquisitionErr(w, "admin: delete acquisition service", err) + return + } + w.WriteHeader(http.StatusNoContent) +} + +// handleTestAcquisitionService implements POST +// /api/admin/acquisition/services/{code}/test. POST because it reaches +// another machine, which a prefetch must not trigger. Always 200 with +// {ok, detail}: a failed connection is the answer, not an error. +func (h *handlers) handleTestAcquisitionService(w http.ResponseWriter, r *http.Request) { + row, err := h.acquisition.Get(r.Context(), chi.URLParam(r, "code")) + if err != nil { + h.writeAcquisitionErr(w, "admin: test acquisition service", err) + return + } + writeJSON(w, http.StatusOK, acquire.Test(r.Context(), row)) +} + +// handleSetAcquisitionSecret implements PUT and DELETE on +// /api/admin/acquisition/services/{code}/api-key and .../password. PUT takes +// {"value": "..."}; DELETE clears it. 204 with no body either way: echoing +// anything here is how a secret ends up in a response. +func (h *handlers) handleSetAcquisitionSecret(field string) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + value := "" + if r.Method == http.MethodPut { + var body struct { + Value string `json:"value"` + } + if !decodeAcquisitionBody(w, r, &body) { + return + } + if body.Value == "" { + writeErr(w, apierror.BadRequest("invalid_setting", "Paste a value, or use Remove to clear the one that is set.")) + return + } + value = body.Value + } + code := chi.URLParam(r, "code") + var err error + if field == "password" { + err = h.acquisition.SetPassword(r.Context(), code, value) + } else { + err = h.acquisition.SetAPIKey(r.Context(), code, value) + } + if err != nil { + h.writeAcquisitionErr(w, "admin: set acquisition "+field, err) + return + } + w.WriteHeader(http.StatusNoContent) + } +} + +func decodeAcquisitionBody(w http.ResponseWriter, r *http.Request, into any) bool { + err := json.NewDecoder(http.MaxBytesReader(w, r.Body, acquisitionBodyLimit)).Decode(into) + if err != nil && !errors.Is(err, io.EOF) { + writeErr(w, apierror.BadRequest("invalid_body", "malformed JSON")) + return false + } + return true +} + +func (h *handlers) writeAcquisitionErr(w http.ResponseWriter, msg string, err error) { + var invalid *acquire.InvalidError + switch { + case errors.As(err, &invalid): + writeErr(w, apierror.BadRequest("invalid_setting", invalid.Message)) + case errors.Is(err, acquire.ErrNotFound): + writeErr(w, apierror.NotFound("service")) + case errors.Is(err, acquire.ErrTooMany): + writeErr(w, apierror.Conflict("too_many_services", "There are already twenty of these. Remove one you no longer use first.")) + default: + writeErrWithLog(w, h.logger, msg, apierror.Internal(err)) + } +} diff --git a/internal/api/api.go b/internal/api/api.go index 1162ec25..ec2419bf 100644 --- a/internal/api/api.go +++ b/internal/api/api.go @@ -13,6 +13,7 @@ import ( "github.com/go-chi/chi/v5" "github.com/jackc/pgx/v5/pgxpool" + "git.fabledsword.com/bvandeusen/minstrel/internal/acquire" "git.fabledsword.com/bvandeusen/minstrel/internal/auth" "git.fabledsword.com/bvandeusen/minstrel/internal/config" "git.fabledsword.com/bvandeusen/minstrel/internal/coverart" @@ -70,6 +71,7 @@ func Mount(r chi.Router, pool *pgxpool.Pool, logger *slog.Logger, events *playev fingerprintSettings: fpSettings, loudnessSettings: loudSettings, acoustIDLookup: acoustIDLookup, + acquisition: acquire.NewStore(pool), leveled: newLeveledRenderer(dataDir, loudSettings, logger), librarySize: recommendation.NewLibrarySize(nil), loginGuard: auth.NewLoginGuard(), @@ -214,6 +216,15 @@ func Mount(r chi.Router, pool *pgxpool.Pool, logger *slog.Logger, events *playev admin.Get("/lidarr/config", h.handleGetLidarrConfig) admin.Put("/lidarr/config", h.handlePutLidarrConfig) admin.Post("/lidarr/test", h.handleTestLidarrConnection) + admin.Get("/acquisition/services", h.handleListAcquisitionServices) + admin.Post("/acquisition/services", h.handleCreateAcquisitionService) + admin.Put("/acquisition/services/{code}", h.handleUpdateAcquisitionService) + admin.Delete("/acquisition/services/{code}", h.handleDeleteAcquisitionService) + admin.Post("/acquisition/services/{code}/test", h.handleTestAcquisitionService) + admin.Put("/acquisition/services/{code}/api-key", h.handleSetAcquisitionSecret("api_key")) + admin.Delete("/acquisition/services/{code}/api-key", h.handleSetAcquisitionSecret("api_key")) + admin.Put("/acquisition/services/{code}/password", h.handleSetAcquisitionSecret("password")) + admin.Delete("/acquisition/services/{code}/password", h.handleSetAcquisitionSecret("password")) admin.Get("/lidarr/quality-profiles", h.handleListQualityProfiles) admin.Get("/lidarr/metadata-profiles", h.handleListMetadataProfiles) admin.Get("/lidarr/root-folders", h.handleListRootFolders) @@ -372,6 +383,8 @@ type handlers struct { // acoustIDLookup is the AcoustID lookup worker (M401): its settings, its // status and "look up now". Nil serves the defaults (off). acoustIDLookup *library.AcoustIDLookupWorker + // acquisition is the operator's indexers and download clients (M502). + acquisition *acquire.Store // leveled renders the leveled streams handed to Sonos/UPnP speakers // (M464 #5001). Nil when its cache directory cannot be made: a level // request then gets the plain stream. diff --git a/internal/db/dbq/acquisition.sql.go b/internal/db/dbq/acquisition.sql.go new file mode 100644 index 00000000..2ab1b648 --- /dev/null +++ b/internal/db/dbq/acquisition.sql.go @@ -0,0 +1,362 @@ +// Code generated by sqlc. DO NOT EDIT. +// versions: +// sqlc v1.31.1 +// source: acquisition.sql + +package dbq + +import ( + "context" + + "github.com/jackc/pgx/v5/pgtype" +) + +const createAcquisitionService = `-- name: CreateAcquisitionService :one +INSERT INTO acquisition_services + (code, kind, protocol, label, category, timeout_ms, searches_per_hour) +VALUES + ($1, $2, $3, $4, + $5, $6, $7) +ON CONFLICT (code) DO NOTHING +RETURNING code, kind, protocol, label, enabled, base_url, api_key, username, password, category, timeout_ms, serves_search, serves_downloads, serves_enrichment, local_path, remote_path, searches_per_hour, created_at, updated_at +` + +type CreateAcquisitionServiceParams struct { + Code string + Kind string + Protocol string + Label string + Category string + TimeoutMs int32 + SearchesPerHour int32 +} + +// ON CONFLICT so the caller can try the next code when this one is taken. +func (q *Queries) CreateAcquisitionService(ctx context.Context, arg CreateAcquisitionServiceParams) (AcquisitionService, error) { + row := q.db.QueryRow(ctx, createAcquisitionService, + arg.Code, + arg.Kind, + arg.Protocol, + arg.Label, + arg.Category, + arg.TimeoutMs, + arg.SearchesPerHour, + ) + var i AcquisitionService + err := row.Scan( + &i.Code, + &i.Kind, + &i.Protocol, + &i.Label, + &i.Enabled, + &i.BaseUrl, + &i.ApiKey, + &i.Username, + &i.Password, + &i.Category, + &i.TimeoutMs, + &i.ServesSearch, + &i.ServesDownloads, + &i.ServesEnrichment, + &i.LocalPath, + &i.RemotePath, + &i.SearchesPerHour, + &i.CreatedAt, + &i.UpdatedAt, + ) + return i, err +} + +const deleteAcquisitionService = `-- name: DeleteAcquisitionService :execrows +DELETE FROM acquisition_services WHERE code = $1 +` + +func (q *Queries) DeleteAcquisitionService(ctx context.Context, code string) (int64, error) { + result, err := q.db.Exec(ctx, deleteAcquisitionService, code) + if err != nil { + return 0, err + } + return result.RowsAffected(), nil +} + +const getAcquisitionService = `-- name: GetAcquisitionService :one +SELECT code, kind, protocol, label, enabled, base_url, api_key, username, password, category, timeout_ms, serves_search, serves_downloads, serves_enrichment, local_path, remote_path, searches_per_hour, created_at, updated_at FROM acquisition_services WHERE code = $1 +` + +// The full row, credentials included. For building a client, never for a +// response. +func (q *Queries) GetAcquisitionService(ctx context.Context, code string) (AcquisitionService, error) { + row := q.db.QueryRow(ctx, getAcquisitionService, code) + var i AcquisitionService + err := row.Scan( + &i.Code, + &i.Kind, + &i.Protocol, + &i.Label, + &i.Enabled, + &i.BaseUrl, + &i.ApiKey, + &i.Username, + &i.Password, + &i.Category, + &i.TimeoutMs, + &i.ServesSearch, + &i.ServesDownloads, + &i.ServesEnrichment, + &i.LocalPath, + &i.RemotePath, + &i.SearchesPerHour, + &i.CreatedAt, + &i.UpdatedAt, + ) + return i, err +} + +const listAcquisitionServices = `-- name: ListAcquisitionServices :many + +SELECT code, kind, protocol, label, enabled, base_url, + (api_key <> '')::boolean AS has_api_key, + (password <> '')::boolean AS has_password, + username, category, timeout_ms, + serves_search, serves_downloads, serves_enrichment, + local_path, remote_path, searches_per_hour, + created_at, updated_at +FROM acquisition_services +ORDER BY kind, label, code +` + +type ListAcquisitionServicesRow struct { + Code string + Kind string + Protocol string + Label string + Enabled bool + BaseUrl string + HasApiKey bool + HasPassword bool + Username string + Category string + TimeoutMs int32 + ServesSearch bool + ServesDownloads bool + ServesEnrichment bool + LocalPath string + RemotePath string + SearchesPerHour int32 + CreatedAt pgtype.Timestamptz + UpdatedAt pgtype.Timestamptz +} + +// Indexers and download clients (M502 #5518). +// Returns no secret, only whether one is set: this feeds the settings page. +func (q *Queries) ListAcquisitionServices(ctx context.Context) ([]ListAcquisitionServicesRow, error) { + rows, err := q.db.Query(ctx, listAcquisitionServices) + if err != nil { + return nil, err + } + defer rows.Close() + var items []ListAcquisitionServicesRow + for rows.Next() { + var i ListAcquisitionServicesRow + if err := rows.Scan( + &i.Code, + &i.Kind, + &i.Protocol, + &i.Label, + &i.Enabled, + &i.BaseUrl, + &i.HasApiKey, + &i.HasPassword, + &i.Username, + &i.Category, + &i.TimeoutMs, + &i.ServesSearch, + &i.ServesDownloads, + &i.ServesEnrichment, + &i.LocalPath, + &i.RemotePath, + &i.SearchesPerHour, + &i.CreatedAt, + &i.UpdatedAt, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const listUsableAcquisitionServices = `-- name: ListUsableAcquisitionServices :many +SELECT code, kind, protocol, label, enabled, base_url, api_key, username, password, category, timeout_ms, serves_search, serves_downloads, serves_enrichment, local_path, remote_path, searches_per_hour, created_at, updated_at FROM acquisition_services +WHERE enabled AND base_url <> '' + AND CASE WHEN protocol = 'exchange' THEN + CASE $1::text + WHEN 'search' THEN serves_search + WHEN 'download' THEN serves_downloads + WHEN 'enrichment' THEN serves_enrichment + ELSE false + END + ELSE kind = $1::text + END +ORDER BY code +` + +// Every enabled, addressed service doing one job. An exchange row does a job +// when its switch for it is on; any other row does the job its kind names. +// Ordered by code so a fan-out is deterministic. +func (q *Queries) ListUsableAcquisitionServices(ctx context.Context, job string) ([]AcquisitionService, error) { + rows, err := q.db.Query(ctx, listUsableAcquisitionServices, job) + if err != nil { + return nil, err + } + defer rows.Close() + var items []AcquisitionService + for rows.Next() { + var i AcquisitionService + if err := rows.Scan( + &i.Code, + &i.Kind, + &i.Protocol, + &i.Label, + &i.Enabled, + &i.BaseUrl, + &i.ApiKey, + &i.Username, + &i.Password, + &i.Category, + &i.TimeoutMs, + &i.ServesSearch, + &i.ServesDownloads, + &i.ServesEnrichment, + &i.LocalPath, + &i.RemotePath, + &i.SearchesPerHour, + &i.CreatedAt, + &i.UpdatedAt, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const setAcquisitionServiceAPIKey = `-- name: SetAcquisitionServiceAPIKey :execrows +UPDATE acquisition_services SET api_key = $1, updated_at = now() +WHERE code = $2 +` + +type SetAcquisitionServiceAPIKeyParams struct { + ApiKey string + Code string +} + +func (q *Queries) SetAcquisitionServiceAPIKey(ctx context.Context, arg SetAcquisitionServiceAPIKeyParams) (int64, error) { + result, err := q.db.Exec(ctx, setAcquisitionServiceAPIKey, arg.ApiKey, arg.Code) + if err != nil { + return 0, err + } + return result.RowsAffected(), nil +} + +const setAcquisitionServicePassword = `-- name: SetAcquisitionServicePassword :execrows +UPDATE acquisition_services SET password = $1, updated_at = now() +WHERE code = $2 +` + +type SetAcquisitionServicePasswordParams struct { + Password string + Code string +} + +func (q *Queries) SetAcquisitionServicePassword(ctx context.Context, arg SetAcquisitionServicePasswordParams) (int64, error) { + result, err := q.db.Exec(ctx, setAcquisitionServicePassword, arg.Password, arg.Code) + if err != nil { + return 0, err + } + return result.RowsAffected(), nil +} + +const updateAcquisitionService = `-- name: UpdateAcquisitionService :one +UPDATE acquisition_services SET + label = $1, + enabled = $2, + base_url = $3, + username = $4, + category = $5, + timeout_ms = $6, + serves_search = $7, + serves_downloads = $8, + serves_enrichment = $9, + local_path = $10, + remote_path = $11, + searches_per_hour = $12, + updated_at = now() +WHERE code = $13 +RETURNING code, kind, protocol, label, enabled, base_url, api_key, username, password, category, timeout_ms, serves_search, serves_downloads, serves_enrichment, local_path, remote_path, searches_per_hour, created_at, updated_at +` + +type UpdateAcquisitionServiceParams struct { + Label string + Enabled bool + BaseUrl string + Username string + Category string + TimeoutMs int32 + ServesSearch bool + ServesDownloads bool + ServesEnrichment bool + LocalPath string + RemotePath string + SearchesPerHour int32 + Code string +} + +// Everything on the settings form except the credentials, which have their +// own write-only calls. +func (q *Queries) UpdateAcquisitionService(ctx context.Context, arg UpdateAcquisitionServiceParams) (AcquisitionService, error) { + row := q.db.QueryRow(ctx, updateAcquisitionService, + arg.Label, + arg.Enabled, + arg.BaseUrl, + arg.Username, + arg.Category, + arg.TimeoutMs, + arg.ServesSearch, + arg.ServesDownloads, + arg.ServesEnrichment, + arg.LocalPath, + arg.RemotePath, + arg.SearchesPerHour, + arg.Code, + ) + var i AcquisitionService + err := row.Scan( + &i.Code, + &i.Kind, + &i.Protocol, + &i.Label, + &i.Enabled, + &i.BaseUrl, + &i.ApiKey, + &i.Username, + &i.Password, + &i.Category, + &i.TimeoutMs, + &i.ServesSearch, + &i.ServesDownloads, + &i.ServesEnrichment, + &i.LocalPath, + &i.RemotePath, + &i.SearchesPerHour, + &i.CreatedAt, + &i.UpdatedAt, + ) + return i, err +} diff --git a/internal/db/dbq/models.go b/internal/db/dbq/models.go index bcaa80ca..64c066c6 100644 --- a/internal/db/dbq/models.go +++ b/internal/db/dbq/models.go @@ -195,6 +195,28 @@ type AcoustidSetting struct { UpdatedAt pgtype.Timestamptz } +type AcquisitionService struct { + Code string + Kind string + Protocol string + Label string + Enabled bool + BaseUrl string + ApiKey string + Username string + Password string + Category string + TimeoutMs int32 + ServesSearch bool + ServesDownloads bool + ServesEnrichment bool + LocalPath string + RemotePath string + SearchesPerHour int32 + CreatedAt pgtype.Timestamptz + UpdatedAt pgtype.Timestamptz +} + type Album struct { ID pgtype.UUID Title string diff --git a/internal/db/migrations/0079_acquisition_services.down.sql b/internal/db/migrations/0079_acquisition_services.down.sql new file mode 100644 index 00000000..49d5d65c --- /dev/null +++ b/internal/db/migrations/0079_acquisition_services.down.sql @@ -0,0 +1 @@ +DROP TABLE IF EXISTS acquisition_services; diff --git a/internal/db/migrations/0079_acquisition_services.up.sql b/internal/db/migrations/0079_acquisition_services.up.sql new file mode 100644 index 00000000..b3b1bc66 --- /dev/null +++ b/internal/db/migrations/0079_acquisition_services.up.sql @@ -0,0 +1,63 @@ +-- 0079_acquisition_services.up.sql — the indexers and download clients the +-- operator adds in Admin → Integrations (Scribe milestone #502, #5518). +-- +-- Minstrel replaces Lidarr, so it searches and downloads itself. Nothing is +-- seeded and nothing is copied from Lidarr: a connection is a row the operator +-- adds, from a list of what Minstrel can speak. Ported from Fabled +-- Librarian's acquisition_services (its migrations 0044, 0055, 0062, 0064, +-- 0070), which settled the shape: +-- +-- kind which list the row appears in: search (Indexers) or download +-- (Download clients). +-- protocol which client is built: prowlarr (its native /api/v1/search, +-- because Prowlarr has no aggregate Torznab feed), torznab (any +-- single feed), qbittorrent (the WebUI v2), exchange. +-- +-- FabledExchange is ONE row under Download clients with a switch per job: +-- downloading, searching its DHT index, and describing a torrent's files and +-- swarm before a grab. The serves_* switches mean something on an exchange +-- row only; every other row does the one job its kind names. +-- +-- Secrets (api_key, password) are write-only: no query the API answers with +-- returns them, only whether one is set. +CREATE TABLE acquisition_services ( + -- code is the stable identity: the protocol for the first of its kind, + -- then protocol-2, protocol-3. Legible in a log, never typed. + code text PRIMARY KEY, + kind text NOT NULL + CONSTRAINT acquisition_services_kind_check CHECK (kind IN ('search', 'download')), + protocol text NOT NULL + CONSTRAINT acquisition_services_protocol_check + CHECK (protocol IN ('prowlarr', 'torznab', 'qbittorrent', 'exchange')), + label text NOT NULL, + -- A new row is off: it has no address yet. + enabled boolean NOT NULL DEFAULT false, + -- The address as the operator's browser opens it. For a Torznab feed it + -- is the whole feed URL up to and including /api. + base_url text NOT NULL DEFAULT '', + api_key text NOT NULL DEFAULT '', + -- qBittorrent only. Both may be empty: a WebUI with auth bypassed for + -- local addresses takes neither. + username text NOT NULL DEFAULT '', + password text NOT NULL DEFAULT '', + -- Download clients: everything Minstrel adds is filed under it. + category text NOT NULL DEFAULT '', + timeout_ms integer NOT NULL DEFAULT 30000 + CONSTRAINT acquisition_services_timeout_check CHECK (timeout_ms BETWEEN 1000 AND 300000), + serves_search boolean NOT NULL DEFAULT true, + serves_downloads boolean NOT NULL DEFAULT true, + serves_enrichment boolean NOT NULL DEFAULT true, + -- One downloads folder, two names: where Minstrel has it mounted, and + -- where the client sees it (empty = the same path, nothing translated). + local_path text NOT NULL DEFAULT '/downloads' + CONSTRAINT acquisition_services_local_path_check CHECK (local_path LIKE '/%'), + remote_path text NOT NULL DEFAULT '' + CONSTRAINT acquisition_services_remote_path_check CHECK (remote_path = '' OR remote_path LIKE '/%'), + -- The most searches the schedule sends this source an hour; 0 = unpaced. + -- Prowlarr's indexers answer 429 when asked too often; an Exchange + -- searches its own database and starts unpaced. + searches_per_hour integer NOT NULL DEFAULT 120 + CONSTRAINT acquisition_services_searches_per_hour_check CHECK (searches_per_hour BETWEEN 0 AND 3600), + created_at timestamptz NOT NULL DEFAULT now(), + updated_at timestamptz NOT NULL DEFAULT now() +); diff --git a/internal/db/queries/acquisition.sql b/internal/db/queries/acquisition.sql new file mode 100644 index 00000000..139b4db5 --- /dev/null +++ b/internal/db/queries/acquisition.sql @@ -0,0 +1,76 @@ +-- Indexers and download clients (M502 #5518). + +-- name: ListAcquisitionServices :many +-- Returns no secret, only whether one is set: this feeds the settings page. +SELECT code, kind, protocol, label, enabled, base_url, + (api_key <> '')::boolean AS has_api_key, + (password <> '')::boolean AS has_password, + username, category, timeout_ms, + serves_search, serves_downloads, serves_enrichment, + local_path, remote_path, searches_per_hour, + created_at, updated_at +FROM acquisition_services +ORDER BY kind, label, code; + +-- name: GetAcquisitionService :one +-- The full row, credentials included. For building a client, never for a +-- response. +SELECT * FROM acquisition_services WHERE code = sqlc.arg(code); + +-- name: ListUsableAcquisitionServices :many +-- Every enabled, addressed service doing one job. An exchange row does a job +-- when its switch for it is on; any other row does the job its kind names. +-- Ordered by code so a fan-out is deterministic. +SELECT * FROM acquisition_services +WHERE enabled AND base_url <> '' + AND CASE WHEN protocol = 'exchange' THEN + CASE sqlc.arg(job)::text + WHEN 'search' THEN serves_search + WHEN 'download' THEN serves_downloads + WHEN 'enrichment' THEN serves_enrichment + ELSE false + END + ELSE kind = sqlc.arg(job)::text + END +ORDER BY code; + +-- name: CreateAcquisitionService :one +-- ON CONFLICT so the caller can try the next code when this one is taken. +INSERT INTO acquisition_services + (code, kind, protocol, label, category, timeout_ms, searches_per_hour) +VALUES + (sqlc.arg(code), sqlc.arg(kind), sqlc.arg(protocol), sqlc.arg(label), + sqlc.arg(category), sqlc.arg(timeout_ms), sqlc.arg(searches_per_hour)) +ON CONFLICT (code) DO NOTHING +RETURNING *; + +-- name: UpdateAcquisitionService :one +-- Everything on the settings form except the credentials, which have their +-- own write-only calls. +UPDATE acquisition_services SET + label = sqlc.arg(label), + enabled = sqlc.arg(enabled), + base_url = sqlc.arg(base_url), + username = sqlc.arg(username), + category = sqlc.arg(category), + timeout_ms = sqlc.arg(timeout_ms), + serves_search = sqlc.arg(serves_search), + serves_downloads = sqlc.arg(serves_downloads), + serves_enrichment = sqlc.arg(serves_enrichment), + local_path = sqlc.arg(local_path), + remote_path = sqlc.arg(remote_path), + searches_per_hour = sqlc.arg(searches_per_hour), + updated_at = now() +WHERE code = sqlc.arg(code) +RETURNING *; + +-- name: DeleteAcquisitionService :execrows +DELETE FROM acquisition_services WHERE code = sqlc.arg(code); + +-- name: SetAcquisitionServiceAPIKey :execrows +UPDATE acquisition_services SET api_key = sqlc.arg(api_key), updated_at = now() +WHERE code = sqlc.arg(code); + +-- name: SetAcquisitionServicePassword :execrows +UPDATE acquisition_services SET password = sqlc.arg(password), updated_at = now() +WHERE code = sqlc.arg(code); diff --git a/internal/dbtest/reset.go b/internal/dbtest/reset.go index 2d80157a..7824ccf0 100644 --- a/internal/dbtest/reset.go +++ b/internal/dbtest/reset.go @@ -102,6 +102,7 @@ var dataTables = []string{ "user_notifications", "user_notification_prefs", "user_notification_email_state", // #5346 + "acquisition_services", // M502 #5518 "tracks", "albums", "artists", diff --git a/internal/downloader/downloader_test.go b/internal/downloader/downloader_test.go new file mode 100644 index 00000000..5af62b57 --- /dev/null +++ b/internal/downloader/downloader_test.go @@ -0,0 +1,156 @@ +package downloader + +import ( + "context" + "errors" + "io" + "net/http" + "net/http/httptest" + "strings" + "sync/atomic" + "testing" +) + +func TestPathMap(t *testing.T) { + m := PathMap{Local: "/downloads", Remote: "/data/torrents/"} + for in, want := range map[string]string{ + "/downloads": "/data/torrents", + "/downloads/abc/x.mp3": "/data/torrents/abc/x.mp3", + "/downloads2/abc": "/downloads2/abc", + "/music/a": "/music/a", + "": "", + } { + if got := m.ToRemote(in); got != want { + t.Errorf("ToRemote(%q) = %q, want %q", in, got, want) + } + } + if got := m.ToLocal("/data/torrents/abc"); got != "/downloads/abc" { + t.Errorf("ToLocal = %q", got) + } + if (PathMap{Local: "/downloads", Remote: "/downloads/"}).Active() { + t.Error("the same folder under one name is not a mapping") + } +} + +// fakeQbit is a qBittorrent WebUI that wants a login, and that expires the +// session once to make the client log in again. +func fakeQbit(t *testing.T, expireOnce bool) (*httptest.Server, *atomic.Int32) { + t.Helper() + var logins atomic.Int32 + var expired atomic.Bool + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Header.Get("Referer") == "" { + w.WriteHeader(http.StatusForbidden) + return + } + switch r.URL.Path { + case "/api/v2/auth/login": + _ = r.ParseForm() + if r.Form.Get("password") != "pw" { + _, _ = w.Write([]byte("Fails.")) + return + } + logins.Add(1) + http.SetCookie(w, &http.Cookie{Name: "SID", Value: "s", Path: "/"}) + _, _ = w.Write([]byte("Ok.")) + case "/api/v2/torrents/info": + if _, err := r.Cookie("SID"); err != nil { + w.WriteHeader(http.StatusForbidden) + return + } + if expireOnce && !expired.Swap(true) { + w.WriteHeader(http.StatusForbidden) + return + } + if r.URL.Query().Get("category") != "minstrel" { + t.Errorf("category = %q", r.URL.Query().Get("category")) + } + _, _ = w.Write([]byte(`[{"hash": "ABC", "name": "x", "save_path": "/data/torrents/abc", "progress": 1}]`)) + case "/api/v2/torrents/add": + body, _ := io.ReadAll(r.Body) + if !strings.Contains(string(body), "/data/torrents/new") { + t.Errorf("add body has no translated save path: %s", body) + } + _, _ = w.Write([]byte("Ok.")) + default: + w.WriteHeader(http.StatusNotFound) + } + })) + t.Cleanup(srv.Close) + return srv, &logins +} + +func TestQBittorrent_LogsInAndLogsInAgainWhenTheSessionExpires(t *testing.T) { + srv, logins := fakeQbit(t, true) + d := Mapped(&Client{Config: Config{BaseURL: srv.URL, Username: "u", Password: "pw", Category: "minstrel"}}, + PathMap{Local: "/downloads", Remote: "/data/torrents"}) + ts, err := d.Torrents(context.Background()) + if err != nil { + t.Fatalf("torrents: %v", err) + } + if logins.Load() != 2 { + t.Errorf("logins = %d, want 2 (the second after the 403)", logins.Load()) + } + if len(ts) != 1 || ts[0].Hash != "abc" || ts[0].SavePath != "/downloads/abc" || !ts[0].Done() { + t.Errorf("torrents = %+v, want one, hash lowercased, path in Minstrel's terms", ts) + } + if err := d.AddTorrent(context.Background(), Add{Magnet: "magnet:?xt=urn:btih:abc", SavePath: "/downloads/new"}); err != nil { + t.Errorf("add: %v", err) + } +} + +func TestQBittorrent_AWrongPasswordIsAuth(t *testing.T) { + srv, _ := fakeQbit(t, false) + c := &Client{Config: Config{BaseURL: srv.URL, Password: "nope"}} + if _, err := c.Torrents(context.Background()); !errors.Is(err, ErrAuth) { + t.Fatalf("err = %v, want ErrAuth", err) + } +} + +func TestExchange_MapsDownloadsAndRefusesWhileRestoring(t *testing.T) { + var restoring, sentKey atomic.Bool + restoring.Store(true) + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Header.Get("Authorization") != "" { + sentKey.Store(true) + } + pending := "0" + if restoring.Load() { + pending = "3" + } + _, _ = io.WriteString(w, `{"enabled": true, "running": false, "held_reason": "disk full", "restore_pending": `+pending+`, "downloads": [ + {"infohash": "ABC", "name": "Album", "save_path": "/dl", "state": "seeding", "progress": 1, "length": 10, "bytes": 10, "category": "minstrel"}, + {"infohash": "DEF", "name": "Other", "save_path": "/dl", "state": "fast", "category": "someone-else"}, + {"infohash": "123", "name": "Broken", "save_path": "/dl", "failed": true, "fail_reason": "tracker gone", "category": "minstrel"} + ]}`) + })) + defer srv.Close() + + c := &Exchange{Config: Config{BaseURL: srv.URL, Category: "minstrel"}} + if _, err := c.Torrents(context.Background()); !errors.Is(err, ErrRestoring) { + t.Fatalf("while restoring: err = %v, want ErrRestoring", err) + } + if sentKey.Load() { + t.Error("an Authorization header was sent with no key set") + } + restoring.Store(false) + ts, err := c.Torrents(context.Background()) + if err != nil { + t.Fatalf("torrents: %v", err) + } + if len(ts) != 2 { + t.Fatalf("torrents = %+v, want the two in Minstrel's category", ts) + } + if ts[0].Hash != "abc" || ts[0].State != "uploading" || ts[0].ContentPath != "/dl/Album" || !strings.Contains(ts[0].EngineDown, "disk full") { + t.Errorf("first = %+v", ts[0]) + } + if !ts[1].Failed() || ts[1].FailReason != "tracker gone" { + t.Errorf("second = %+v, want failed with Exchange's reason", ts[1]) + } +} + +func TestFor_RefusesAnUnknownProtocol(t *testing.T) { + if _, err := For("transmission", Config{}); err == nil { + t.Fatal("want an error for a client Minstrel cannot drive") + } +} diff --git a/internal/downloader/driver.go b/internal/downloader/driver.go new file mode 100644 index 00000000..90e3bf9d --- /dev/null +++ b/internal/downloader/driver.go @@ -0,0 +1,198 @@ +// Package downloader drives the download clients Minstrel hands a grab to +// (M502 #5518): qBittorrent's WebUI v2, and FabledExchange's native +// /api/downloads. Ported from Fabled Librarian's internal/downloader. +// +// Both answer through one Driver, and Torrent is qBittorrent's shape: the +// Exchange driver maps its own downloads onto it, so the grab, the importer +// and the stall watchdog read one kind of record whichever client holds the +// torrent. +package downloader + +import ( + "context" + "errors" + "fmt" + "time" +) + +// Protocols a download client row may name. +const ( + KindQBittorrent = "qbittorrent" + KindExchange = "exchange" +) + +// ErrUnconfigured is a client with no address. +var ErrUnconfigured = errors.New("download client is not configured") + +// ErrUnreachable is a call that could not be made, or a refusal that is not +// about credentials. +var ErrUnreachable = errors.New("download client unreachable") + +// ErrAuth is credentials the client refused. +var ErrAuth = errors.New("download client refused the credentials") + +// ErrRestoring is an Exchange still bringing its downloads back after a +// restart: its list is incomplete, so nothing may be concluded from it. +var ErrRestoring = errors.New("download client is still restoring its downloads after a restart") + +// ErrDuplicate is a torrent the client already holds. +var ErrDuplicate = errors.New("download client already holds this torrent") + +// ErrUnsupported is a call this client has no equivalent for. +var ErrUnsupported = errors.New("download client cannot do that") + +// UserAgent names Minstrel in the client's logs. +const UserAgent = "Minstrel (self-hosted music server; download client)" + +// DefaultTimeout is what an unset Timeout means: a download client answers +// from its own state. +const DefaultTimeout = 15 * time.Second + +// Config is one download client. +type Config struct { + BaseURL string + // Username and Password are qBittorrent's. Both may be empty for a WebUI + // that bypasses auth for local addresses. + Username string + Password string + // APIKey is Exchange's, sent as a bearer only when set. + APIKey string + // Category files everything Minstrel adds, so its torrents can be told + // from the operator's own. + Category string + Timeout time.Duration +} + +// Driver is what Minstrel needs from a download client. +type Driver interface { + AddTorrent(ctx context.Context, a Add) error + // Torrents lists this client's torrents in Config.Category, or only the + // given hashes. + Torrents(ctx context.Context, hashes ...string) ([]Torrent, error) + // Connected says whether the client can reach any peer at all. + Connected(ctx context.Context) (bool, error) + Stop(ctx context.Context, hashes ...string) error + Start(ctx context.Context, hashes ...string) error + Reannounce(ctx context.Context, hashes ...string) error + Recheck(ctx context.Context, hashes ...string) error + Delete(ctx context.Context, deleteFiles bool, hashes ...string) error +} + +// FileSkipper sets which files of a torrent are downloaded. +type FileSkipper interface { + SkipFiles(ctx context.Context, hash string, indices []int) error + KeepFiles(ctx context.Context, hash string, indices []int) error +} + +// File is one file inside a torrent. +type File struct { + Index int + Path string + Size int64 +} + +// FileLister lists a torrent's files. +type FileLister interface { + Files(ctx context.Context, hash string) ([]File, error) +} + +// FileState is one file's progress. +type FileState struct { + Path string + Size int64 + Done int64 + Skipped bool +} + +// Unfinished says whether the file is wanted and not yet whole. +func (f FileState) Unfinished() bool { return !f.Skipped && f.Done < f.Size } + +// FileProgress reports per-file progress. +type FileProgress interface { + FileStates(ctx context.Context, hash string) ([]FileState, error) +} + +var ( + _ Driver = (*Client)(nil) + _ Driver = (*Exchange)(nil) + _ FileSkipper = (*Client)(nil) + _ FileSkipper = (*Exchange)(nil) + _ FileSkipper = (*mapped)(nil) + _ FileLister = (*Client)(nil) + _ FileLister = (*Exchange)(nil) + _ FileLister = (*mapped)(nil) + _ FileProgress = (*Client)(nil) + _ FileProgress = (*Exchange)(nil) + _ FileProgress = (*mapped)(nil) +) + +// For builds the driver a protocol names. +func For(protocol string, cfg Config) (Driver, error) { + switch protocol { + case KindQBittorrent: + return &Client{Config: cfg}, nil + case KindExchange: + return &Exchange{Config: cfg}, nil + } + return nil, fmt.Errorf("download client %q is not one Minstrel can drive", protocol) +} + +// Torrent is one torrent in qBittorrent's JSON shape. +type Torrent struct { + Hash string `json:"hash"` + Name string `json:"name"` + Size int64 `json:"size"` + Progress float64 `json:"progress"` + State string `json:"state"` + Category string `json:"category"` + Tags string `json:"tags"` + SavePath string `json:"save_path"` + ContentPath string `json:"content_path"` + AddedOn int64 `json:"added_on"` + NumSeeds int `json:"num_seeds"` + NumLeechs int `json:"num_leechs"` + AmountLeft int64 `json:"amount_left"` + Ratio float64 `json:"ratio"` + DLSpeed int64 `json:"dlspeed"` + ETA int64 `json:"eta"` + // EngineDown is why the client's whole engine is not moving anything, in + // words; empty when it is running. Exchange says so; qBittorrent does not. + EngineDown string `json:"-"` + // FailReason is the client's own reason for a failed torrent. + FailReason string `json:"-"` +} + +// Done says the torrent is whole. +func (t Torrent) Done() bool { return t.Progress >= 1 } + +// Failed says the client gave up on it. +func (t Torrent) Failed() bool { return t.State == "error" || t.State == "missingFiles" } + +// Fetching says it is still fetching metadata. +func (t Torrent) Fetching() bool { return t.State == "metaDL" || t.State == "forcedMetaDL" } + +// Alone says nobody is sharing an unfinished torrent. +func (t Torrent) Alone() bool { return !t.Done() && t.NumSeeds == 0 && t.NumLeechs == 0 } + +// Added is when the client took it. +func (t Torrent) Added() time.Time { + if t.AddedOn <= 0 { + return time.Time{} + } + return time.Unix(t.AddedOn, 0) +} + +// Add is one torrent to hand a client: a magnet or the .torrent bytes. +type Add struct { + Magnet string + Torrent []byte + // SavePath is where it lands, as Minstrel sees it; Mapped translates it. + // Required: a download with no save path lands outside the folder + // Minstrel imports from. + SavePath string + Tags []string + Paused bool + // StopAtMetadata asks qBittorrent to stop once it has the file list, so + // unwanted files can be skipped before any data moves. + StopAtMetadata bool +} diff --git a/internal/downloader/exchange.go b/internal/downloader/exchange.go new file mode 100644 index 00000000..e1b04fd0 --- /dev/null +++ b/internal/downloader/exchange.go @@ -0,0 +1,340 @@ +package downloader + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "net/url" + "path" + "strings" + "time" +) + +// Exchange drives FabledExchange's native /api/downloads: not its qBittorrent +// facade, so its own states, failure reasons and engine status come through. +type Exchange struct { + Config Config + // HTTP is injectable for tests. Nil takes a plain client. + HTTP *http.Client +} + +type exchangeItem struct { + InfoHash string `json:"infohash"` + Name string `json:"name"` + Bytes int64 `json:"bytes"` + Length int64 `json:"length"` + Progress float64 `json:"progress"` + State string `json:"state"` + Peers int `json:"peers"` + Category string `json:"category"` + Tags []string `json:"tags"` + SavePath string `json:"save_path"` + AddedAt time.Time `json:"added_at"` + Failed bool `json:"failed"` + FailReason string `json:"fail_reason"` + Ratio float64 `json:"ratio"` +} + +type exchangeSnapshot struct { + Enabled bool `json:"enabled"` + Running bool `json:"running"` + HeldReason string `json:"held_reason"` + InboundConns int64 `json:"inbound_conns"` + RestorePending int `json:"restore_pending"` + Downloads []exchangeItem `json:"downloads"` +} + +func (c *Exchange) AddTorrent(ctx context.Context, a Add) error { + if a.Magnet == "" && len(a.Torrent) == 0 { + return fmt.Errorf("add: neither a magnet nor torrent bytes") + } + if a.SavePath == "" { + return fmt.Errorf("add: no save path, which would land the download outside the folder Minstrel imports from") + } + if len(a.Torrent) > 0 { + q := url.Values{"save_path": {a.SavePath}} + if c.Config.Category != "" { + q.Set("category", c.Config.Category) + } + for _, t := range a.Tags { + q.Add("tag", t) + } + if a.Paused { + q.Set("paused", "true") + } + _, err := c.call(ctx, http.MethodPost, "/api/downloads/torrent?"+q.Encode(), "application/x-bittorrent", a.Torrent) + return err + } + body, err := json.Marshal(map[string]any{ + "magnet": a.Magnet, "category": c.Config.Category, "save_path": a.SavePath, + "tags": a.Tags, "paused": a.Paused, + }) + if err != nil { + return fmt.Errorf("add: %w", err) + } + _, err = c.call(ctx, http.MethodPost, "/api/downloads", "application/json", body) + return err +} + +// Torrents maps Exchange's downloads onto qBittorrent's shape. While Exchange +// is still restoring after a restart its list is incomplete, so it answers +// ErrRestoring rather than a list that looks like downloads vanished. +func (c *Exchange) Torrents(ctx context.Context, hashes ...string) ([]Torrent, error) { + snap, err := c.snapshot(ctx) + if err != nil { + return nil, err + } + if snap.RestorePending > 0 { + return nil, fmt.Errorf("%w (%d still to come back)", ErrRestoring, snap.RestorePending) + } + want := map[string]bool{} + for _, h := range hashes { + want[strings.ToLower(strings.TrimSpace(h))] = true + } + down := "" + switch { + case snap.Running: + case !snap.Enabled: + down = "the download engine is switched off in FabledExchange" + case snap.HeldReason != "": + down = "the download engine is held (" + snap.HeldReason + ") and resumes by itself" + default: + down = "the download engine is enabled but not running; FabledExchange is retrying it" + } + out := make([]Torrent, 0, len(snap.Downloads)) + for _, it := range snap.Downloads { + if c.Config.Category != "" && it.Category != c.Config.Category { + continue + } + t := asTorrent(it) + t.EngineDown = down + if len(want) > 0 && !want[t.Hash] { + continue + } + out = append(out, t) + } + return out, nil +} + +func asTorrent(it exchangeItem) Torrent { + content := it.SavePath + if it.Name != "" { + content = path.Join(it.SavePath, it.Name) + } + left := max(it.Length-it.Bytes, 0) + state := exchangeState(it.State, it.Progress) + if it.Failed { + state = "error" + } + var added int64 + if !it.AddedAt.IsZero() { + added = it.AddedAt.Unix() + } + return Torrent{ + Hash: strings.ToLower(it.InfoHash), Name: it.Name, Size: it.Length, + Progress: it.Progress, State: state, Category: it.Category, + Tags: strings.Join(it.Tags, ","), SavePath: it.SavePath, ContentPath: content, + AddedOn: added, NumLeechs: it.Peers, AmountLeft: left, + FailReason: it.FailReason, Ratio: it.Ratio, + } +} + +// exchangeState names Exchange's state in qBittorrent's vocabulary. +func exchangeState(tier string, progress float64) string { + switch tier { + case "fast", "slow": + return "downloading" + case "metadata": + return "metaDL" + case "stalled": + return "stalledDL" + case "queued": + return "queuedDL" + case "seeding": + return "uploading" + case "relocating": + return "moving" + case "paused": + if progress >= 1 { + return "pausedUP" + } + return "pausedDL" + } + return "unknown" +} + +// Connected is a running engine that has had an inbound connection: the +// nearest Exchange comes to qBittorrent's "connected". +func (c *Exchange) Connected(ctx context.Context) (bool, error) { + snap, err := c.snapshot(ctx) + if err != nil { + return false, err + } + return snap.Running && snap.InboundConns > 0, nil +} + +func (c *Exchange) Stop(ctx context.Context, hashes ...string) error { + return c.each(ctx, hashes, func(h string) error { + _, err := c.call(ctx, http.MethodPost, "/api/downloads/"+url.PathEscape(h)+"/pause", "", nil) + return err + }) +} + +func (c *Exchange) Start(ctx context.Context, hashes ...string) error { + return c.each(ctx, hashes, func(h string) error { + _, err := c.call(ctx, http.MethodPost, "/api/downloads/"+url.PathEscape(h)+"/resume", "", nil) + return err + }) +} + +func (c *Exchange) SkipFiles(ctx context.Context, hash string, indices []int) error { + return c.filePriority(ctx, hash, indices, "skip") +} + +func (c *Exchange) KeepFiles(ctx context.Context, hash string, indices []int) error { + return c.filePriority(ctx, hash, indices, "normal") +} + +func (c *Exchange) filePriority(ctx context.Context, hash string, indices []int, priority string) error { + if len(indices) == 0 { + return nil + } + body, err := json.Marshal(map[string]any{"indices": indices, "priority": priority}) + if err != nil { + return err + } + _, err = c.call(ctx, http.MethodPatch, "/api/downloads/"+url.PathEscape(hash)+"/files", "application/json", body) + return err +} + +func (c *Exchange) Files(ctx context.Context, hash string) ([]File, error) { + raw, err := c.files(ctx, hash) + if err != nil { + return nil, err + } + out := make([]File, len(raw)) + for i, f := range raw { + out[i] = File{Index: f.Index, Path: f.Path, Size: f.Length} + } + return out, nil +} + +func (c *Exchange) FileStates(ctx context.Context, hash string) ([]FileState, error) { + raw, err := c.files(ctx, hash) + if err != nil { + return nil, err + } + out := make([]FileState, len(raw)) + for i, f := range raw { + if f.Bytes == nil { + return nil, fmt.Errorf("%w: FabledExchange does not say how much of each file it has", ErrUnsupported) + } + out[i] = FileState{Path: f.Path, Size: f.Length, Done: *f.Bytes, Skipped: f.Priority == "skip"} + } + return out, nil +} + +type exchangeFile struct { + Index int `json:"index"` + Path string `json:"path"` + Length int64 `json:"length"` + Bytes *int64 `json:"bytes"` + Priority string `json:"priority"` +} + +func (c *Exchange) files(ctx context.Context, hash string) ([]exchangeFile, error) { + body, err := c.call(ctx, http.MethodGet, "/api/downloads/"+url.PathEscape(hash), "", nil) + if err != nil { + return nil, err + } + var detail struct { + Files []exchangeFile `json:"files"` + } + if err := json.Unmarshal(body, &detail); err != nil { + return nil, fmt.Errorf("%w: parse download detail: %v", ErrUnreachable, err) + } + return detail.Files, nil +} + +func (c *Exchange) Reannounce(context.Context, ...string) error { + return fmt.Errorf("%w: FabledExchange schedules its own announces", ErrUnsupported) +} + +func (c *Exchange) Recheck(context.Context, ...string) error { + return fmt.Errorf("%w: FabledExchange has no recheck", ErrUnsupported) +} + +func (c *Exchange) Delete(ctx context.Context, deleteFiles bool, hashes ...string) error { + return c.each(ctx, hashes, func(h string) error { + p := "/api/downloads/" + url.PathEscape(h) + if deleteFiles { + p += "?delete_files=true" + } + _, err := c.call(ctx, http.MethodDelete, p, "", nil) + return err + }) +} + +func (c *Exchange) each(ctx context.Context, hashes []string, do func(string) error) error { + for _, h := range hashes { + if ctx.Err() != nil { + return ctx.Err() + } + if err := do(strings.ToLower(strings.TrimSpace(h))); err != nil { + return err + } + } + return nil +} + +func (c *Exchange) snapshot(ctx context.Context) (exchangeSnapshot, error) { + var snap exchangeSnapshot + body, err := c.call(ctx, http.MethodGet, "/api/downloads", "", nil) + if err != nil { + return snap, err + } + if err := json.Unmarshal(body, &snap); err != nil { + return snap, fmt.Errorf("%w: parse download list: %v", ErrUnreachable, err) + } + return snap, nil +} + +func (c *Exchange) call(ctx context.Context, method, p, contentType string, body []byte) ([]byte, error) { + if c.Config.BaseURL == "" { + return nil, ErrUnconfigured + } + httpc := c.HTTP + if httpc == nil { + httpc = &http.Client{} + } + status, out, err := send(ctx, httpc, c.Config.Timeout, method, + strings.TrimRight(c.Config.BaseURL, "/")+p, contentType, body, func(h http.Header) { + h.Set("Accept", "application/json") + if c.Config.APIKey != "" { + h.Set("Authorization", "Bearer "+c.Config.APIKey) + } + }) + if err != nil { + return nil, err + } + if status >= 200 && status < 300 { + return out, nil + } + var e struct { + Error string `json:"error"` + Reason string `json:"reason"` + } + _ = json.Unmarshal(out, &e) + said := strings.TrimSpace(e.Error) + if said == "" { + said = fmt.Sprintf("HTTP %d", status) + } + switch { + case status == http.StatusUnauthorized || status == http.StatusForbidden: + return nil, fmt.Errorf("%w: FabledExchange refused the API key (%s)", ErrAuth, said) + case status == http.StatusConflict && e.Reason == "duplicate": + return nil, fmt.Errorf("%w: %s", ErrDuplicate, said) + } + return nil, fmt.Errorf("%w: %s", ErrUnreachable, said) +} diff --git a/internal/downloader/pathmap.go b/internal/downloader/pathmap.go new file mode 100644 index 00000000..be43772e --- /dev/null +++ b/internal/downloader/pathmap.go @@ -0,0 +1,107 @@ +package downloader + +import ( + "context" + "path" + "strings" +) + +// PathMap is one downloads folder under two names: where Minstrel has it +// mounted (Local) and where the client sees it (Remote). The two usually run +// in different containers. Every path crossing between them swaps one prefix +// for the other. +type PathMap struct { + Local string + Remote string +} + +// Active says the two names differ, so there is something to translate. +func (m PathMap) Active() bool { + return m.Remote != "" && m.Local != "" && path.Clean(m.Remote) != path.Clean(m.Local) +} + +// ToRemote is a Minstrel path as the client names it. +func (m PathMap) ToRemote(p string) string { return swap(p, m.Local, m.Remote) } + +// ToLocal is a client's path as Minstrel names it. +func (m PathMap) ToLocal(p string) string { return swap(p, m.Remote, m.Local) } + +// swap replaces the prefix from with to on segment boundaries +// (/downloads2 is not inside /downloads). A path outside from is unchanged. +func swap(p, from, to string) string { + if p == "" || from == "" || to == "" { + return p + } + from, to = path.Clean(from), path.Clean(to) + clean := path.Clean(p) + if clean == from { + return to + } + if from == "/" { + return path.Join(to, clean) + } + if rest, ok := strings.CutPrefix(clean, from+"/"); ok { + return path.Join(to, rest) + } + return p +} + +// Mapped wraps d so every path it is given or answers with is translated. +// File indices and in-torrent paths need no translation. +func Mapped(d Driver, m PathMap) Driver { + if !m.Active() { + return d + } + return &mapped{Driver: d, m: m} +} + +type mapped struct { + Driver + m PathMap +} + +func (c *mapped) AddTorrent(ctx context.Context, a Add) error { + a.SavePath = c.m.ToRemote(a.SavePath) + return c.Driver.AddTorrent(ctx, a) +} + +func (c *mapped) Torrents(ctx context.Context, hashes ...string) ([]Torrent, error) { + ts, err := c.Driver.Torrents(ctx, hashes...) + for i := range ts { + ts[i].SavePath = c.m.ToLocal(ts[i].SavePath) + ts[i].ContentPath = c.m.ToLocal(ts[i].ContentPath) + } + return ts, err +} + +func (c *mapped) SkipFiles(ctx context.Context, hash string, indices []int) error { + fs, ok := c.Driver.(FileSkipper) + if !ok { + return ErrUnsupported + } + return fs.SkipFiles(ctx, hash, indices) +} + +func (c *mapped) KeepFiles(ctx context.Context, hash string, indices []int) error { + fs, ok := c.Driver.(FileSkipper) + if !ok { + return ErrUnsupported + } + return fs.KeepFiles(ctx, hash, indices) +} + +func (c *mapped) Files(ctx context.Context, hash string) ([]File, error) { + fl, ok := c.Driver.(FileLister) + if !ok { + return nil, ErrUnsupported + } + return fl.Files(ctx, hash) +} + +func (c *mapped) FileStates(ctx context.Context, hash string) ([]FileState, error) { + fp, ok := c.Driver.(FileProgress) + if !ok { + return nil, ErrUnsupported + } + return fp.FileStates(ctx, hash) +} diff --git a/internal/downloader/qbittorrent.go b/internal/downloader/qbittorrent.go new file mode 100644 index 00000000..73d1eb6b --- /dev/null +++ b/internal/downloader/qbittorrent.go @@ -0,0 +1,367 @@ +package downloader + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "mime/multipart" + "net/http" + "net/http/cookiejar" + "net/url" + "strconv" + "strings" + "sync" + "time" +) + +// Client drives one qBittorrent through its WebUI API v2. +type Client struct { + Config Config + + mu sync.Mutex + // http carries the SID cookie once logged in; nil until then. + http *http.Client +} + +// AddTorrent hands qBittorrent a magnet or .torrent bytes, filed under the +// configured category with the save path already set. +func (c *Client) AddTorrent(ctx context.Context, a Add) error { + if a.Magnet == "" && len(a.Torrent) == 0 { + return fmt.Errorf("add: neither a magnet nor torrent bytes") + } + if a.SavePath == "" { + return fmt.Errorf("add: no save path, which would land the download outside the folder Minstrel imports from") + } + body := &bytes.Buffer{} + form := multipart.NewWriter(body) + fields := map[string]string{ + "savepath": a.SavePath, + "category": c.Config.Category, + // Both, because qBittorrent 5 renamed paused to stopped and an older + // one ignores the field it does not know. + "paused": strconv.FormatBool(a.Paused), + "stopped": strconv.FormatBool(a.Paused), + } + if len(a.Tags) > 0 { + fields["tags"] = strings.Join(a.Tags, ",") + } + if a.StopAtMetadata && !a.Paused { + fields["stopCondition"] = "MetadataReceived" + } + if a.Magnet != "" { + fields["urls"] = a.Magnet + } + for k, v := range fields { + if v == "" { + continue + } + if err := form.WriteField(k, v); err != nil { + return fmt.Errorf("add: build form: %w", err) + } + } + if len(a.Torrent) > 0 { + part, err := form.CreateFormFile("torrents", "release.torrent") + if err != nil { + return fmt.Errorf("add: build form: %w", err) + } + if _, err := part.Write(a.Torrent); err != nil { + return fmt.Errorf("add: build form: %w", err) + } + } + if err := form.Close(); err != nil { + return fmt.Errorf("add: build form: %w", err) + } + resp, err := c.post(ctx, "/api/v2/torrents/add", form.FormDataContentType(), body.Bytes()) + if err != nil { + return err + } + // qBittorrent refuses with HTTP 200 and the body "Fails.". + if strings.HasPrefix(strings.TrimSpace(string(resp)), "Fails") { + return fmt.Errorf("%w: the client refused the torrent", ErrUnreachable) + } + return nil +} + +// Torrents lists the configured category's torrents, or the given hashes. +func (c *Client) Torrents(ctx context.Context, hashes ...string) ([]Torrent, error) { + params := url.Values{} + if c.Config.Category != "" { + params.Set("category", c.Config.Category) + } + if len(hashes) > 0 { + params.Set("hashes", strings.Join(hashes, "|")) + } + body, err := c.get(ctx, "/api/v2/torrents/info", params) + if err != nil { + return nil, err + } + var out []Torrent + if err := json.Unmarshal(body, &out); err != nil { + return nil, fmt.Errorf("%w: parse torrent list: %v", ErrUnreachable, err) + } + for i := range out { + out[i].Hash = strings.ToLower(out[i].Hash) + } + return out, nil +} + +// Connected reads qBittorrent's own connection status. +func (c *Client) Connected(ctx context.Context) (bool, error) { + body, err := c.get(ctx, "/api/v2/transfer/info", nil) + if err != nil { + return false, err + } + var info struct { + ConnectionStatus string `json:"connection_status"` + } + if err := json.Unmarshal(body, &info); err != nil { + return false, fmt.Errorf("%w: parse transfer info: %v", ErrUnreachable, err) + } + return info.ConnectionStatus == "connected", nil +} + +// Stop and Start try qBittorrent 5's names first and fall back to 4's. +func (c *Client) Stop(ctx context.Context, hashes ...string) error { + return c.command(ctx, hashes, "/api/v2/torrents/stop", "/api/v2/torrents/pause") +} + +func (c *Client) Start(ctx context.Context, hashes ...string) error { + return c.command(ctx, hashes, "/api/v2/torrents/start", "/api/v2/torrents/resume") +} + +func (c *Client) Reannounce(ctx context.Context, hashes ...string) error { + return c.command(ctx, hashes, "/api/v2/torrents/reannounce", "/api/v2/torrents/reannounce") +} + +func (c *Client) Recheck(ctx context.Context, hashes ...string) error { + return c.command(ctx, hashes, "/api/v2/torrents/recheck", "/api/v2/torrents/recheck") +} + +func (c *Client) Delete(ctx context.Context, deleteFiles bool, hashes ...string) error { + if len(hashes) == 0 { + return nil + } + params := url.Values{ + "hashes": {strings.Join(hashes, "|")}, + "deleteFiles": {strconv.FormatBool(deleteFiles)}, + } + _, err := c.post(ctx, "/api/v2/torrents/delete", "application/x-www-form-urlencoded", []byte(params.Encode())) + return err +} + +func (c *Client) SkipFiles(ctx context.Context, hash string, indices []int) error { + return c.filePriority(ctx, hash, indices, "0") +} + +func (c *Client) KeepFiles(ctx context.Context, hash string, indices []int) error { + return c.filePriority(ctx, hash, indices, "1") +} + +func (c *Client) filePriority(ctx context.Context, hash string, indices []int, priority string) error { + if len(indices) == 0 { + return nil + } + ids := make([]string, len(indices)) + for i, n := range indices { + ids[i] = strconv.Itoa(n) + } + params := url.Values{"hash": {hash}, "id": {strings.Join(ids, "|")}, "priority": {priority}} + _, err := c.post(ctx, "/api/v2/torrents/filePrio", "application/x-www-form-urlencoded", []byte(params.Encode())) + return err +} + +func (c *Client) Files(ctx context.Context, hash string) ([]File, error) { + raw, err := c.files(ctx, hash) + if err != nil { + return nil, err + } + out := make([]File, len(raw)) + for i, f := range raw { + out[i] = File{Index: f.Index, Path: f.Name, Size: f.Size} + } + return out, nil +} + +func (c *Client) FileStates(ctx context.Context, hash string) ([]FileState, error) { + raw, err := c.files(ctx, hash) + if err != nil { + return nil, err + } + out := make([]FileState, len(raw)) + for i, f := range raw { + done := f.Size + if f.Progress < 1 { + // Progress is a float; never round an unfinished file up to whole. + done = min(int64(f.Progress*float64(f.Size)), f.Size-1) + } + out[i] = FileState{Path: f.Name, Size: f.Size, Done: max(done, 0), Skipped: f.Priority == 0} + } + return out, nil +} + +type qbitFile struct { + Index int `json:"index"` + Name string `json:"name"` + Size int64 `json:"size"` + Progress float64 `json:"progress"` + Priority int `json:"priority"` +} + +func (c *Client) files(ctx context.Context, hash string) ([]qbitFile, error) { + body, err := c.get(ctx, "/api/v2/torrents/files", url.Values{"hash": {hash}}) + if err != nil { + return nil, err + } + var raw []qbitFile + if err := json.Unmarshal(body, &raw); err != nil { + return nil, fmt.Errorf("%w: parse file list: %v", ErrUnreachable, err) + } + return raw, nil +} + +func (c *Client) command(ctx context.Context, hashes []string, primary, fallback string) error { + if len(hashes) == 0 { + return nil + } + params := []byte(url.Values{"hashes": {strings.Join(hashes, "|")}}.Encode()) + _, err := c.post(ctx, primary, "application/x-www-form-urlencoded", params) + var status statusError + if errors.As(err, &status) && (status.code == http.StatusNotFound || status.code == http.StatusMethodNotAllowed) { + _, err = c.post(ctx, fallback, "application/x-www-form-urlencoded", params) + } + return err +} + +type statusError struct { + code int + what string +} + +func (e statusError) Error() string { return fmt.Sprintf("%s: HTTP %d", e.what, e.code) } + +func (e statusError) Unwrap() error { return ErrUnreachable } + +func (c *Client) get(ctx context.Context, path string, params url.Values) ([]byte, error) { + endpoint := path + if len(params) > 0 { + endpoint += "?" + params.Encode() + } + return c.do(ctx, http.MethodGet, endpoint, "", nil) +} + +func (c *Client) post(ctx context.Context, path, contentType string, body []byte) ([]byte, error) { + return c.do(ctx, http.MethodPost, path, contentType, body) +} + +// do logs in when there is no session, and once more on a 403, which is how +// qBittorrent says a session expired. +func (c *Client) do(ctx context.Context, method, path, contentType string, body []byte) ([]byte, error) { + if c.Config.BaseURL == "" { + return nil, ErrUnconfigured + } + if err := c.ensureSession(ctx, false); err != nil { + return nil, err + } + out, err := c.call(ctx, method, path, contentType, body) + var status statusError + if errors.As(err, &status) && status.code == http.StatusForbidden { + if err := c.ensureSession(ctx, true); err != nil { + return nil, err + } + return c.call(ctx, method, path, contentType, body) + } + return out, err +} + +func (c *Client) ensureSession(ctx context.Context, force bool) error { + c.mu.Lock() + defer c.mu.Unlock() + if c.http != nil && !force { + return nil + } + jar, err := cookiejar.New(nil) + if err != nil { + return fmt.Errorf("cookie jar: %w", err) + } + httpc := &http.Client{Jar: jar} + params := url.Values{"username": {c.Config.Username}, "password": {c.Config.Password}} + out, err := c.callWith(ctx, httpc, http.MethodPost, "/api/v2/auth/login", + "application/x-www-form-urlencoded", []byte(params.Encode())) + if err != nil { + var status statusError + if errors.As(err, &status) && status.code == http.StatusForbidden { + return fmt.Errorf("%w: too many failed attempts, or this address is banned", ErrAuth) + } + return err + } + if strings.HasPrefix(strings.TrimSpace(string(out)), "Fails") { + return fmt.Errorf("%w: wrong username or password", ErrAuth) + } + c.http = httpc + return nil +} + +func (c *Client) call(ctx context.Context, method, path, contentType string, body []byte) ([]byte, error) { + c.mu.Lock() + httpc := c.http + c.mu.Unlock() + if httpc == nil { + httpc = &http.Client{} + } + return c.callWith(ctx, httpc, method, path, contentType, body) +} + +func (c *Client) callWith(ctx context.Context, httpc *http.Client, method, path, contentType string, body []byte) ([]byte, error) { + base := strings.TrimRight(c.Config.BaseURL, "/") + status, out, err := send(ctx, httpc, c.Config.Timeout, method, base+path, contentType, body, func(h http.Header) { + // qBittorrent's CSRF check refuses a request without a matching + // Referer. + h.Set("Referer", base) + }) + if err != nil { + return nil, err + } + if status < 200 || status >= 300 { + return nil, statusError{code: status, what: path} + } + return out, nil +} + +// send makes one request to a download client under its timeout, lets the +// caller add headers, and reads at most 16 MB. It returns the status for the +// caller to read, because each client's refusals are its own. +func send(ctx context.Context, httpc *http.Client, timeout time.Duration, method, target, contentType string, body []byte, set func(http.Header)) (int, []byte, error) { + if timeout <= 0 { + timeout = DefaultTimeout + } + ctx, cancel := context.WithTimeout(ctx, timeout) + defer cancel() + var reader io.Reader + if body != nil { + reader = bytes.NewReader(body) + } + req, err := http.NewRequestWithContext(ctx, method, target, reader) + if err != nil { + return 0, nil, fmt.Errorf("%w: build request: %v", ErrUnreachable, err) + } + req.Header.Set("User-Agent", UserAgent) + if contentType != "" { + req.Header.Set("Content-Type", contentType) + } + if set != nil { + set(req.Header) + } + resp, err := httpc.Do(req) + if err != nil { + return 0, nil, fmt.Errorf("%w: %v", ErrUnreachable, err) + } + defer func() { _ = resp.Body.Close() }() + out, err := io.ReadAll(io.LimitReader(resp.Body, 16<<20)) + if err != nil { + return 0, nil, fmt.Errorf("%w: read body: %v", ErrUnreachable, err) + } + return resp.StatusCode, out, nil +} diff --git a/internal/exchange/exchange.go b/internal/exchange/exchange.go new file mode 100644 index 00000000..a5417b73 --- /dev/null +++ b/internal/exchange/exchange.go @@ -0,0 +1,182 @@ +// Package exchange talks to FabledExchange's native API for the two jobs that +// are not downloading (M502 #5518): searching its DHT index (/api/search) and +// describing one torrent (/api/torrents/{infohash}). Downloading is +// internal/downloader's. Ported from Fabled Librarian's internal/exchange. +// +// The native search rather than Exchange's Torznab feed, because it reports +// whether a swarm was measured and how many files a torrent holds, which the +// feed cannot say. +package exchange + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "log/slog" + "net/http" + "strings" + "time" + + "git.fabledsword.com/bvandeusen/minstrel/internal/indexer" +) + +// ErrUnconfigured is an Exchange with no address. +var ErrUnconfigured = errors.New("exchange is not configured") + +// ErrUnreachable is a call that could not be made or was not understood. +var ErrUnreachable = errors.New("exchange unreachable") + +// ErrAuth is an API key Exchange refused, or a secured Exchange asked with +// none. +var ErrAuth = errors.New("exchange refused the API key") + +// ErrUnknown is a torrent Exchange has not crawled: a JSON 404, which is an +// answer and not a fault. +var ErrUnknown = errors.New("exchange has not crawled this torrent") + +// UserAgent names Minstrel in Exchange's logs. +const UserAgent = "Minstrel (self-hosted music server; release search)" + +// DefaultTimeout is what an unset Timeout means. Exchange answers from its +// own database. +const DefaultTimeout = 20 * time.Second + +// Config is one Exchange. +type Config struct { + BaseURL string + // APIKey is optional and sent only when set: an empty bearer makes a + // secured Exchange answer 401. + APIKey string + Timeout time.Duration +} + +// Client asks one Exchange. +type Client struct { + Config Config + // HTTP is injectable for tests. Nil takes http.DefaultClient. + HTTP *http.Client + // Code is the acquisition_services row, stamped onto search results. + Code string + Logger *slog.Logger +} + +func (c *Client) log() *slog.Logger { + if c == nil || c.Logger == nil { + return slog.Default() + } + return c.Logger +} + +// Detail is what Exchange knows about one torrent. +type Detail struct { + InfoHash string `json:"infohash"` + Name string `json:"name"` + Size int64 `json:"size"` + FilesCount int `json:"files_count"` + Seeders int `json:"seeders"` + SwarmKnown bool `json:"swarm_known"` + Leechers int `json:"leechers"` + LeechersKnown bool `json:"leechers_known"` + ContentType string `json:"content_type"` + Files []DetailFile `json:"files"` +} + +// DetailFile is one file inside a torrent. +type DetailFile struct { + Index int `json:"index"` + Path string `json:"path"` + Size int64 `json:"size"` +} + +// Torrent asks Exchange about one v1 infohash. +func (c *Client) Torrent(ctx context.Context, infohash string) (Detail, error) { + if strings.TrimSpace(c.Config.BaseURL) == "" { + return Detail{}, ErrUnconfigured + } + hash := indexer.NormalizeHash(infohash) + if len(hash) != 40 || !indexer.IsInfoHash(hash) { + return Detail{}, fmt.Errorf("%w: %q is not a 40-character infohash", ErrUnreachable, infohash) + } + body, err := c.get(ctx, "/api/torrents/"+hash) + if err != nil { + return Detail{}, err + } + var out Detail + if err := json.Unmarshal(body, &out); err != nil { + return Detail{}, fmt.Errorf("%w: parse torrent detail: %v", ErrUnreachable, err) + } + if out.InfoHash == "" { + return Detail{}, ErrUnknown + } + return out, nil +} + +func (c *Client) get(ctx context.Context, path string) ([]byte, error) { + timeout := c.Config.Timeout + if timeout <= 0 { + timeout = DefaultTimeout + } + ctx, cancel := context.WithTimeout(ctx, timeout) + defer cancel() + + target := strings.TrimRight(c.Config.BaseURL, "/") + path + req, err := http.NewRequestWithContext(ctx, http.MethodGet, target, nil) + if err != nil { + return nil, fmt.Errorf("%w: %v", ErrUnreachable, err) + } + req.Header.Set("User-Agent", UserAgent) + req.Header.Set("Accept", "application/json") + if key := strings.TrimSpace(c.Config.APIKey); key != "" { + req.Header.Set("Authorization", "Bearer "+key) + } + client := c.HTTP + if client == nil { + client = http.DefaultClient + } + res, err := client.Do(req) + if err != nil { + return nil, fmt.Errorf("%w: %v", ErrUnreachable, err) + } + defer func() { _ = res.Body.Close() }() + body, err := io.ReadAll(io.LimitReader(res.Body, 1<<20)) + if err != nil { + return nil, fmt.Errorf("%w: read answer: %v", ErrUnreachable, err) + } + switch { + case res.StatusCode == http.StatusNotFound: + // A 404 that is not JSON came from something else at this address, + // which must not look like a healthy Exchange that lacks a torrent. + if !looksLikeJSON(body) { + return nil, fmt.Errorf("%w: something at this address answered 404, "+ + "but it was not FabledExchange", ErrUnreachable) + } + return nil, ErrUnknown + case res.StatusCode == http.StatusUnauthorized || res.StatusCode == http.StatusForbidden: + return nil, fmt.Errorf("%w: %s", ErrAuth, + message(body, "set this FabledExchange's API key in Minstrel's settings")) + case res.StatusCode == http.StatusServiceUnavailable: + return nil, fmt.Errorf("%w: %s", ErrUnreachable, message(body, "no index is configured")) + case res.StatusCode < 200 || res.StatusCode >= 300: + return nil, fmt.Errorf("%w: %s", ErrUnreachable, + message(body, fmt.Sprintf("returned %d", res.StatusCode))) + } + return body, nil +} + +// message is Exchange's own {"error": "..."} text, or fallback. +func message(body []byte, fallback string) string { + var env struct { + Error string `json:"error"` + } + if err := json.Unmarshal(body, &env); err == nil && strings.TrimSpace(env.Error) != "" { + return strings.TrimSpace(env.Error) + } + return fallback +} + +func looksLikeJSON(body []byte) bool { + t := strings.TrimSpace(string(body)) + return strings.HasPrefix(t, "{") || strings.HasPrefix(t, "[") +} diff --git a/internal/exchange/exchange_test.go b/internal/exchange/exchange_test.go new file mode 100644 index 00000000..bfd5aa92 --- /dev/null +++ b/internal/exchange/exchange_test.go @@ -0,0 +1,108 @@ +package exchange + +import ( + "context" + "errors" + "fmt" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "git.fabledsword.com/bvandeusen/minstrel/internal/indexer" +) + +func TestTorrent_SendsTheKeyOnlyWhenSet(t *testing.T) { + auth := make(chan string, 2) + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + auth <- r.Header.Get("Authorization") + _, _ = w.Write([]byte(`{"infohash": "` + strings.Repeat("a", 40) + `", "files_count": 12}`)) + })) + defer srv.Close() + + for _, key := range []string{"", "k"} { + c := &Client{Config: Config{BaseURL: srv.URL, APIKey: key}} + d, err := c.Torrent(context.Background(), strings.Repeat("A", 40)) + if err != nil || d.FilesCount != 12 { + t.Fatalf("key %q: detail %+v err %v", key, d, err) + } + } + if without, with := <-auth, <-auth; without != "" || with != "Bearer k" { + t.Errorf("Authorization = %q then %q, want none without a key and a bearer with one", without, with) + } +} + +func TestTorrent_TellsAnUncrawledTorrentFromAnotherService(t *testing.T) { + for name, tc := range map[string]struct { + status int + body string + want error + }{ + "Exchange has not crawled it": {http.StatusNotFound, `{"error": "unknown torrent"}`, ErrUnknown}, + "not FabledExchange": {http.StatusNotFound, `404`, ErrUnreachable}, + "a refused key": {http.StatusUnauthorized, `{"error": "bad key"}`, ErrAuth}, + } { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(tc.status) + _, _ = w.Write([]byte(tc.body)) + })) + _, err := (&Client{Config: Config{BaseURL: srv.URL}}).Torrent(context.Background(), strings.Repeat("0", 40)) + srv.Close() + if !errors.Is(err, tc.want) { + t.Errorf("%s: err = %v, want %v", name, err, tc.want) + } + } +} + +func TestSearch_PagesAndKeepsMeasuredSwarmsOnly(t *testing.T) { + queries := make(chan string, 4) + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + queries <- r.URL.RawQuery + offset := r.URL.Query().Get("offset") + var rows []string + if offset == "0" { + for i := 0; i < SearchLimit; i++ { + rows = append(rows, fmt.Sprintf(`{"infohash": "%040x", "name": "r%d", "content_type": "music", "seeders": 5, "swarm_known": %t}`, i, i, i == 0)) + } + } else { + // A repeat of page one's first hash, which must not be listed twice. + rows = append(rows, fmt.Sprintf(`{"infohash": "%040x", "name": "again"}`, 0)) + } + _, _ = fmt.Fprintf(w, `{"total": %d, "results": [%s]}`, SearchLimit+1, strings.Join(rows, ",")) + })) + defer srv.Close() + + c := &Client{Code: "exchange", Config: Config{BaseURL: srv.URL}} + results, err := c.Search(context.Background(), indexer.Query{Text: "x", Categories: indexer.MusicCategories}) + if err != nil { + t.Fatalf("search: %v", err) + } + close(queries) + var asked []string + for q := range queries { + asked = append(asked, q) + } + if len(asked) != 2 || !strings.Contains(asked[0], "type=music") { + t.Errorf("queries = %v, want two pages asking for music", asked) + } + if len(results) != SearchLimit { + t.Errorf("results = %d, want %d (the repeat dropped)", len(results), SearchLimit) + } + if r := results[0]; !r.SwarmKnown || r.Seeders != 5 || r.Category != 3000 || r.Source != "exchange" || !strings.HasPrefix(r.Magnet, "magnet:") { + t.Errorf("first = %+v", r) + } + if r := results[1]; r.SwarmKnown || r.Seeders != 0 { + t.Errorf("an unmeasured swarm reported seeders: %+v", r) + } +} + +func TestSearch_ARefusedKeyReadsAsAuth(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusForbidden) + })) + defer srv.Close() + _, err := (&Client{Config: Config{BaseURL: srv.URL}}).Search(context.Background(), indexer.Query{Text: "x"}) + if !errors.Is(err, indexer.ErrAuth) { + t.Fatalf("err = %v, want indexer.ErrAuth", err) + } +} diff --git a/internal/exchange/search.go b/internal/exchange/search.go new file mode 100644 index 00000000..d2fc8724 --- /dev/null +++ b/internal/exchange/search.go @@ -0,0 +1,169 @@ +package exchange + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "net/url" + "strings" + + "git.fabledsword.com/bvandeusen/minstrel/internal/indexer" +) + +// SearchLimit is one page of /api/search; SearchDepth is the most read across +// pages. A query matching more than SearchDepth is too broad to judge anyway. +const ( + SearchLimit = 100 + SearchDepth = 300 +) + +type searchResult struct { + InfoHash string `json:"infohash"` + Name string `json:"name"` + Size int64 `json:"size"` + FilesCount int `json:"files_count"` + ContentType string `json:"content_type"` + Seeders int `json:"seeders"` + SwarmKnown bool `json:"swarm_known"` + Leechers int `json:"leechers"` + LeechersKnown bool `json:"leechers_known"` +} + +type searchPage struct { + Total int `json:"total"` + Results []searchResult `json:"results"` +} + +// Search asks Exchange's DHT index. Every result is a magnet, so there is no +// .torrent to fetch through it. A later page that fails keeps the earlier +// ones; a first page that fails is the search failing. +func (c *Client) Search(ctx context.Context, q indexer.Query) ([]indexer.Result, error) { + if strings.TrimSpace(c.Config.BaseURL) == "" { + return nil, indexer.ErrUnconfigured + } + params := url.Values{"q": {q.Text}, "limit": {fmt.Sprint(SearchLimit)}} + if t := contentType(q.Categories); t != "" { + params.Set("type", t) + } + var ( + out []indexer.Result + seen = map[string]bool{} + ) + for offset := 0; offset < SearchDepth; offset += SearchLimit { + params.Set("offset", fmt.Sprint(offset)) + page, err := c.searchPage(ctx, params) + if err != nil { + if offset == 0 { + return nil, err + } + c.log().Warn("exchange: a later page of a search could not be read; keeping the pages before it", + "q", q.Text, "offset", offset, "err", err) + break + } + for _, r := range page.Results { + res, ok := c.result(r) + if !ok || seen[res.InfoHash] { + continue + } + seen[res.InfoHash] = true + out = append(out, res) + } + if len(page.Results) < SearchLimit || offset+SearchLimit >= page.Total { + break + } + } + return out, nil +} + +func (c *Client) searchPage(ctx context.Context, params url.Values) (searchPage, error) { + var page searchPage + body, err := c.get(ctx, "/api/search?"+params.Encode()) + if err != nil { + return page, searchError(err) + } + if err := json.Unmarshal(body, &page); err != nil { + return page, fmt.Errorf("%w: parse search results: %v", indexer.ErrUnreachable, err) + } + return page, nil +} + +func (c *Client) result(r searchResult) (indexer.Result, bool) { + hash := indexer.NormalizeHash(r.InfoHash) + if hash == "" || strings.TrimSpace(r.Name) == "" { + return indexer.Result{}, false + } + res := indexer.Result{ + Title: r.Name, + InfoHash: hash, + Magnet: magnet(hash, r.Name), + SizeBytes: r.Size, + Category: category(r.ContentType), + Source: c.Code, + FilesCount: r.FilesCount, + SwarmKnown: r.SwarmKnown, + } + // Seeders only when measured: an unmeasured count is not health. + if r.SwarmKnown { + res.Seeders, res.Peers = r.Seeders, r.Seeders + if r.LeechersKnown { + res.Peers = r.Seeders + r.Leechers + } + } + return res, true +} + +// searchError puts Exchange's errors in the vocabulary every search source +// shares, so a refused key reads the same whichever source refused it. +func searchError(err error) error { + switch { + case errors.Is(err, ErrAuth): + return fmt.Errorf("%w: %v", indexer.ErrAuth, err) + case errors.Is(err, ErrUnknown): + return fmt.Errorf("%w: this FabledExchange has no search API", indexer.ErrUnreachable) + } + return fmt.Errorf("%w: %v", indexer.ErrUnreachable, err) +} + +// contentType narrows Exchange's search to music when the query asks only +// for music categories (3000s, not 3030 audiobooks). +func contentType(cats []int) string { + music := false + for _, c := range cats { + if c < 3000 || c >= 4000 || c == 3030 { + return "" + } + music = true + } + if music { + return "music" + } + return "" +} + +func category(contentType string) int { + switch contentType { + case "music": + return 3000 + case "audiobook": + return 3030 + case "ebook": + return 7020 + case "comic": + return 7030 + case "movie": + return 2000 + case "tv": + return 5000 + } + return 8000 +} + +// magnet is a trackerless magnet for a hash Exchange found on the DHT. +func magnet(hash, name string) string { + m := "magnet:?xt=urn:btih:" + hash + if name != "" { + m += "&dn=" + url.QueryEscape(name) + } + return m +} diff --git a/internal/indexer/indexer.go b/internal/indexer/indexer.go new file mode 100644 index 00000000..df554a94 --- /dev/null +++ b/internal/indexer/indexer.go @@ -0,0 +1,157 @@ +// Package indexer is what every search source answers with, and the Torznab +// client (M502 #5518). Ported from Fabled Librarian's internal/indexer. +// +// A client here holds ONE endpoint and knows nothing about the others. The +// fan-out across sources, the merge and what a source being down means belong +// to internal/acquire, because they are decisions about a search rather than +// about one source. +// +// Prowlarr is not served from here: it has no aggregate Torznab feed (`all` +// 404s like any id it does not have, measured by Fabled Librarian, #4304), so +// asking it everything at once means its native API, which internal/prowlarr +// speaks. +package indexer + +import ( + "errors" + "fmt" + "io" + "net/http" + "strings" + "time" +) + +// ErrUnconfigured is a source with no address (or, for Prowlarr, no key): not +// a failure, an instance that does not search there. +var ErrUnconfigured = errors.New("indexer is not configured") + +// ErrUnreachable is a search that could not be made: no network, a timeout, a +// malformed answer, a status outside 2xx. +var ErrUnreachable = errors.New("indexer unreachable") + +// ErrAuth is a key the source refused. Separate from ErrUnreachable because +// it needs a different sentence and a different action: a refused key is a +// field in Settings, an outage is something to wait out. +var ErrAuth = errors.New("the API key was refused") + +// UserAgent names Minstrel in the operator's own Prowlarr and indexer logs. +const UserAgent = "Minstrel (self-hosted music server; release search)" + +// MusicCategories are the Newznab categories a music search asks for: 3000 +// Audio, with its MP3 (3010) and Lossless (3040) children alongside, because +// an indexer that files music under a child answers only for the child. +// 3030 (Audiobook) is deliberately absent. +var MusicCategories = []int{3000, 3010, 3040} + +// Query is one search: the words, and which categories to ask in. Empty +// Categories asks every category. +type Query struct { + Text string + Categories []int +} + +// Result is one search result, as the source described it. +type Result struct { + Title string + // InfoHash is the torrent's identity, lowercased; empty when the source + // gave none. + InfoHash string + // Magnet is a magnet URI when the source published one. + Magnet string + // DownloadURL is a .torrent URL when it did not. It must be fetched + // through the source that offered it (it holds the tracker credentials), + // never handed to the download client. + DownloadURL string + SizeBytes int64 + Category int + // Indexer is the tracker BEHIND the source, as the source named it. + Indexer string + // Source is the acquisition_services code that was asked: where a + // .torrent fetch must be routed back to. + Source string + Published time.Time + // Seeders and Peers as reported. A feed that has not sampled a swarm + // reports a nominal 1, so these are not health unless SwarmKnown. + Seeders int + Peers int + // SwarmKnown says Seeders was measured, which only FabledExchange's own + // search can say. + SwarmKnown bool + // FilesCount is how many files the torrent holds, 0 when not said. + FilesCount int +} + +// Send sends req through c, stopping at a redirect to a magnet (returned as a +// *MagnetLink), and reads at most 8 MB of the answer. A call that never got +// an answer is ErrUnreachable; the status is the caller's to read. +func Send(c *http.Client, req *http.Request) (*http.Response, []byte, error) { + resp, err := StopAtMagnet(c).Do(req) + if m, ok := AsMagnet(err); ok { + return nil, nil, &MagnetLink{URI: m} + } + if err != nil { + return nil, nil, fmt.Errorf("%w: %v", ErrUnreachable, err) + } + defer func() { _ = resp.Body.Close() }() + body, err := io.ReadAll(io.LimitReader(resp.Body, 8<<20)) + if err != nil { + return nil, nil, fmt.Errorf("%w: read body: %v", ErrUnreachable, err) + } + return resp, body, nil +} + +// StatusError is what a non-2xx status means for a search source: a 429 is +// RateLimited naming who, anything else ErrUnreachable. Nil for a 2xx. +func StatusError(resp *http.Response, who string) error { + switch { + case resp.StatusCode == http.StatusTooManyRequests: + return &RateLimited{Who: who, RetryAfter: RetryAfter(resp.Header, time.Now())} + case resp.StatusCode < 200 || resp.StatusCode >= 300: + return fmt.Errorf("%w: HTTP %d", ErrUnreachable, resp.StatusCode) + } + return nil +} + +// NormalizeHash is a torrent's identity in its one canonical spelling. Every +// comparison of infohashes goes through it, so a hash differing only in case +// is never two releases. +func NormalizeHash(s string) string { + return strings.ToLower(strings.TrimSpace(s)) +} + +// IsInfoHash says whether s is shaped like a v1 (40) or v2 (64) hex infohash. +func IsInfoHash(s string) bool { + s = NormalizeHash(s) + if len(s) != 40 && len(s) != 64 { + return false + } + for _, r := range s { + if (r < '0' || r > '9') && (r < 'a' || r > 'f') { + return false + } + } + return true +} + +// MagnetHash pulls the infohash out of a magnet link, or returns "". It does +// not validate it. +func MagnetHash(magnet string) string { + const prefix = "urn:btih:" + at := strings.Index(magnet, prefix) + if at < 0 { + return "" + } + hash := magnet[at+len(prefix):] + if cut := strings.IndexAny(hash, "&/?"); cut >= 0 { + hash = hash[:cut] + } + return strings.ToLower(hash) +} + +// FirstLine is an error's first line, for a sentence on the settings page. +func FirstLine(s string) string { + if i := strings.IndexByte(s, '\n'); i >= 0 { + return s[:i] + } + return s +} diff --git a/internal/indexer/magnet.go b/internal/indexer/magnet.go new file mode 100644 index 00000000..123e61c2 --- /dev/null +++ b/internal/indexer/magnet.go @@ -0,0 +1,58 @@ +package indexer + +import ( + "bytes" + "errors" + "net/http" +) + +// MagnetLink is a download link that turned out to be a magnet. Returned as +// an error by a .torrent fetch so the grab hands the magnet to the download +// client instead. Public trackers do this routinely: Prowlarr's download link +// answers with a redirect to `magnet:?xt=…`, which Go's client would otherwise +// fail on as an unsupported scheme. +type MagnetLink struct{ URI string } + +func (m *MagnetLink) Error() string { return "the download link is a magnet link" } + +// AsMagnet says whether err is a magnet link, and which. +func AsMagnet(err error) (string, bool) { + var m *MagnetLink + if errors.As(err, &m) && m.URI != "" { + return m.URI, true + } + return "", false +} + +// StopAtMagnet returns a copy of c that stops at a redirect to a magnet URI +// and reports it as a *MagnetLink. Every other redirect behaves as c would. +func StopAtMagnet(c *http.Client) *http.Client { + if c == nil { + c = http.DefaultClient + } + out := *c + next := c.CheckRedirect + out.CheckRedirect = func(req *http.Request, via []*http.Request) error { + if req.URL != nil && req.URL.Scheme == "magnet" { + return &MagnetLink{URI: req.URL.String()} + } + if next != nil { + return next(req, via) + } + if len(via) >= 10 { + return errors.New("stopped after 10 redirects") + } + return nil + } + return &out +} + +// MagnetBody is a response whose whole body is a magnet URI, which some +// indexers send with a 200 instead of a redirect. Nil otherwise. +func MagnetBody(body []byte) error { + trimmed := bytes.TrimSpace(body) + if bytes.HasPrefix(trimmed, []byte("magnet:")) && !bytes.ContainsAny(trimmed, " \n<") { + return &MagnetLink{URI: string(trimmed)} + } + return nil +} diff --git a/internal/indexer/ratelimit.go b/internal/indexer/ratelimit.go new file mode 100644 index 00000000..5be3ca9f --- /dev/null +++ b/internal/indexer/ratelimit.go @@ -0,0 +1,50 @@ +package indexer + +import ( + "errors" + "fmt" + "net/http" + "strconv" + "time" +) + +// RateLimited is a source answering 429. Named so the search pacing can hold +// that source back for as long as it asked; still an ErrUnreachable to +// everything else, because this search did not reach it. +type RateLimited struct { + // RetryAfter is what the source asked for; zero when it did not say. + RetryAfter time.Duration + // Who names the source in the sentence, e.g. "Prowlarr". + Who string +} + +func (e *RateLimited) Error() string { + return fmt.Sprintf("%v: HTTP 429, %s is rate-limiting this search", ErrUnreachable, e.Who) +} + +func (e *RateLimited) Unwrap() error { return ErrUnreachable } + +// AsRateLimited finds a RateLimited anywhere in err's chain. +func AsRateLimited(err error) (*RateLimited, bool) { + var rl *RateLimited + if errors.As(err, &rl) { + return rl, true + } + return nil, false +} + +// RetryAfter reads a Retry-After header: seconds, or an HTTP date. Zero when +// absent or unreadable. +func RetryAfter(h http.Header, now time.Time) time.Duration { + v := h.Get("Retry-After") + if v == "" { + return 0 + } + if s, err := strconv.Atoi(v); err == nil && s > 0 { + return time.Duration(s) * time.Second + } + if t, err := http.ParseTime(v); err == nil && t.After(now) { + return t.Sub(now) + } + return 0 +} diff --git a/internal/indexer/torznab.go b/internal/indexer/torznab.go new file mode 100644 index 00000000..03c9a5f4 --- /dev/null +++ b/internal/indexer/torznab.go @@ -0,0 +1,254 @@ +package indexer + +import ( + "context" + "encoding/xml" + "fmt" + "net/http" + "net/url" + "strconv" + "strings" + "time" +) + +// DefaultTimeout is what an unset Timeout means. Longer than a metadata +// call: a feed may itself fan out to several trackers before answering. +const DefaultTimeout = 30 * time.Second + +// TorznabConfig is one Torznab feed. +type TorznabConfig struct { + // Code is the acquisition_services row this client was built from, + // stamped onto every Result. + Code string + // FeedURL is the feed's whole address up to and including /api, as + // Jackett or a Prowlarr indexer shows it — e.g. + // http://prowlarr:9696/12/api. Query parameters are appended to it. + // One field rather than Fabled Librarian's base + path + indexer id: a + // pasted URL is more reliable than Minstrel being right about a third + // party's routes. + FeedURL string + // APIKey is optional: Torznab makes it so. A feed that needs one says so + // with Newznab error 100, which is ErrAuth. + APIKey string + Timeout time.Duration +} + +// Torznab searches one Torznab feed. +type Torznab struct { + Config TorznabConfig + // HTTP is injectable for tests. Nil takes http.DefaultClient. + HTTP *http.Client +} + +// Search asks the feed for q. `t=search` is free text, which every indexer +// supports; `t=music` takes artist and album separately, and a release name +// does not reliably say which of its parts is which. +func (c *Torznab) Search(ctx context.Context, q Query) ([]Result, error) { + params := url.Values{"t": {"search"}, "q": {q.Text}} + for _, cat := range q.Categories { + params.Add("cat", strconv.Itoa(cat)) + } + body, err := c.get(ctx, c.endpoint(params)) + if err != nil { + return nil, err + } + out, err := parseFeed(body) + if err != nil { + return nil, err + } + for i := range out { + out[i].Source = c.Config.Code + } + return out, nil +} + +// Download fetches a result's .torrent bytes through the feed, which holds +// the tracker's credentials. A result with a magnet needs none of this. +func (c *Torznab) Download(ctx context.Context, r Result) ([]byte, error) { + if r.DownloadURL == "" { + return nil, fmt.Errorf("%w: %q has no download URL", ErrUnreachable, r.Title) + } + body, err := c.get(ctx, r.DownloadURL) + if err != nil { + return nil, err + } + return torrentBody(r.Title, body) +} + +// torrentBody checks a fetched .torrent. A torrent is a bencoded dictionary +// and starts with 'd'; an expired session gets an HTML login page with HTTP +// 200, which the download client would otherwise reject as an unsupported +// file from a component that did nothing wrong. +func torrentBody(title string, body []byte) ([]byte, error) { + if err := MagnetBody(body); err != nil { + return nil, err + } + if len(body) == 0 || body[0] != 'd' { + return nil, fmt.Errorf("%w: %q returned %d bytes that are not a torrent", ErrUnreachable, title, len(body)) + } + return body, nil +} + +// TorrentBody is torrentBody for the other search clients' fetches. +func TorrentBody(title string, body []byte) ([]byte, error) { return torrentBody(title, body) } + +func (c *Torznab) endpoint(params url.Values) string { + if c.Config.APIKey != "" { + params.Set("apikey", c.Config.APIKey) + } + feed := strings.TrimSpace(c.Config.FeedURL) + sep := "?" + if strings.Contains(feed, "?") { + sep = "&" + } + return feed + sep + params.Encode() +} + +func (c *Torznab) get(ctx context.Context, endpoint string) ([]byte, error) { + if strings.TrimSpace(c.Config.FeedURL) == "" { + return nil, ErrUnconfigured + } + timeout := c.Config.Timeout + if timeout <= 0 { + timeout = DefaultTimeout + } + ctx, cancel := context.WithTimeout(ctx, timeout) + defer cancel() + req, err := http.NewRequestWithContext(ctx, http.MethodGet, endpoint, nil) + if err != nil { + return nil, fmt.Errorf("%w: build request: %v", ErrUnreachable, err) + } + req.Header.Set("User-Agent", UserAgent) + resp, body, err := Send(c.HTTP, req) + if err != nil { + return nil, err + } + if err := StatusError(resp, "the indexer"); err != nil { + return nil, err + } + // Newznab reports errors as HTTP 200 with an document, so a + // client checking only the status reads a refused key as an empty result + // set: search quietly stops finding things. + if err := errorDocument(body); err != nil { + return nil, err + } + return body, nil +} + +// errorDocument reads an answer. Newznab's 1xx +// codes (100 wrong key, 101 suspended, 102 insufficient privileges) are all +// about who is asking, so they are ErrAuth. +func errorDocument(body []byte) error { + head := body + if len(head) > 512 { + head = head[:512] + } + if !strings.Contains(string(head), " + // and repeats it in , either populated by version. + for _, candidate := range []string{it.Link, it.Enclosure.URL} { + if strings.HasPrefix(candidate, "magnet:") && r.Magnet == "" { + r.Magnet = candidate + } else if candidate != "" && r.DownloadURL == "" { + r.DownloadURL = candidate + } + } + // The infohash from the explicit attr, then the magnet, then the GUID + // only when it is shaped like a hash: Prowlarr puts a URL there as + // often as a hash. + r.InfoHash = NormalizeHash(it.attr("infohash")) + if r.InfoHash == "" { + r.InfoHash = MagnetHash(r.Magnet) + } + if r.InfoHash == "" && IsInfoHash(it.GUID) { + r.InfoHash = NormalizeHash(it.GUID) + } + if t, err := time.Parse(time.RFC1123Z, strings.TrimSpace(it.PubDate)); err == nil { + r.Published = t + } else if t, err := time.Parse(time.RFC1123, strings.TrimSpace(it.PubDate)); err == nil { + r.Published = t + } + out = append(out, r) + } + return out, nil +} + +func atoi(s string) int { + n, err := strconv.Atoi(strings.TrimSpace(s)) + if err != nil { + return 0 + } + return n +} diff --git a/internal/indexer/torznab_test.go b/internal/indexer/torznab_test.go new file mode 100644 index 00000000..2855f0af --- /dev/null +++ b/internal/indexer/torznab_test.go @@ -0,0 +1,188 @@ +package indexer + +import ( + "context" + "errors" + "net/http" + "net/http/httptest" + "testing" + "time" +) + +const sampleFeed = ` + + + + Radiohead - OK Computer (1997) [FLAC] + https://tracker.example/details/1 + https://prowlarr.example/1/download?link=abc + 512000000 + Tue, 07 Oct 2026 10:00:00 +0000 + + + + + + + + Radiohead - Kid A (2000) [MP3 320] + 0123456789abcdef0123456789abcdef01234567 + magnet:?xt=urn:btih:FFEEDDCCBBAA99887766554433221100FFEEDDCC&dn=kid+a + + + + + +` + +func TestTorznabSearch_ParsesTheFeedAndStampsTheSource(t *testing.T) { + reqs := make(chan *http.Request, 1) + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + reqs <- r + _, _ = w.Write([]byte(sampleFeed)) + })) + defer srv.Close() + + c := &Torznab{Config: TorznabConfig{Code: "torznab-2", FeedURL: srv.URL + "/12/api", APIKey: "k"}} + results, err := c.Search(context.Background(), Query{Text: "radiohead", Categories: MusicCategories}) + if err != nil { + t.Fatalf("search: %v", err) + } + got := <-reqs + if got.URL.Path != "/12/api" || got.URL.Query().Get("t") != "search" || got.URL.Query().Get("apikey") != "k" { + t.Errorf("request = %s, want the feed URL with t=search and the key", got.URL) + } + if cats := got.URL.Query()["cat"]; len(cats) != 3 || cats[0] != "3000" { + t.Errorf("categories = %v, want the music categories", cats) + } + if len(results) != 2 { + t.Fatalf("results = %d, want 2 (the untitled item dropped)", len(results)) + } + first := results[0] + if first.InfoHash != "aabbccddeeff00112233445566778899aabbccdd" || first.Seeders != 12 || + first.Category != 3040 || first.Indexer != "RED" || first.DownloadURL == "" || first.Source != "torznab-2" { + t.Errorf("first = %+v", first) + } + if first.Published.IsZero() { + t.Error("pubDate was not parsed") + } + // The magnet's hash wins over a GUID shaped like another hash. + if results[1].Magnet == "" || results[1].InfoHash != "ffeeddccbbaa99887766554433221100ffeeddcc" { + t.Errorf("second = %+v, want the magnet's hash", results[1]) + } +} + +func TestTorznabSearch_KeepsAQueryAlreadyInTheFeedURL(t *testing.T) { + reqs := make(chan *http.Request, 1) + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + reqs <- r + _, _ = w.Write([]byte(``)) + })) + defer srv.Close() + + c := &Torznab{Config: TorznabConfig{FeedURL: srv.URL + "/api?passkey=p"}} + if _, err := c.Search(context.Background(), Query{Text: "x"}); err != nil { + t.Fatalf("search: %v", err) + } + got := <-reqs + if got.URL.Query().Get("passkey") != "p" || got.URL.Query().Get("q") != "x" { + t.Errorf("request = %s, want the pasted query kept and ours added", got.URL) + } +} + +// Newznab reports a refused key as HTTP 200 with an document. Read +// as a feed, that is an empty result set: search would quietly find nothing. +func TestTorznabSearch_ReadsAnErrorDocument(t *testing.T) { + for name, tc := range map[string]struct { + doc string + want error + }{ + "a wrong key": {``, ErrAuth}, + "anything else": {``, ErrUnreachable}, + } { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + _, _ = w.Write([]byte(`` + tc.doc)) + })) + c := &Torznab{Config: TorznabConfig{FeedURL: srv.URL + "/api"}} + _, err := c.Search(context.Background(), Query{Text: "x"}) + srv.Close() + if !errors.Is(err, tc.want) { + t.Errorf("%s: err = %v, want %v", name, err, tc.want) + } + } +} + +func TestTorznabSearch_A429IsRateLimitedWithItsRetryAfter(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Retry-After", "90") + w.WriteHeader(http.StatusTooManyRequests) + })) + defer srv.Close() + + c := &Torznab{Config: TorznabConfig{FeedURL: srv.URL + "/api"}} + _, err := c.Search(context.Background(), Query{Text: "x"}) + rl, ok := AsRateLimited(err) + if !ok || rl.RetryAfter != 90*time.Second { + t.Fatalf("err = %v, want RateLimited for 90s", err) + } + if !errors.Is(err, ErrUnreachable) { + t.Error("a 429 should still read as unreachable to everything else") + } +} + +func TestTorznabSearch_NoAddressIsUnconfigured(t *testing.T) { + _, err := (&Torznab{}).Search(context.Background(), Query{Text: "x"}) + if !errors.Is(err, ErrUnconfigured) { + t.Fatalf("err = %v, want ErrUnconfigured", err) + } +} + +func TestTorznabDownload_RefusesWhatIsNotATorrent(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/login": + _, _ = w.Write([]byte(`please log in`)) + case "/magnet": + _, _ = w.Write([]byte("magnet:?xt=urn:btih:abc")) + case "/redirect": + http.Redirect(w, r, "magnet:?xt=urn:btih:def", http.StatusFound) + default: + _, _ = w.Write([]byte("d8:announce0:e")) + } + })) + defer srv.Close() + + c := &Torznab{Config: TorznabConfig{FeedURL: srv.URL + "/api"}} + ctx := context.Background() + if _, err := c.Download(ctx, Result{Title: "t", DownloadURL: srv.URL + "/login"}); !errors.Is(err, ErrUnreachable) { + t.Errorf("an HTML page: err = %v, want ErrUnreachable", err) + } + for _, path := range []string{"/magnet", "/redirect"} { + _, err := c.Download(ctx, Result{Title: "t", DownloadURL: srv.URL + path}) + if _, ok := AsMagnet(err); !ok { + t.Errorf("%s: err = %v, want a MagnetLink", path, err) + } + } + body, err := c.Download(ctx, Result{Title: "t", DownloadURL: srv.URL + "/file"}) + if err != nil || string(body) != "d8:announce0:e" { + t.Errorf("a torrent: body %q err %v", body, err) + } +} + +func TestRetryAfter(t *testing.T) { + now := time.Date(2026, 10, 10, 0, 0, 0, 0, time.UTC) + for v, want := range map[string]time.Duration{ + "": 0, + "30": 30 * time.Second, + "soon": 0, + "Sat, 10 Oct 2026 00:02:00 GMT": 2 * time.Minute, + } { + h := http.Header{} + if v != "" { + h.Set("Retry-After", v) + } + if got := RetryAfter(h, now); got != want { + t.Errorf("RetryAfter(%q) = %v, want %v", v, got, want) + } + } +} diff --git a/internal/prowlarr/prowlarr.go b/internal/prowlarr/prowlarr.go new file mode 100644 index 00000000..17b3e639 --- /dev/null +++ b/internal/prowlarr/prowlarr.go @@ -0,0 +1,177 @@ +// Package prowlarr searches through Prowlarr's native API (M502 #5518). +// Ported from Fabled Librarian's internal/prowlarr. +// +// Native rather than Torznab because Prowlarr has no aggregate Torznab feed: +// `/api/v1/indexer/all/newznab` 404s exactly like an indexer id that does not +// exist (measured by Fabled Librarian against a real Prowlarr, its #4304). +// `/api/v1/search` is the one place Prowlarr asks every indexer at once. +package prowlarr + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "net/url" + "strings" + "time" + + "git.fabledsword.com/bvandeusen/minstrel/internal/indexer" +) + +// SearchPath is Prowlarr's aggregate search. +const SearchPath = "/api/v1/search" + +// DefaultTimeout is longer than a Torznab feed's: Prowlarr waits on every +// indexer it has before answering. +const DefaultTimeout = 60 * time.Second + +// Config is one Prowlarr. +type Config struct { + // Code is the acquisition_services row, stamped onto every Result. + Code string + BaseURL string + // APIKey is required: Prowlarr's API refuses every call without one. + APIKey string + Timeout time.Duration +} + +// Client searches one Prowlarr and fetches .torrent files through it. +type Client struct { + Config Config + // HTTP is injectable for tests. Nil takes http.DefaultClient. + HTTP *http.Client +} + +type searchResult struct { + Title string `json:"title"` + InfoHash string `json:"infoHash"` + GUID string `json:"guid"` + MagnetURL string `json:"magnetUrl"` + DownloadURL string `json:"downloadUrl"` + Size int64 `json:"size"` + Seeders int `json:"seeders"` + Leechers int `json:"leechers"` + Indexer string `json:"indexer"` + PublishDate string `json:"publishDate"` + Categories []struct { + ID int `json:"id"` + } `json:"categories"` +} + +// Search asks every indexer in this Prowlarr. `type=search` is free text; +// `type=music` takes artist and album separately, and a release name does not +// reliably say which part is which. +func (c *Client) Search(ctx context.Context, q indexer.Query) ([]indexer.Result, error) { + params := url.Values{"query": {q.Text}, "type": {"search"}} + for _, cat := range q.Categories { + params.Add("categories", fmt.Sprint(cat)) + } + endpoint := strings.TrimRight(c.Config.BaseURL, "/") + SearchPath + "?" + params.Encode() + body, err := c.get(ctx, endpoint) + if err != nil { + return nil, err + } + out, err := parseResults(body) + if err != nil { + return nil, err + } + for i := range out { + out[i].Source = c.Config.Code + } + return out, nil +} + +func parseResults(body []byte) ([]indexer.Result, error) { + var rows []searchResult + if err := json.Unmarshal(body, &rows); err != nil { + return nil, fmt.Errorf("%w: parse search results: %v", indexer.ErrUnreachable, err) + } + out := make([]indexer.Result, 0, len(rows)) + for _, row := range rows { + title := strings.TrimSpace(row.Title) + if title == "" { + continue + } + r := indexer.Result{ + Title: title, + SizeBytes: row.Size, + Seeders: row.Seeders, + Peers: row.Leechers, + Indexer: strings.TrimSpace(row.Indexer), + Magnet: firstMagnet(row.MagnetURL, row.GUID), + DownloadURL: strings.TrimSpace(row.DownloadURL), + } + if len(row.Categories) > 0 { + r.Category = row.Categories[0].ID + } + r.InfoHash = indexer.NormalizeHash(row.InfoHash) + if r.InfoHash == "" { + r.InfoHash = indexer.MagnetHash(r.Magnet) + } + if r.InfoHash == "" && indexer.IsInfoHash(row.GUID) { + r.InfoHash = indexer.NormalizeHash(row.GUID) + } + if t, err := time.Parse(time.RFC3339, strings.TrimSpace(row.PublishDate)); err == nil { + r.Published = t + } + out = append(out, r) + } + return out, nil +} + +func firstMagnet(candidates ...string) string { + for _, c := range candidates { + c = strings.TrimSpace(c) + if strings.HasPrefix(c, "magnet:") { + return c + } + } + return "" +} + +// Download fetches a result's .torrent through Prowlarr, which holds the +// tracker credentials. A public tracker's link often redirects to a magnet, +// which comes back as an *indexer.MagnetLink. +func (c *Client) Download(ctx context.Context, r indexer.Result) ([]byte, error) { + if r.DownloadURL == "" { + return nil, fmt.Errorf("%w: %q has no download URL", indexer.ErrUnreachable, r.Title) + } + body, err := c.get(ctx, r.DownloadURL) + if err != nil { + return nil, err + } + return indexer.TorrentBody(r.Title, body) +} + +func (c *Client) get(ctx context.Context, endpoint string) ([]byte, error) { + if c.Config.BaseURL == "" || c.Config.APIKey == "" { + return nil, indexer.ErrUnconfigured + } + timeout := c.Config.Timeout + if timeout <= 0 { + timeout = DefaultTimeout + } + ctx, cancel := context.WithTimeout(ctx, timeout) + defer cancel() + req, err := http.NewRequestWithContext(ctx, http.MethodGet, endpoint, nil) + if err != nil { + return nil, fmt.Errorf("%w: build request: %v", indexer.ErrUnreachable, err) + } + req.Header.Set("User-Agent", indexer.UserAgent) + // In a header, not ?apikey=, so the key stays out of URLs, logs and + // error strings. + req.Header.Set("X-Api-Key", c.Config.APIKey) + req.Header.Set("Accept", "application/json") + resp, body, err := indexer.Send(c.HTTP, req) + if err != nil { + return nil, err + } + if resp.StatusCode == http.StatusUnauthorized || resp.StatusCode == http.StatusForbidden { + return nil, fmt.Errorf("%w: HTTP %d", indexer.ErrAuth, resp.StatusCode) + } + if err := indexer.StatusError(resp, "Prowlarr"); err != nil { + return nil, err + } + return body, nil +} diff --git a/internal/prowlarr/prowlarr_test.go b/internal/prowlarr/prowlarr_test.go new file mode 100644 index 00000000..2d4f3011 --- /dev/null +++ b/internal/prowlarr/prowlarr_test.go @@ -0,0 +1,78 @@ +package prowlarr + +import ( + "context" + "errors" + "net/http" + "net/http/httptest" + "testing" + + "git.fabledsword.com/bvandeusen/minstrel/internal/indexer" +) + +const sampleResults = `[ + {"title": "Radiohead - In Rainbows (2007) [FLAC]", "infoHash": "AABBCCDDEEFF00112233445566778899AABBCCDD", + "downloadUrl": "http://prowlarr/1/download?link=x", "size": 400000000, "seeders": 30, "leechers": 4, + "indexer": "RED", "publishDate": "2026-10-01T12:00:00Z", "categories": [{"id": 3040}]}, + {"title": "Radiohead - Amnesiac", "guid": "magnet:?xt=urn:btih:ffeeddccbbaa99887766554433221100ffeeddcc"}, + {"title": ""} +]` + +func TestSearch_AsksTheNativeAPIWithTheKeyInAHeader(t *testing.T) { + reqs := make(chan *http.Request, 1) + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + reqs <- r + _, _ = w.Write([]byte(sampleResults)) + })) + defer srv.Close() + + c := &Client{Config: Config{Code: "prowlarr", BaseURL: srv.URL + "/", APIKey: "secret"}} + results, err := c.Search(context.Background(), indexer.Query{Text: "radiohead", Categories: []int{3000}}) + if err != nil { + t.Fatalf("search: %v", err) + } + got := <-reqs + if got.URL.Path != SearchPath || got.URL.Query().Get("type") != "search" || got.URL.Query().Get("categories") != "3000" { + t.Errorf("request = %s, want %s with type=search and the categories", got.URL, SearchPath) + } + if got.Header.Get("X-Api-Key") != "secret" || got.URL.Query().Get("apikey") != "" { + t.Error("the key belongs in X-Api-Key, never in the URL") + } + if len(results) != 2 { + t.Fatalf("results = %d, want 2", len(results)) + } + if r := results[0]; r.InfoHash != "aabbccddeeff00112233445566778899aabbccdd" || r.Peers != 4 || + r.Category != 3040 || r.Source != "prowlarr" || r.Published.IsZero() { + t.Errorf("first = %+v", r) + } + if r := results[1]; r.Magnet == "" || r.InfoHash != "ffeeddccbbaa99887766554433221100ffeeddcc" { + t.Errorf("second = %+v, want the GUID's magnet and its hash", r) + } +} + +func TestSearch_Refusals(t *testing.T) { + for name, tc := range map[string]struct { + status int + want error + }{ + "a refused key": {http.StatusUnauthorized, indexer.ErrAuth}, + "an outage": {http.StatusBadGateway, indexer.ErrUnreachable}, + } { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(tc.status) + })) + c := &Client{Config: Config{BaseURL: srv.URL, APIKey: "k"}} + _, err := c.Search(context.Background(), indexer.Query{Text: "x"}) + srv.Close() + if !errors.Is(err, tc.want) { + t.Errorf("%s: err = %v, want %v", name, err, tc.want) + } + } +} + +func TestSearch_NoKeyIsUnconfigured(t *testing.T) { + c := &Client{Config: Config{BaseURL: "http://prowlarr:9696"}} + if _, err := c.Search(context.Background(), indexer.Query{Text: "x"}); !errors.Is(err, indexer.ErrUnconfigured) { + t.Fatalf("err = %v, want ErrUnconfigured", err) + } +} diff --git a/web/src/lib/api/admin.ts b/web/src/lib/api/admin.ts index 77e1a3b7..b56a8ad8 100644 --- a/web/src/lib/api/admin.ts +++ b/web/src/lib/api/admin.ts @@ -1009,3 +1009,82 @@ export async function updateNotificationEmailSettings( ): Promise { return api.put('/api/admin/notification-email', s); } + +// Indexers and download clients (M502 #5518) ------------------------------- + +export type AcquisitionProtocol = 'prowlarr' | 'torznab' | 'qbittorrent' | 'exchange'; + +/** One indexer or download client. Secrets never come back, only whether set. */ +export type AcquisitionService = { + code: string; + kind: 'search' | 'download'; + protocol: AcquisitionProtocol; + label: string; + enabled: boolean; + base_url: string; + has_api_key: boolean; + has_password: boolean; + username: string; + category: string; + timeout_ms: number; + /** The three switches mean something on an exchange row only. */ + serves_search: boolean; + serves_downloads: boolean; + serves_enrichment: boolean; + local_path: string; + remote_path: string; + searches_per_hour: number; + /** Switched on and addressed. */ + usable: boolean; +}; + +export type AcquisitionUpdate = Omit< + AcquisitionService, + 'code' | 'kind' | 'protocol' | 'has_api_key' | 'has_password' | 'usable' +>; + +export type AcquisitionTestResult = { ok: boolean; detail: string }; + +const acquisitionBase = '/api/admin/acquisition/services'; + +export async function listAcquisitionServices(): Promise { + const res = await api.get<{ services: AcquisitionService[] }>(acquisitionBase); + return res.services; +} + +export async function addAcquisitionService( + protocol: AcquisitionProtocol +): Promise { + return api.post(acquisitionBase, { protocol }); +} + +export async function updateAcquisitionService( + code: string, + u: AcquisitionUpdate +): Promise { + return api.put(`${acquisitionBase}/${encodeURIComponent(code)}`, u); +} + +export async function removeAcquisitionService(code: string): Promise { + await api.del(`${acquisitionBase}/${encodeURIComponent(code)}`); +} + +export async function testAcquisitionService(code: string): Promise { + return api.post(`${acquisitionBase}/${encodeURIComponent(code)}/test`, {}); +} + +/** Stores a key or password. Write-only: nothing comes back. */ +export async function setAcquisitionSecret( + code: string, + which: 'api-key' | 'password', + value: string +): Promise { + await api.put(`${acquisitionBase}/${encodeURIComponent(code)}/${which}`, { value }); +} + +export async function clearAcquisitionSecret( + code: string, + which: 'api-key' | 'password' +): Promise { + await api.del(`${acquisitionBase}/${encodeURIComponent(code)}/${which}`); +} diff --git a/web/src/lib/components/AcquisitionCard.svelte b/web/src/lib/components/AcquisitionCard.svelte new file mode 100644 index 00000000..5e711b97 --- /dev/null +++ b/web/src/lib/components/AcquisitionCard.svelte @@ -0,0 +1,196 @@ + + +{#snippet addMenu(kind: 'search' | 'download', label: string)} + {#if adding === kind} +
+ {#each choices[kind] as choice (choice.protocol)} + + {/each} + +
+ {:else} + + {/if} +{/snippet} + +
+
+

+ Indexers and download clients +

+

+ Two jobs, kept apart: something to search with, and something to download with. Minstrel + speaks Prowlarr, Torznab and the qBittorrent WebUI, so a stock setup is a complete one. +

+
+ + {#if loadError} +

+ Couldn't load indexers and download clients. + +

+ {:else if services === null} +

Loading indexers and download clients…

+ {:else} +

+ {#if !canFetch} + + No download client is switched on, so nothing can be downloaded. + + {:else if !canSearch} + + No indexer is switched on, so nothing new can be found. + + {:else} + Music can be searched for and downloaded. + {/if} +

+ +
+
+

Indexers: where music is looked for

+

+ Every indexer switched on here is searched, and the results are merged. If one is + unreachable the search still runs, narrower. +

+
+ {#each indexers as service (service.code)} + + {:else} +

No indexer yet. Prowlarr is the usual choice.

+ {/each} + {#if searchingExchanges.length > 0} +

+ Also searched: {searchingExchanges.map((s) => s.label).join(', ')}, set up under download + clients below. +

+ {/if} + {@render addMenu('search', 'Add indexer')} +
+ +
+
+

+ Download clients: what actually downloads +

+

+ What a grab is handed to. A FabledExchange here can do more than download, and its entry + says which of its jobs are switched on. +

+
+ {#each clients as service (service.code)} + + {:else} +

No download client yet.

+ {/each} + {@render addMenu('download', 'Add download client')} +
+ {/if} +
diff --git a/web/src/lib/components/AcquisitionCard.test.ts b/web/src/lib/components/AcquisitionCard.test.ts new file mode 100644 index 00000000..3db6448f --- /dev/null +++ b/web/src/lib/components/AcquisitionCard.test.ts @@ -0,0 +1,168 @@ +import { afterEach, describe, expect, test, vi } from 'vitest'; +import { render, screen, fireEvent, waitFor, within } from '@testing-library/svelte'; +import type { AcquisitionService } from '#lib/api/admin.js'; + +vi.mock('#lib/api/admin.js', () => ({ + listAcquisitionServices: vi.fn(), + addAcquisitionService: vi.fn(), + updateAcquisitionService: vi.fn(), + removeAcquisitionService: vi.fn(), + testAcquisitionService: vi.fn(), + setAcquisitionSecret: vi.fn(), + clearAcquisitionSecret: vi.fn() +})); + +vi.mock('#lib/stores/toast.svelte.js', () => ({ pushToast: vi.fn() })); + +import AcquisitionCard from './AcquisitionCard.svelte'; +import { + addAcquisitionService, + listAcquisitionServices, + setAcquisitionSecret, + testAcquisitionService, + updateAcquisitionService +} from '#lib/api/admin.js'; + +function service(over: Partial = {}): AcquisitionService { + return { + code: 'prowlarr', + kind: 'search', + protocol: 'prowlarr', + label: 'Prowlarr', + enabled: false, + base_url: '', + has_api_key: false, + has_password: false, + username: '', + category: '', + timeout_ms: 60000, + serves_search: true, + serves_downloads: true, + serves_enrichment: true, + local_path: '/downloads', + remote_path: '', + searches_per_hour: 120, + usable: false, + ...over + }; +} + +const exchange = (over: Partial = {}) => + service({ + code: 'exchange', + kind: 'download', + protocol: 'exchange', + label: 'FabledExchange', + category: 'minstrel', + searches_per_hour: 0, + ...over + }); + +afterEach(() => vi.clearAllMocks()); + +async function renderCard(services: AcquisitionService[]) { + vi.mocked(listAcquisitionServices).mockResolvedValue(services); + render(AcquisitionCard); + await screen.findByTestId('acquisition-health'); +} + +describe('AcquisitionCard', () => { + test('starts empty: nothing is seeded', async () => { + await renderCard([]); + expect(screen.getByText(/no indexer yet/i)).toBeInTheDocument(); + expect(screen.getByText(/no download client yet/i)).toBeInTheDocument(); + expect(screen.getByTestId('acquisition-health')).toHaveTextContent(/nothing can be downloaded/i); + }); + + test('adding offers the types each list takes', async () => { + await renderCard([]); + await fireEvent.click(screen.getByRole('button', { name: 'Add indexer' })); + const indexerChoices = within(screen.getByTestId('add-search')); + expect(indexerChoices.getByRole('button', { name: /prowlarr/i })).toBeInTheDocument(); + expect(indexerChoices.getByRole('button', { name: /torznab feed/i })).toBeInTheDocument(); + expect(indexerChoices.queryByRole('button', { name: /qbittorrent/i })).toBeNull(); + + vi.mocked(addAcquisitionService).mockResolvedValue(service()); + await fireEvent.click(indexerChoices.getByRole('button', { name: /prowlarr/i })); + expect(addAcquisitionService).toHaveBeenCalledWith('prowlarr'); + + await fireEvent.click(screen.getByRole('button', { name: 'Add download client' })); + const clientChoices = within(screen.getByTestId('add-download')); + expect(clientChoices.getByRole('button', { name: /qbittorrent/i })).toBeInTheDocument(); + expect(clientChoices.getByRole('button', { name: /fabledexchange/i })).toBeInTheDocument(); + }); + + test('an exchange sits under download clients with a switch per job; others have none', async () => { + await renderCard([ + service({ code: 'qbittorrent', kind: 'download', protocol: 'qbittorrent', label: 'qBittorrent' }), + exchange() + ]); + const clients = within(screen.getByTestId('download-clients')); + expect(clients.getAllByTestId('acquisition-service')).toHaveLength(2); + // Only the exchange carries the job switches. + expect(clients.getAllByTestId('exchange-jobs')).toHaveLength(1); + expect(clients.getByRole('checkbox', { name: /downloads what is grabbed/i })).toBeChecked(); + expect(clients.getByRole('checkbox', { name: /searches its dht index/i })).toBeChecked(); + expect(clients.getByRole('checkbox', { name: /checks a release/i })).toBeChecked(); + // qBittorrent asks for a username and a password; the exchange for an optional key. + expect(clients.getByText('Username')).toBeInTheDocument(); + expect(clients.getByText(/save password/i)).toBeInTheDocument(); + expect(clients.getByText(/save key/i)).toBeInTheDocument(); + }); + + test('an exchange searching is named under indexers too', async () => { + await renderCard([exchange({ usable: true, enabled: true, base_url: 'http://ex' })]); + expect(within(screen.getByTestId('indexers')).getByText(/also searched: fabledexchange/i)).toBeInTheDocument(); + expect(screen.getByTestId('acquisition-health')).toHaveTextContent(/searched for and downloaded/i); + }); + + test('switching a job off and saving sends the switches', async () => { + const ex = exchange({ base_url: 'http://ex' }); + await renderCard([ex]); + await fireEvent.click(screen.getByRole('checkbox', { name: /checks a release/i })); + vi.mocked(updateAcquisitionService).mockResolvedValue({ ...ex, serves_enrichment: false }); + await fireEvent.click(screen.getByRole('button', { name: /^save$/i })); + await waitFor(() => expect(updateAcquisitionService).toHaveBeenCalled()); + const [code, sent] = vi.mocked(updateAcquisitionService).mock.calls[0]; + expect(code).toBe('exchange'); + expect(sent).toMatchObject({ serves_enrichment: false, serves_search: true, serves_downloads: true }); + }); + + test('a refused setting is shown beside the form in the server’s words', async () => { + await renderCard([service({ base_url: 'http://p' })]); + await fireEvent.input(screen.getByRole('textbox', { name: /^name$/i }), { + target: { value: 'Mine' } + }); + vi.mocked(updateAcquisitionService).mockRejectedValue({ + code: 'invalid_setting', + message: 'Address: start with http:// or https://.', + status: 400 + }); + await fireEvent.click(screen.getByRole('button', { name: /^save$/i })); + expect(await screen.findByTestId('acquisition-problem')).toHaveTextContent( + 'Address: start with http:// or https://.' + ); + }); + + test('Test shows what the server found, and needs an address', async () => { + await renderCard([service({ code: 'a' }), service({ code: 'b', label: 'Second', base_url: 'http://p' })]); + const [noAddress, addressed] = screen.getAllByRole('button', { name: 'Test' }); + expect(noAddress).toBeDisabled(); + vi.mocked(testAcquisitionService).mockResolvedValue({ + ok: false, + detail: 'the API key was refused: HTTP 401. Check it on the service’s own settings page.' + }); + await fireEvent.click(addressed); + expect(testAcquisitionService).toHaveBeenCalledWith('b'); + expect(await screen.findByTestId('acquisition-test-result')).toHaveTextContent(/key was refused/i); + }); + + test('a key is sent write-only and never shown back', async () => { + await renderCard([service({ base_url: 'http://p', has_api_key: true })]); + expect(screen.getByLabelText(/replace the api key/i)).toHaveValue(''); + await fireEvent.input(screen.getByLabelText(/replace the api key/i), { target: { value: 'k2' } }); + vi.mocked(setAcquisitionSecret).mockResolvedValue(undefined); + await fireEvent.click(screen.getByRole('button', { name: /save key/i })); + await waitFor(() => expect(setAcquisitionSecret).toHaveBeenCalledWith('prowlarr', 'api-key', 'k2')); + }); +}); diff --git a/web/src/lib/components/AcquisitionServiceRow.svelte b/web/src/lib/components/AcquisitionServiceRow.svelte new file mode 100644 index 00000000..dfe2ad99 --- /dev/null +++ b/web/src/lib/components/AcquisitionServiceRow.svelte @@ -0,0 +1,394 @@ + + +
+
+ +
+ {#each jobs as job (job)} + {job} + {/each} + {#if service.usable} + On + {:else if service.enabled} + On, but no address + {:else} + Off + {/if} +
+
+ +

{blurb[service.protocol]}

+ +
+ Connection +
+ + + + + {#if isExchange} +
+ What it does here + + + +
+ {/if} + + {#if isQbit} + + {/if} + + {#if fetches} + + + + {/if} + + {#if searches} + + {/if} + + +
+ +
+ +
+ + + {#if secretSet} + + {/if} +
+ + {#if service.protocol === 'prowlarr'} + Prowlarr shows it under Settings → General. + {:else if isExchange} + Optional: only if your FabledExchange asks for one. Give Minstrel a key of its own there, + so it can be revoked on its own. + {:else if service.protocol === 'torznab'} + Optional: only if the feed asks for one. + {:else} + Empty if your WebUI does not ask for one. + {/if} + {secretSet ? 'One is saved.' : ''} Never shown again once saved. + +
+
+ + {#if problem} +

{problem}

+ {/if} + {#if result} +

+ {result.detail} +

+ {/if} + +
+ {#if confirmRemove} + + Remove {service.label}? Minstrel forgets its address and credentials. + Nothing on the service itself changes. + + + + {:else} + + + + {/if} +
+
diff --git a/web/src/lib/styles/error-copy.json b/web/src/lib/styles/error-copy.json index b3b56229..caf14ca5 100644 --- a/web/src/lib/styles/error-copy.json +++ b/web/src/lib/styles/error-copy.json @@ -53,5 +53,7 @@ "album_not_found": "That album no longer exists.", "artist_not_found": "That artist no longer exists.", "playlist_not_found": "That playlist no longer exists.", - "user_not_found": "That user no longer exists." + "user_not_found": "That user no longer exists.", + "service_not_found": "That connection no longer exists. Refresh the page.", + "too_many_services": "There are already twenty of these. Remove one you no longer use first." } diff --git a/web/src/routes/admin/integrations/+page.svelte b/web/src/routes/admin/integrations/+page.svelte index 81d06060..ae0a137c 100644 --- a/web/src/routes/admin/integrations/+page.svelte +++ b/web/src/routes/admin/integrations/+page.svelte @@ -28,6 +28,7 @@ import { pushToast } from '#lib/stores/toast.svelte.js'; import Modal from '#lib/components/Modal.svelte'; import AcoustIDCard from '#lib/components/AcoustIDCard.svelte'; + import AcquisitionCard from '#lib/components/AcquisitionCard.svelte'; import NetworkSettingsCard from '#lib/components/NetworkSettingsCard.svelte'; import PublicAddressCard from '#lib/components/PublicAddressCard.svelte'; import NotificationEmailCard from '#lib/components/NotificationEmailCard.svelte'; @@ -420,6 +421,8 @@ {/if} + +
diff --git a/web/src/routes/admin/integrations/integrations.test.ts b/web/src/routes/admin/integrations/integrations.test.ts index cfb148e9..577ee94c 100644 --- a/web/src/routes/admin/integrations/integrations.test.ts +++ b/web/src/routes/admin/integrations/integrations.test.ts @@ -37,7 +37,10 @@ vi.mock('#lib/api/admin.js', () => ({ listUnsettledAcoustID: vi.fn(), // NotificationEmailCard loads on mount; left pending like AcoustIDCard. getNotificationEmailSettings: vi.fn(() => new Promise(() => {})), - updateNotificationEmailSettings: vi.fn() + updateNotificationEmailSettings: vi.fn(), + // AcquisitionCard loads on mount; left pending like AcoustIDCard. + listAcquisitionServices: vi.fn(() => new Promise(() => {})), + addAcquisitionService: vi.fn() })); import IntegrationsPage from './+page.svelte';