From e17556dce92e65ac35e06af3b38f509d74898996 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Thu, 8 Oct 2026 22:32:30 -0400 Subject: [PATCH 001/117] feat: Lidarr searches for a better copy of a rip that has none (M498 #5447) The operator chose option 2: an album holding a video rip that is the only copy of its song is searched in Lidarr (AlbumSearch), at most once a week and ten albums a pass. Lidarr grabs only when the quality profile allows an upgrade, so nothing is removed and the rip stays held back either way. - lidarr.Client.SearchAlbums posts the AlbumSearch command. - ListAlbumsWithSoleCopyRips finds the albums: a held-back rip with no present clean copy sharing its song or its pending group, and not searched in the last week (audit action lidarr_rip_search). - Suspect sources shows when Lidarr last searched the track's album. Co-Authored-By: Claude Opus 5.5 --- internal/api/admin_suspect_sources.go | 7 ++ internal/audit/audit.go | 5 ++ internal/db/dbq/tracks.sql.go | 83 ++++++++++++++++++- internal/db/queries/tracks.sql | 46 +++++++++- internal/library/duplicate_resolve.go | 68 ++++++++++++++- internal/library/duplicate_resolve_test.go | 63 ++++++++++++++ internal/lidarr/releases.go | 21 +++++ internal/lidarr/releases_test.go | 22 +++++ web/src/lib/api/admin.suspect-sources.test.ts | 3 +- web/src/lib/api/types.ts | 2 + .../routes/admin/suspect-sources/+page.svelte | 11 ++- .../suspect-sources/suspect-sources.test.ts | 13 ++- 12 files changed, 337 insertions(+), 7 deletions(-) diff --git a/internal/api/admin_suspect_sources.go b/internal/api/admin_suspect_sources.go index cca97934..6161d0c4 100644 --- a/internal/api/admin_suspect_sources.go +++ b/internal/api/admin_suspect_sources.go @@ -31,6 +31,9 @@ type suspectTrackView struct { // first pass (M498 #5439). VerdictAt is when it was last set. Verdict *string `json:"verdict"` VerdictAt *string `json:"verdict_at"` + // SearchedAt is when the resolver last asked Lidarr to search the album for + // a better copy (#5447): null when it has not. + SearchedAt *string `json:"searched_at"` } // suspectGroupView is a folder's worth of flagged tracks. As on the @@ -112,6 +115,10 @@ func groupSuspectByDirectory(rows []dbq.ListSuspectSourceTracksRow) []suspectGro at := row.SourceVerdictAt.Time.UTC().Format(time.RFC3339) t.VerdictAt = &at } + if row.SearchedAt.Valid { + at := row.SearchedAt.Time.UTC().Format(time.RFC3339) + t.SearchedAt = &at + } if n := len(groups); n > 0 && groups[n-1].Directory == row.Directory { groups[n-1].Tracks = append(groups[n-1].Tracks, t) continue diff --git a/internal/audit/audit.go b/internal/audit/audit.go index 3097bc23..f574daf5 100644 --- a/internal/audit/audit.go +++ b/internal/audit/audit.go @@ -66,6 +66,11 @@ const ( // Written with no actor; the metadata names the album and both releases. ActionLidarrReleaseChange Action = "lidarr_release_change" + // Lidarr rip search (M498 #5447): the resolver asked Lidarr to search an + // album whose only copy of a song is a video rip, for a better release. + // Written with no actor; the metadata names the album and how many rips. + ActionLidarrRipSearch Action = "lidarr_rip_search" + // Subsonic password (#5026): generated in Settings for t/s-only clients. ActionSubsonicPasswordSet Action = "subsonic_password_set" ActionSubsonicPasswordClear Action = "subsonic_password_clear" diff --git a/internal/db/dbq/tracks.sql.go b/internal/db/dbq/tracks.sql.go index 6a9de912..9cc88467 100644 --- a/internal/db/dbq/tracks.sql.go +++ b/internal/db/dbq/tracks.sql.go @@ -405,6 +405,78 @@ func (q *Queries) GetTracksByIDs(ctx context.Context, dollar_1 []pgtype.UUID) ([ return items, nil } +const listAlbumsWithSoleCopyRips = `-- name: ListAlbumsWithSoleCopyRips :many +SELECT a.id AS album_id, + a.release_group_mbid::text AS release_group_mbid, + a.title AS album_title, + ar.name AS artist_name, + count(*)::bigint AS rips + FROM tracks t + JOIN albums a ON a.id = t.album_id + JOIN artists ar ON ar.id = a.artist_id + WHERE t.source_verdict = 'suspect' + AND t.missing_since IS NULL + AND a.release_group_mbid IS NOT NULL + AND NOT EXISTS ( + SELECT 1 FROM tracks c + WHERE c.song_key = t.song_key AND c.id <> t.id + AND c.missing_since IS NULL AND c.source_verdict IS DISTINCT FROM 'suspect') + AND NOT EXISTS ( + SELECT 1 FROM duplicate_group_members m + JOIN duplicate_groups g ON g.id = m.group_id AND g.status = 'pending' + JOIN duplicate_group_members m2 ON m2.group_id = g.id AND m2.track_id <> t.id + JOIN tracks c ON c.id = m2.track_id + WHERE m.track_id = t.id + AND c.missing_since IS NULL AND c.source_verdict IS DISTINCT FROM 'suspect') + AND NOT EXISTS ( + SELECT 1 FROM audit_log l + WHERE l.action = 'lidarr_rip_search' + AND l.metadata->>'album_id' = a.id::text + AND l.created_at > now() - interval '7 days') + GROUP BY a.id, a.release_group_mbid, a.title, ar.name + ORDER BY count(*) DESC, a.id + LIMIT $1 +` + +type ListAlbumsWithSoleCopyRipsRow struct { + AlbumID pgtype.UUID + ReleaseGroupMbid string + AlbumTitle string + ArtistName string + Rips int64 +} + +// Albums holding a held-back video rip that is the only copy of its song +// (#5447): no present copy that is not a rip shares its song, and no pending +// duplicate group pairs it with one. Lidarr counts such a track fulfilled, so +// the resolver asks it to search the album for a better release. Albums +// searched within the last week are left out, most rips first. +func (q *Queries) ListAlbumsWithSoleCopyRips(ctx context.Context, pageLimit int32) ([]ListAlbumsWithSoleCopyRipsRow, error) { + rows, err := q.db.Query(ctx, listAlbumsWithSoleCopyRips, pageLimit) + if err != nil { + return nil, err + } + defer rows.Close() + var items []ListAlbumsWithSoleCopyRipsRow + for rows.Next() { + var i ListAlbumsWithSoleCopyRipsRow + if err := rows.Scan( + &i.AlbumID, + &i.ReleaseGroupMbid, + &i.AlbumTitle, + &i.ArtistName, + &i.Rips, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + const listArtistTracksForUser = `-- name: ListArtistTracksForUser :many SELECT t.id, t.title, t.album_id, t.artist_id, t.track_number, t.disc_number, t.duration_ms, t.file_path, t.file_size, t.file_format, t.bitrate, t.mbid, t.genre, t.added_at, t.updated_at, t.tag_source, t.tag_sources_version, t.tag_read_version, t.missing_since, t.mbid_source, t.song_id, t.song_key, t.source_verdict, t.source_verdict_at, albums.title AS album_title, @@ -659,10 +731,17 @@ SELECT t.id, artists.id AS artist_id, artists.name AS artist_name, t.source_verdict, - t.source_verdict_at + t.source_verdict_at, + searched.at AS searched_at FROM tracks t JOIN albums ON albums.id = t.album_id JOIN artists ON artists.id = t.artist_id + -- When the resolver last asked Lidarr to search the album for a better copy + -- (#5447). + LEFT JOIN LATERAL ( + SELECT max(l.created_at)::timestamptz AS at FROM audit_log l + WHERE l.action = 'lidarr_rip_search' AND l.metadata->>'album_id' = albums.id::text + ) searched ON true WHERE t.missing_since IS NULL AND regexp_replace(t.file_path, '^.*/', '') ~* $1::text ORDER BY directory, t.disc_number NULLS FIRST, t.track_number NULLS FIRST, t.title @@ -689,6 +768,7 @@ type ListSuspectSourceTracksRow struct { ArtistName string SourceVerdict *string SourceVerdictAt pgtype.Timestamptz + SearchedAt pgtype.Timestamptz } // The admin report of present tracks whose filename looks like a video rip @@ -724,6 +804,7 @@ func (q *Queries) ListSuspectSourceTracks(ctx context.Context, arg ListSuspectSo &i.ArtistName, &i.SourceVerdict, &i.SourceVerdictAt, + &i.SearchedAt, ); err != nil { return nil, err } diff --git a/internal/db/queries/tracks.sql b/internal/db/queries/tracks.sql index b8e7d9d3..dd7b367d 100644 --- a/internal/db/queries/tracks.sql +++ b/internal/db/queries/tracks.sql @@ -287,10 +287,17 @@ SELECT t.id, artists.id AS artist_id, artists.name AS artist_name, t.source_verdict, - t.source_verdict_at + t.source_verdict_at, + searched.at AS searched_at FROM tracks t JOIN albums ON albums.id = t.album_id JOIN artists ON artists.id = t.artist_id + -- When the resolver last asked Lidarr to search the album for a better copy + -- (#5447). + LEFT JOIN LATERAL ( + SELECT max(l.created_at)::timestamptz AS at FROM audit_log l + WHERE l.action = 'lidarr_rip_search' AND l.metadata->>'album_id' = albums.id::text + ) searched ON true WHERE t.missing_since IS NULL AND regexp_replace(t.file_path, '^.*/', '') ~* sqlc.arg(pattern)::text ORDER BY directory, t.disc_number NULLS FIRST, t.track_number NULLS FIRST, t.title @@ -327,3 +334,40 @@ UPDATE tracks SET source_verdict = sqlc.arg(verdict)::text, source_verdict_at = now() WHERE id = sqlc.arg(id) AND source_verdict IS NOT NULL; + +-- name: ListAlbumsWithSoleCopyRips :many +-- Albums holding a held-back video rip that is the only copy of its song +-- (#5447): no present copy that is not a rip shares its song, and no pending +-- duplicate group pairs it with one. Lidarr counts such a track fulfilled, so +-- the resolver asks it to search the album for a better release. Albums +-- searched within the last week are left out, most rips first. +SELECT a.id AS album_id, + a.release_group_mbid::text AS release_group_mbid, + a.title AS album_title, + ar.name AS artist_name, + count(*)::bigint AS rips + FROM tracks t + JOIN albums a ON a.id = t.album_id + JOIN artists ar ON ar.id = a.artist_id + WHERE t.source_verdict = 'suspect' + AND t.missing_since IS NULL + AND a.release_group_mbid IS NOT NULL + AND NOT EXISTS ( + SELECT 1 FROM tracks c + WHERE c.song_key = t.song_key AND c.id <> t.id + AND c.missing_since IS NULL AND c.source_verdict IS DISTINCT FROM 'suspect') + AND NOT EXISTS ( + SELECT 1 FROM duplicate_group_members m + JOIN duplicate_groups g ON g.id = m.group_id AND g.status = 'pending' + JOIN duplicate_group_members m2 ON m2.group_id = g.id AND m2.track_id <> t.id + JOIN tracks c ON c.id = m2.track_id + WHERE m.track_id = t.id + AND c.missing_since IS NULL AND c.source_verdict IS DISTINCT FROM 'suspect') + AND NOT EXISTS ( + SELECT 1 FROM audit_log l + WHERE l.action = 'lidarr_rip_search' + AND l.metadata->>'album_id' = a.id::text + AND l.created_at > now() - interval '7 days') + GROUP BY a.id, a.release_group_mbid, a.title, ar.name + ORDER BY count(*) DESC, a.id + LIMIT sqlc.arg(page_limit); diff --git a/internal/library/duplicate_resolve.go b/internal/library/duplicate_resolve.go index 5dd4b9ee..b4e79da5 100644 --- a/internal/library/duplicate_resolve.go +++ b/internal/library/duplicate_resolve.go @@ -50,6 +50,7 @@ type LidarrLibrary interface { GetAlbumReleases(ctx context.Context, albumID int) ([]lidarr.AlbumRelease, error) ListReleaseTracks(ctx context.Context, releaseID int) ([]lidarr.ReleaseTrack, error) SetMonitoredRelease(ctx context.Context, albumID, releaseID int) error + SearchAlbums(ctx context.Context, albumIDs []int) error } const ( @@ -94,6 +95,9 @@ type DuplicateResolveResult struct { // HeldBack counts tracks newly held back from radio and the mixes as video // rips; Released counts held-back tracks whose name no longer says so. HeldBack, Released int64 + // RipSearches counts albums Lidarr was asked to search for a better copy + // of a rip that has no clean copy (#5447). + RipSearches int } // LidarrPathKey is the part of a path Minstrel and Lidarr agree on: the last @@ -265,6 +269,13 @@ func ResolveDuplicates( return res, err } + // Last, rips with no clean copy anywhere: ask Lidarr for something better. + searched, err := searchSoleCopyRips(ctx, q, pool, logger, lid) + res.RipSearches = searched + if err != nil { + logger.Warn("duplicate resolve: rip search failed", "err", err) + } + if res.Merged > 0 || len(changes) > 0 { notifyAdmins(ctx, notifications.KindDuplicatesResolved, notifications.Payload{ Count: int64(res.Merged + len(changes)), @@ -334,6 +345,61 @@ func linkSong(ctx context.Context, pool *pgxpool.Pool, ids []pgtype.UUID) (bool, return true, tx.Commit(ctx) } +// ripSearchCap bounds the albums one pass asks Lidarr to search, so the first +// pass after deploy does not queue the whole backlog of rips at once. +const ripSearchCap = 10 + +// searchSoleCopyRips asks Lidarr to search each album holding a video rip that +// is the only copy of its song (#5447, operator's choice: option 2). While +// Lidarr maps the rip it counts the track fulfilled, so nothing else asks for +// better. An album search grabs only when the quality profile allows an +// upgrade, so this removes nothing and may change nothing: the rip stays held +// back either way. An album is searched at most once a week (the query reads +// the audit log), and an album Lidarr does not hold is skipped. +func searchSoleCopyRips( + ctx context.Context, q *dbq.Queries, pool *pgxpool.Pool, logger *slog.Logger, lid LidarrLibrary, +) (int, error) { + albums, err := q.ListAlbumsWithSoleCopyRips(ctx, ripSearchCap) + if err != nil { + return 0, fmt.Errorf("list albums with sole-copy rips: %w", err) + } + type found struct { + row dbq.ListAlbumsWithSoleCopyRipsRow + lidarrID int + } + var targets []found + for _, a := range albums { + la, err := lid.LookupAlbumByMBID(ctx, a.ReleaseGroupMbid) + if errors.Is(err, lidarr.ErrNotFound) { + continue + } + if err != nil { + return 0, fmt.Errorf("look up %s: %w", a.ReleaseGroupMbid, err) + } + targets = append(targets, found{row: a, lidarrID: la.ID}) + } + if len(targets) == 0 { + return 0, nil + } + ids := make([]int, len(targets)) + for i, t := range targets { + ids[i] = t.lidarrID + } + if err := lid.SearchAlbums(ctx, ids); err != nil { + return 0, fmt.Errorf("search albums: %w", err) + } + for _, t := range targets { + audit.WriteOrLog(ctx, pool, logger, pgtype.UUID{}, pgtype.UUID{}, audit.ActionLidarrRipSearch, map[string]any{ + "album_id": syncpkg.FormatUUID(t.row.AlbumID), + "album_title": t.row.AlbumTitle, + "artist_name": t.row.ArtistName, + "lidarr_album_id": t.lidarrID, + "rips": t.row.Rips, + }) + } + return len(targets), nil +} + func foldResolveGroups(rows []dbq.ListDuplicateGroupsForResolveRow) []*resolveGroup { var groups []*resolveGroup for _, r := range rows { @@ -787,5 +853,5 @@ func (w *DuplicateResolveWorker) tickOnce(ctx context.Context) { w.logger.Info("duplicate resolve complete", "groups", res.Groups, "lidarr", res.LidarrConsulted, "merged", res.Merged, "merge_failed", res.MergeFailed, "release_changes", len(res.ReleaseChanges), "songs_linked", res.SongsLinked, - "held_back", res.HeldBack, "released", res.Released) + "held_back", res.HeldBack, "released", res.Released, "rip_searches", res.RipSearches) } diff --git a/internal/library/duplicate_resolve_test.go b/internal/library/duplicate_resolve_test.go index d1b67a83..20778fc6 100644 --- a/internal/library/duplicate_resolve_test.go +++ b/internal/library/duplicate_resolve_test.go @@ -82,6 +82,7 @@ type fakeLidarr struct { releases []lidarr.AlbumRelease releaseTracks map[int][]lidarr.ReleaseTrack setCalls [][2]int + searchCalls [][]int } func (f *fakeLidarr) ListUnmappedTrackFiles(context.Context) ([]lidarr.TrackFile, error) { @@ -116,6 +117,11 @@ func (f *fakeLidarr) ListReleaseTracks(_ context.Context, releaseID int) ([]lida return f.releaseTracks[releaseID], nil } +func (f *fakeLidarr) SearchAlbums(_ context.Context, albumIDs []int) error { + f.searchCalls = append(f.searchCalls, albumIDs) + return nil +} + func (f *fakeLidarr) SetMonitoredRelease(_ context.Context, albumID, releaseID int) error { f.setCalls = append(f.setCalls, [2]int{albumID, releaseID}) for i := range f.releases { @@ -502,3 +508,60 @@ func TestResolveDuplicates_HoldsBackRips_Integration(t *testing.T) { t.Errorf("set on an unflagged track: %d, %v; want 0 rows", n, err) } } + +// A rip that is the only copy of its song: Lidarr is asked to search its album +// for a better release (#5447), once a week at most, and the rip is not touched. +// A rip with a clean copy elsewhere is not searched for. +func TestResolveDuplicates_SearchesForASoleCopyRip_Integration(t *testing.T) { + f := newCrossReleaseRipFixture(t) + ctx := context.Background() + exec := func(sql string, args ...any) { + t.Helper() + if _, err := f.pool.Exec(ctx, sql, args...); err != nil { + t.Fatalf("exec %q: %v", sql, err) + } + } + // The single's rip has a clean copy on the album: never searched for. + exec(`UPDATE albums SET release_group_mbid = 'rg-single' WHERE id = $1`, f.rip.AlbumID) + lid := &fakeLidarr{album: lidarr.LidarrAlbum{ID: 77, ForeignAlbumID: "rg-single"}} + if _, err := ResolveDuplicates(ctx, f.pool, nil, "", lid, true); err != nil { + t.Fatal(err) + } + if len(lid.searchCalls) != 0 { + t.Fatalf("searched %v while a clean copy exists", lid.searchCalls) + } + + // Without the group and the song link, the rip is the only copy. + exec(`DELETE FROM duplicate_groups WHERE id = $1`, f.groupID) + exec(`UPDATE tracks SET song_id = NULL WHERE id IN ($1, $2)`, f.rip.ID, f.clean.ID) + res, err := ResolveDuplicates(ctx, f.pool, nil, "", lid, true) + if err != nil { + t.Fatal(err) + } + if res.RipSearches != 1 || len(lid.searchCalls) != 1 || len(lid.searchCalls[0]) != 1 || lid.searchCalls[0][0] != 77 { + t.Fatalf("rip searches %d, calls %v; want album 77 searched once", res.RipSearches, lid.searchCalls) + } + if !exists(f.ripPath) { + t.Error("the rip was removed; a search must not touch it") + } + + // The next pass, inside the week, leaves the album alone. + if res, err := ResolveDuplicates(ctx, f.pool, nil, "", lid, true); err != nil || res.RipSearches != 0 { + t.Errorf("second pass searched %d (%v), want 0", res.RipSearches, err) + } + var searchedAt pgtype.Timestamptz + rows, err := dbq.New(f.pool).ListSuspectSourceTracks(ctx, dbq.ListSuspectSourceTracksParams{ + Pattern: SuspectSourcePattern, PageLimit: 10, + }) + if err != nil { + t.Fatal(err) + } + for _, r := range rows { + if r.ID == f.rip.ID { + searchedAt = r.SearchedAt + } + } + if !searchedAt.Valid { + t.Error("the suspect-sources report doesn't show the search") + } +} diff --git a/internal/lidarr/releases.go b/internal/lidarr/releases.go index 2513af12..a952d52e 100644 --- a/internal/lidarr/releases.go +++ b/internal/lidarr/releases.go @@ -152,3 +152,24 @@ func (c *Client) SetMonitoredRelease(ctx context.Context, albumID, releaseID int _ = putResp.Body.Close() return nil } + +// SearchAlbums asks Lidarr to search its indexers for the albums +// (POST /api/v1/command, AlbumSearch). Lidarr grabs a release only when it +// beats the files the album already has under the quality profile, so for an +// album whose tracks are all present this is an upgrade search: harmless when +// nothing better exists. The command is queued; the call does not wait for it. +func (c *Client) SearchAlbums(ctx context.Context, albumIDs []int) error { + if len(albumIDs) == 0 { + return nil + } + body, err := json.Marshal(map[string]any{"name": "AlbumSearch", "albumIds": albumIDs}) + if err != nil { + return fmt.Errorf("%w: marshal command: %v", ErrInvalidPayload, err) + } + resp, err := c.post(ctx, "/api/v1/command", body) + if err != nil { + return err + } + _ = resp.Body.Close() + return nil +} diff --git a/internal/lidarr/releases_test.go b/internal/lidarr/releases_test.go index 5e3c769e..e89a39d2 100644 --- a/internal/lidarr/releases_test.go +++ b/internal/lidarr/releases_test.go @@ -114,3 +114,25 @@ func TestSetMonitoredRelease_UnknownReleaseSendsNothing(t *testing.T) { t.Errorf("err = %v, want ErrNotFound", err) } } + +func TestSearchAlbums_PostsTheCommand(t *testing.T) { + var got map[string]any + c, srv := newTestClient(func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost || r.URL.Path != "/api/v1/command" { + t.Errorf("request = %s %s, want POST /api/v1/command", r.Method, r.URL.Path) + } + b, _ := io.ReadAll(r.Body) + _ = json.Unmarshal(b, &got) + w.WriteHeader(http.StatusCreated) + _, _ = w.Write([]byte(`{"id":1,"name":"AlbumSearch"}`)) + }) + defer srv.Close() + + if err := c.SearchAlbums(context.Background(), []int{4723, 12}); err != nil { + t.Fatalf("SearchAlbums: %v", err) + } + ids, _ := got["albumIds"].([]any) + if got["name"] != "AlbumSearch" || len(ids) != 2 || ids[0] != float64(4723) || ids[1] != float64(12) { + t.Errorf("command = %v", got) + } +} diff --git a/web/src/lib/api/admin.suspect-sources.test.ts b/web/src/lib/api/admin.suspect-sources.test.ts index 0d90aee9..59017a66 100644 --- a/web/src/lib/api/admin.suspect-sources.test.ts +++ b/web/src/lib/api/admin.suspect-sources.test.ts @@ -16,7 +16,8 @@ function track(id: string): AdminSuspectTrack { track_number: null, markers: [], verdict: null, - verdict_at: null + verdict_at: null, + searched_at: null }; } diff --git a/web/src/lib/api/types.ts b/web/src/lib/api/types.ts index f752b91e..2fef805d 100644 --- a/web/src/lib/api/types.ts +++ b/web/src/lib/api/types.ts @@ -443,6 +443,8 @@ export type AdminSuspectTrack = { // back. null: the resolver hasn't looked yet (M498 #5439). verdict: SuspectVerdict | null; verdict_at: string | null; + // When Lidarr was last asked to search the album for a better copy (#5447). + searched_at: string | null; }; export type SuspectVerdict = 'suspect' | 'fine'; diff --git a/web/src/routes/admin/suspect-sources/+page.svelte b/web/src/routes/admin/suspect-sources/+page.svelte index 101362fb..db39fd60 100644 --- a/web/src/routes/admin/suspect-sources/+page.svelte +++ b/web/src/routes/admin/suspect-sources/+page.svelte @@ -17,7 +17,8 @@ // The duplicate resolver holds each one back from radio and the mixes, and // merges it away where Lidarr keeps a clean copy (M498 #5439). A marker is a // reason to look, not proof, so each row says what was done and offers the - // undo: "This one is fine" lets the track back for good. + // undo: "This one is fine" lets the track back for good. A rip that is the + // only copy of its song has its album searched in Lidarr (#5447). const queryStore = createSuspectSourcesQuery(); const query = $derived($queryStore); @@ -101,7 +102,8 @@ a video site, and can be the wrong audio: a live take, a music-video edit, or something else entirely. Each one is kept out of radio and the mixes, though it still plays when you choose it, and it is removed where Lidarr keeps a clean copy of the same - song. If one is fine, say so and it goes back in. + song. Where it is the only copy, Lidarr is asked once a week to search the album for a + better release. If one is fine, say so and it goes back in.

@@ -178,6 +180,11 @@ Held back{#if t.verdict_at && !changed[t.track_id]} {relativeTime(t.verdict_at)}{/if} + {#if t.searched_at} + + Lidarr searched for a better copy {relativeTime(t.searched_at)} + + {/if}

- Tracks that hold the same recording more than once. Copies Lidarr doesn't use are merged on - their own, and a copy Lidarr uses is never removed. What's left here needs you. + Tracks that hold the same recording more than once. A song held twice on one album is merged on + its own, and so is a video rip with a clean copy elsewhere. What's left here needs you.

@@ -265,7 +258,7 @@

Nothing resolved automatically yet.

- Merges and Lidarr release changes made on their own are listed here. + Merges made on their own are listed here.

{:else} @@ -279,7 +272,7 @@ {#if item.details.survivor_reason} ({item.details.survivor_reason}) {/if} - and removed {fileCountLabel(removed.length)} Lidarr doesn't use. + and removed {fileCountLabel(removed.length)}.

{#each removed as r (r.track_id)}

{r.file_path}

@@ -354,7 +347,6 @@
    {#each groups as group (group.id)} {@const titles = differingTitles(group)} - {@const blocked = trackedLosers(group)}
  • @@ -373,7 +365,7 @@
    {m.artist_name} · {m.album_title}{position(m) ? ` · ${position(m)}` : ''} diff --git a/web/src/routes/admin/duplicates/duplicates.test.ts b/web/src/routes/admin/duplicates/duplicates.test.ts index 4f452295..1060fca1 100644 --- a/web/src/routes/admin/duplicates/duplicates.test.ts +++ b/web/src/routes/admin/duplicates/duplicates.test.ts @@ -66,7 +66,6 @@ function member(id: string, extra: Partial { expect(dismissDuplicateGroup).toHaveBeenCalledWith('g-1'); }); - test('Resolved automatically lists merges and release changes', async () => { + // Release changes stopped with Lidarr (M502), but the ones made before stay + // in the history. + test('Resolved automatically lists merges and past release changes', async () => { renderWith(response(), [ { id: 'a-1', @@ -340,7 +341,7 @@ describe('admin duplicates', () => { created_at: ago(HOUR), details: { survivor_path: '/music/Gorillaz/Humanz (2017)/03 - Saturnz Barz.mp3', - survivor_reason: 'the copy Lidarr tracks', + survivor_reason: 'the copy that is not a video rip', removed: [{ track_id: 't-2', file_path: '/music/Gorillaz/Humanz (2017)/03 - Saturnz Barz (Official Video).mp3' }] } }, @@ -358,47 +359,26 @@ describe('admin duplicates', () => { ]); await fireEvent.click(screen.getByRole('tab', { name: /resolved automatically/i })); const items = screen.getAllByTestId('resolved-item').map(text); - expect(items[0]).toContain('Kept 03 - Saturnz Barz.mp3 (the copy Lidarr tracks) and removed 1 file'); + expect(items[0]).toContain('Kept 03 - Saturnz Barz.mp3 (the copy that is not a video rip) and removed 1 file.'); expect(items[1]).toContain('Lidarr now monitors Digital Media (40 tracks) of Humanz by Gorillaz'); expect(screen.queryByTestId('duplicate-group')).toBeNull(); }); }); - describe("Lidarr's view of each copy", () => { - function tracked() { + describe("the resolver's note", () => { + test('shows above the group it explains', () => { const r = response(); - r.groups[0] = { - ...r.groups[0], - resolve_note: 'Lidarr tracks every copy as its own track.', - members: [member('www-01', { lidarr_state: 'tracked' }), member('www-02', { lidarr_state: 'unmapped' })] - }; - return r; - } - - test('each copy says whether Lidarr uses it, and the resolver note shows', () => { - renderWith(tracked()); - expect(screen.getAllByTestId('lidarr-state').map(text)).toEqual([ - 'Lidarr uses this', - "Lidarr doesn't use this" - ]); - expect(text(screen.getByTestId('resolve-note'))).toBe('Lidarr tracks every copy as its own track.'); + r.groups[0] = { ...r.groups[0], resolve_note: 'Identical audio filed under different titles.' }; + renderWith(r); + expect(text(screen.getByTestId('resolve-note'))).toBe('Identical audio filed under different titles.'); }); - // Keeping the unmapped copy would remove the one Lidarr uses. The server - // refuses that; the confirm says so first and offers no button to press. - test('a merge that would remove the copy Lidarr uses is blocked before it is sent', async () => { - renderWith(tracked()); + test('a merge confirms and sends the copy chosen to keep', async () => { + renderWith(response()); await fireEvent.click(screen.getAllByRole('radio')[1]); await fireEvent.click(screen.getByRole('button', { name: 'Merge…' })); - expect(text(screen.getByTestId('merge-blocked'))).toContain('Lidarr uses a copy this would remove'); - const confirm = screen.getByRole('button', { name: 'Remove 1 file and merge' }) as HTMLButtonElement; - expect(confirm.disabled).toBe(true); - - // Choosing the copy Lidarr uses clears it. - await fireEvent.click(screen.getAllByRole('radio')[0]); - expect(screen.queryByTestId('merge-blocked')).toBeNull(); await fireEvent.click(screen.getByRole('button', { name: 'Remove 1 file and merge' })); - expect(mergeDuplicateGroup).toHaveBeenCalledWith('g-1', { survivor_track_id: 'www-01', unmonitor: false }); + expect(mergeDuplicateGroup).toHaveBeenCalledWith('g-1', { survivor_track_id: 'www-02', unmonitor: false }); }); }); }); -- 2.54.0 From a2e1890d4c4c2fa48426c4285864ad91a8bcafcd Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Fri, 9 Oct 2026 20:34:51 -0400 Subject: [PATCH 005/117] feat: indexers and download clients you add and configure (M502 #5518) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Admin → Integrations gains "Indexers and download clients": two lists of connections the operator adds, Fabled Librarian's model. Nothing is seeded or copied from Lidarr. - Indexers: Prowlarr (its native /api/v1/search, which asks every indexer at once; Prowlarr has no aggregate Torznab feed) and any Torznab feed. - Download clients: qBittorrent (WebUI v2, with re-login on an expired session) and FabledExchange, one entry with a switch per job: downloads (/api/downloads), search (/api/search) and release details (/api/torrents/{infohash}). All three start on; its key is optional. - Each entry: enable, name, address, a write-only key or password, Test. A download client's Test lists its torrents and checks the downloads folder is visible and writable from Minstrel; an Exchange's Test reports each job switched on. - Download clients carry a path mapping (Minstrel's /downloads vs the client's own name for it); search sources carry searches per hour. Migration 0079 adds acquisition_services with its CHECKs. Ported from Fabled Librarian: internal/indexer (Torznab, 429 Retry-After, magnet redirects), internal/prowlarr, internal/exchange, internal/downloader, and internal/acquire's one builder per kind, which Test already uses so a passing Test vouches for the path a grab will take. Co-Authored-By: Claude Opus 5.5 --- internal/acquire/acquire_test.go | 180 ++++++++ internal/acquire/settings.go | 170 ++++++++ internal/acquire/store.go | 191 +++++++++ internal/acquire/store_test.go | 108 +++++ internal/acquire/test.go | 134 ++++++ internal/api/admin_acquisition.go | 209 ++++++++++ internal/api/api.go | 13 + internal/db/dbq/acquisition.sql.go | 362 ++++++++++++++++ internal/db/dbq/models.go | 22 + .../0079_acquisition_services.down.sql | 1 + .../0079_acquisition_services.up.sql | 63 +++ internal/db/queries/acquisition.sql | 76 ++++ internal/dbtest/reset.go | 1 + internal/downloader/downloader_test.go | 156 +++++++ internal/downloader/driver.go | 198 +++++++++ internal/downloader/exchange.go | 340 +++++++++++++++ internal/downloader/pathmap.go | 107 +++++ internal/downloader/qbittorrent.go | 367 ++++++++++++++++ internal/exchange/exchange.go | 182 ++++++++ internal/exchange/exchange_test.go | 108 +++++ internal/exchange/search.go | 169 ++++++++ internal/indexer/indexer.go | 157 +++++++ internal/indexer/magnet.go | 58 +++ internal/indexer/ratelimit.go | 50 +++ internal/indexer/torznab.go | 254 +++++++++++ internal/indexer/torznab_test.go | 188 +++++++++ internal/prowlarr/prowlarr.go | 177 ++++++++ internal/prowlarr/prowlarr_test.go | 78 ++++ web/src/lib/api/admin.ts | 79 ++++ web/src/lib/components/AcquisitionCard.svelte | 196 +++++++++ .../lib/components/AcquisitionCard.test.ts | 168 ++++++++ .../components/AcquisitionServiceRow.svelte | 394 ++++++++++++++++++ web/src/lib/styles/error-copy.json | 4 +- .../routes/admin/integrations/+page.svelte | 3 + .../admin/integrations/integrations.test.ts | 5 +- 35 files changed, 4966 insertions(+), 2 deletions(-) create mode 100644 internal/acquire/acquire_test.go create mode 100644 internal/acquire/settings.go create mode 100644 internal/acquire/store.go create mode 100644 internal/acquire/store_test.go create mode 100644 internal/acquire/test.go create mode 100644 internal/api/admin_acquisition.go create mode 100644 internal/db/dbq/acquisition.sql.go create mode 100644 internal/db/migrations/0079_acquisition_services.down.sql create mode 100644 internal/db/migrations/0079_acquisition_services.up.sql create mode 100644 internal/db/queries/acquisition.sql create mode 100644 internal/downloader/downloader_test.go create mode 100644 internal/downloader/driver.go create mode 100644 internal/downloader/exchange.go create mode 100644 internal/downloader/pathmap.go create mode 100644 internal/downloader/qbittorrent.go create mode 100644 internal/exchange/exchange.go create mode 100644 internal/exchange/exchange_test.go create mode 100644 internal/exchange/search.go create mode 100644 internal/indexer/indexer.go create mode 100644 internal/indexer/magnet.go create mode 100644 internal/indexer/ratelimit.go create mode 100644 internal/indexer/torznab.go create mode 100644 internal/indexer/torznab_test.go create mode 100644 internal/prowlarr/prowlarr.go create mode 100644 internal/prowlarr/prowlarr_test.go create mode 100644 web/src/lib/components/AcquisitionCard.svelte create mode 100644 web/src/lib/components/AcquisitionCard.test.ts create mode 100644 web/src/lib/components/AcquisitionServiceRow.svelte diff --git a/internal/acquire/acquire_test.go b/internal/acquire/acquire_test.go new file mode 100644 index 00000000..ac0ebf16 --- /dev/null +++ b/internal/acquire/acquire_test.go @@ -0,0 +1,180 @@ +package acquire + +import ( + "context" + "errors" + "net/http" + "net/http/httptest" + "path/filepath" + "strings" + "testing" + + "git.fabledsword.com/bvandeusen/minstrel/internal/db/dbq" + "git.fabledsword.com/bvandeusen/minstrel/internal/downloader" + "git.fabledsword.com/bvandeusen/minstrel/internal/exchange" + "git.fabledsword.com/bvandeusen/minstrel/internal/indexer" + "git.fabledsword.com/bvandeusen/minstrel/internal/prowlarr" +) + +func validUpdate() Update { + return Update{Label: "Prowlarr", BaseURL: "http://prowlarr:9696", TimeoutMs: 30000, LocalPath: "/downloads", SearchesPerHour: 120} +} + +func TestUpdateValidate(t *testing.T) { + if err := validUpdate().validate(); err != nil { + t.Fatalf("a valid form: %v", err) + } + for name, tc := range map[string]struct { + edit func(*Update) + field string + }{ + "no name": {func(u *Update) { u.Label = "" }, "label"}, + "switched on with no address": {func(u *Update) { u.Enabled, u.BaseURL = true, "" }, "base_url"}, + "an address with no scheme": {func(u *Update) { u.BaseURL = "prowlarr:9696" }, "base_url"}, + "a timeout too short": {func(u *Update) { u.TimeoutMs = 10 }, "timeout_ms"}, + "a relative downloads folder": {func(u *Update) { u.LocalPath = "downloads" }, "local_path"}, + "a relative client folder": {func(u *Update) { u.RemotePath = "data" }, "remote_path"}, + "too many searches": {func(u *Update) { u.SearchesPerHour = 4000 }, "searches_per_hour"}, + } { + u := validUpdate() + tc.edit(&u) + var invalid *InvalidError + if err := u.validate(); !errors.As(err, &invalid) || invalid.Field != tc.field { + t.Errorf("%s: err = %v, want an InvalidError on %s", name, err, tc.field) + } + } + // Switched off, an empty address is a row still being set up. + u := validUpdate() + u.BaseURL = "" + if err := u.validate(); err != nil { + t.Errorf("switched off with no address: %v", err) + } +} + +func TestSourceFor_BuildsTheClientTheProtocolNames(t *testing.T) { + for protocol, check := range map[string]func(Source) bool{ + ProtocolProwlarr: func(s Source) bool { _, ok := s.Search.(*prowlarr.Client); return ok && s.Fetch != nil }, + ProtocolTorznab: func(s Source) bool { _, ok := s.Search.(*indexer.Torznab); return ok && s.Fetch != nil }, + // Exchange's results are all magnets: nothing to fetch through it. + ProtocolExchange: func(s Source) bool { _, ok := s.Search.(*exchange.Client); return ok && s.Fetch == nil }, + } { + src := SourceFor(dbq.AcquisitionService{Code: protocol, Protocol: protocol, SearchesPerHour: 7}) + if !check(src) || src.Code != protocol || src.PerHour != 7 { + t.Errorf("%s: source %+v", protocol, src) + } + } +} + +func TestJobs_AnExchangeDoesWhatItsSwitchesAllow(t *testing.T) { + ex := dbq.AcquisitionService{Kind: KindDownload, Protocol: ProtocolExchange, ServesSearch: true} + if !Searches(ex) || Fetches(ex) { + t.Error("an exchange with only search on should search and not fetch") + } + // A qBittorrent's switches mean nothing: it fetches, whatever they say. + qb := dbq.AcquisitionService{Kind: KindDownload, Protocol: ProtocolQBittorrent, ServesSearch: true} + if Searches(qb) || !Fetches(qb) { + t.Error("a qBittorrent fetches and never searches") + } +} + +func TestTest_NoAddressIsNothingToReach(t *testing.T) { + r := Test(context.Background(), dbq.AcquisitionService{Protocol: ProtocolProwlarr, Kind: KindSearch}) + if r.OK || !strings.Contains(r.Detail, "No address") { + t.Fatalf("result = %+v", r) + } +} + +func TestTest_SearchReportsResultsAndARefusedKey(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Header.Get("X-Api-Key") != "good" { + w.WriteHeader(http.StatusUnauthorized) + return + } + _, _ = w.Write([]byte(`[{"title": "Radiohead - OK Computer"}]`)) + })) + defer srv.Close() + + row := dbq.AcquisitionService{Kind: KindSearch, Protocol: ProtocolProwlarr, BaseUrl: srv.URL, ApiKey: "good", TimeoutMs: 5000} + if r := Test(context.Background(), row); !r.OK || !strings.Contains(r.Detail, "1 results") { + t.Errorf("a working key: %+v", r) + } + row.ApiKey = "bad" + if r := Test(context.Background(), row); r.OK || !strings.Contains(r.Detail, "refused") { + t.Errorf("a refused key: %+v", r) + } +} + +// fakeExchange answers the three native APIs. +func fakeExchange(t *testing.T) *httptest.Server { + t.Helper() + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch { + case r.URL.Path == "/api/downloads": + _, _ = w.Write([]byte(`{"enabled": true, "running": true, "inbound_conns": 3, "downloads": []}`)) + case r.URL.Path == "/api/search": + _, _ = w.Write([]byte(`{"total": 0, "results": []}`)) + case strings.HasPrefix(r.URL.Path, "/api/torrents/"): + w.WriteHeader(http.StatusNotFound) + _, _ = w.Write([]byte(`{"error": "unknown torrent"}`)) + default: + w.WriteHeader(http.StatusNotFound) + } + })) + t.Cleanup(srv.Close) + return srv +} + +func TestTest_AnExchangeReportsEachJobSwitchedOn(t *testing.T) { + srv := fakeExchange(t) + row := dbq.AcquisitionService{ + Kind: KindDownload, Protocol: ProtocolExchange, BaseUrl: srv.URL, TimeoutMs: 5000, + ServesDownloads: true, ServesSearch: true, ServesEnrichment: true, LocalPath: t.TempDir(), + } + r := Test(context.Background(), row) + if !r.OK { + t.Fatalf("all three jobs: %+v", r) + } + for _, job := range []string{"Downloads:", "Search:", "Release details:"} { + if !strings.Contains(r.Detail, job) { + t.Errorf("detail %q does not report %s", r.Detail, job) + } + } + row.ServesDownloads, row.ServesEnrichment = false, false + if r := Test(context.Background(), row); strings.Contains(r.Detail, "Downloads:") || !strings.Contains(r.Detail, "Search:") { + t.Errorf("only search on: %+v", r) + } + row.ServesSearch = false + if r := Test(context.Background(), row); r.OK || !strings.Contains(r.Detail, "switched off") { + t.Errorf("every job off: %+v", r) + } +} + +func TestTest_ADownloadClientNeedsAFolderMinstrelCanSee(t *testing.T) { + srv := fakeExchange(t) + row := dbq.AcquisitionService{ + Kind: KindDownload, Protocol: ProtocolExchange, BaseUrl: srv.URL, TimeoutMs: 5000, + ServesDownloads: true, Category: DefaultCategory, + LocalPath: filepath.Join(t.TempDir(), "not-mounted"), + } + r := Test(context.Background(), row) + if r.OK || !strings.Contains(r.Detail, "cannot see its downloads folder") { + t.Fatalf("a missing folder: %+v", r) + } + row.LocalPath = t.TempDir() + if r := Test(context.Background(), row); !r.OK || !strings.Contains(r.Detail, `"minstrel" category`) { + t.Errorf("a visible, writable folder: %+v", r) + } +} + +func TestDownloadClient_WrapsThePathMapping(t *testing.T) { + d, err := DownloadClient(dbq.AcquisitionService{Protocol: ProtocolQBittorrent, LocalPath: "/downloads", RemotePath: "/data"}) + if err != nil { + t.Fatal(err) + } + if _, plain := d.(*downloader.Client); plain { + t.Error("a row naming two folders should come back wrapped in its mapping") + } + if _, err := DownloadClient(dbq.AcquisitionService{Protocol: ProtocolProwlarr}); err == nil { + t.Error("a search protocol is not a download client") + } +} diff --git a/internal/acquire/settings.go b/internal/acquire/settings.go new file mode 100644 index 00000000..a7a9c9ea --- /dev/null +++ b/internal/acquire/settings.go @@ -0,0 +1,170 @@ +// Package acquire turns the operator's configured indexers and download +// clients into working clients (M502 #5518). Ported from Fabled Librarian's +// internal/acquire. +// +// ONE BUILDER PER KIND. SourceFor makes a search source from a row and +// DownloadClient makes a download client, path mapping included. The grab, +// the importer, the stall watchdog and the settings page's Test all build +// through them, so a Test that passes vouches for the path real traffic takes. +// Clients are rebuilt from the row on every use, so a changed address or a new +// key takes effect on the next call. +package acquire + +import ( + "context" + "time" + + "github.com/jackc/pgx/v5/pgxpool" + + "git.fabledsword.com/bvandeusen/minstrel/internal/db/dbq" + "git.fabledsword.com/bvandeusen/minstrel/internal/downloader" + "git.fabledsword.com/bvandeusen/minstrel/internal/exchange" + "git.fabledsword.com/bvandeusen/minstrel/internal/indexer" + "git.fabledsword.com/bvandeusen/minstrel/internal/prowlarr" +) + +// Kinds: which list a row appears in. +const ( + KindSearch = "search" + KindDownload = "download" +) + +// Jobs a row can do. An exchange row does each its switch allows; every +// other row does the one its kind names. +const ( + JobSearch = "search" + JobDownload = "download" + JobEnrichment = "enrichment" +) + +// Protocols: which client a row builds. +const ( + ProtocolProwlarr = "prowlarr" + ProtocolTorznab = "torznab" + ProtocolQBittorrent = "qbittorrent" + ProtocolExchange = "exchange" +) + +// DefaultCategory files everything Minstrel adds to a download client. +const DefaultCategory = "minstrel" + +// addable is what a new row of a protocol starts as. +type addable struct { + kind string + label string + category string + timeoutMs int32 + perHour int32 +} + +// catalogue is every protocol the operator can add. Exchange starts unpaced +// (it searches its own database); the others at 120 searches an hour, which +// keeps a Prowlarr's indexers under the rate most trackers allow. +var catalogue = map[string]addable{ + ProtocolProwlarr: {kind: KindSearch, label: "Prowlarr", timeoutMs: 60000, perHour: 120}, + ProtocolTorznab: {kind: KindSearch, label: "Torznab feed", timeoutMs: 30000, perHour: 120}, + ProtocolQBittorrent: {kind: KindDownload, label: "qBittorrent", category: DefaultCategory, timeoutMs: 15000, perHour: 0}, + ProtocolExchange: {kind: KindDownload, label: "FabledExchange", category: DefaultCategory, timeoutMs: 30000, perHour: 0}, +} + +// Addable says whether protocol is one the operator can add. +func Addable(protocol string) bool { + _, ok := catalogue[protocol] + return ok +} + +// Searches says whether a row is a search source. +func Searches(row dbq.AcquisitionService) bool { + if row.Protocol == ProtocolExchange { + return row.ServesSearch + } + return row.Kind == KindSearch +} + +// Fetches says whether a row is a download client. +func Fetches(row dbq.AcquisitionService) bool { + if row.Protocol == ProtocolExchange { + return row.ServesDownloads + } + return row.Kind == KindDownload +} + +// Searcher is a search source. +type Searcher interface { + Search(ctx context.Context, q indexer.Query) ([]indexer.Result, error) +} + +// Fetcher fetches a result's .torrent through the source that offered it. +type Fetcher interface { + Download(ctx context.Context, r indexer.Result) ([]byte, error) +} + +// Source is one configured search source. +type Source struct { + Code string + Label string + // PerHour is the most searches the schedule sends it an hour; 0 = unpaced. + PerHour int + Search Searcher + // Fetch is nil for a source whose results are all magnets. + Fetch Fetcher +} + +// SourceFor builds the search source a row describes. +func SourceFor(row dbq.AcquisitionService) Source { + src := Source{Code: row.Code, Label: row.Label, PerHour: int(row.SearchesPerHour)} + timeout := msDuration(row.TimeoutMs) + switch row.Protocol { + case ProtocolProwlarr: + c := &prowlarr.Client{Config: prowlarr.Config{ + Code: row.Code, BaseURL: row.BaseUrl, APIKey: row.ApiKey, Timeout: timeout, + }} + src.Search, src.Fetch = c, c + case ProtocolExchange: + src.Search = ExchangeFor(row) + default: + c := &indexer.Torznab{Config: indexer.TorznabConfig{ + Code: row.Code, FeedURL: row.BaseUrl, APIKey: row.ApiKey, Timeout: timeout, + }} + src.Search, src.Fetch = c, c + } + return src +} + +// ExchangeFor builds the search and enrichment client for an exchange row. +func ExchangeFor(row dbq.AcquisitionService) *exchange.Client { + return &exchange.Client{ + Code: row.Code, + Config: exchange.Config{ + BaseURL: row.BaseUrl, APIKey: row.ApiKey, Timeout: msDuration(row.TimeoutMs), + }, + } +} + +// DownloadClient builds the download client a row describes, wrapped in its +// path mapping. +func DownloadClient(row dbq.AcquisitionService) (downloader.Driver, error) { + d, err := downloader.For(row.Protocol, downloader.Config{ + BaseURL: row.BaseUrl, Username: row.Username, Password: row.Password, + APIKey: row.ApiKey, Category: row.Category, Timeout: msDuration(row.TimeoutMs), + }) + if err != nil { + return nil, err + } + return downloader.Mapped(d, downloader.PathMap{Local: row.LocalPath, Remote: row.RemotePath}), nil +} + +// Sources builds every enabled search source. +func Sources(ctx context.Context, pool *pgxpool.Pool) ([]Source, error) { + rows, err := dbq.New(pool).ListUsableAcquisitionServices(ctx, JobSearch) + if err != nil { + return nil, err + } + out := make([]Source, len(rows)) + for i, r := range rows { + out[i] = SourceFor(r) + } + return out, nil +} + +func msDuration(ms int32) time.Duration { return time.Duration(ms) * time.Millisecond } diff --git a/internal/acquire/store.go b/internal/acquire/store.go new file mode 100644 index 00000000..833435b9 --- /dev/null +++ b/internal/acquire/store.go @@ -0,0 +1,191 @@ +package acquire + +import ( + "context" + "errors" + "fmt" + "net/url" + "strings" + + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgxpool" + + "git.fabledsword.com/bvandeusen/minstrel/internal/db/dbq" +) + +// ErrNotFound is a code with no row. +var ErrNotFound = errors.New("no such service") + +// ErrTooMany is a twenty-first row of one protocol. +var ErrTooMany = errors.New("there are already twenty of these") + +// InvalidError is a setting the operator has to change, in words for the +// settings card. +type InvalidError struct { + Field string + Message string +} + +func (e *InvalidError) Error() string { return e.Message } + +func invalid(field, msg string) error { return &InvalidError{Field: field, Message: msg} } + +// maxPerProtocol bounds the codes tried for a new row. Nobody runs twenty +// Prowlarrs; the bound keeps a loop over a full table finite. +const maxPerProtocol = 20 + +// Store reads and writes acquisition_services. +type Store struct { + pool *pgxpool.Pool +} + +// NewStore returns a Store over pool. +func NewStore(pool *pgxpool.Pool) *Store { return &Store{pool: pool} } + +// List returns every row without its secrets. +func (s *Store) List(ctx context.Context) ([]dbq.ListAcquisitionServicesRow, error) { + return dbq.New(s.pool).ListAcquisitionServices(ctx) +} + +// Get returns one row, secrets included: for building a client only. +func (s *Store) Get(ctx context.Context, code string) (dbq.AcquisitionService, error) { + row, err := dbq.New(s.pool).GetAcquisitionService(ctx, code) + if errors.Is(err, pgx.ErrNoRows) { + return row, ErrNotFound + } + return row, err +} + +// Create adds a row of protocol, switched off and with no address. Its code +// is the protocol for the first, then protocol-2, protocol-3. +func (s *Store) Create(ctx context.Context, protocol string) (dbq.AcquisitionService, error) { + a, ok := catalogue[protocol] + if !ok { + return dbq.AcquisitionService{}, invalid("protocol", "One of prowlarr, torznab, qbittorrent, exchange.") + } + q := dbq.New(s.pool) + for n := 1; n <= maxPerProtocol; n++ { + code, label := protocol, a.label + if n > 1 { + code, label = fmt.Sprintf("%s-%d", protocol, n), fmt.Sprintf("%s %d", a.label, n) + } + row, err := q.CreateAcquisitionService(ctx, dbq.CreateAcquisitionServiceParams{ + Code: code, Kind: a.kind, Protocol: protocol, Label: label, + Category: a.category, TimeoutMs: a.timeoutMs, SearchesPerHour: a.perHour, + }) + if errors.Is(err, pgx.ErrNoRows) { + continue + } + return row, err + } + return dbq.AcquisitionService{}, ErrTooMany +} + +// Update is everything on the settings form except the credentials. +type Update struct { + Label string + Enabled bool + BaseURL string + Username string + Category string + TimeoutMs int32 + ServesSearch bool + ServesDownloads bool + ServesEnrichment bool + LocalPath string + RemotePath string + SearchesPerHour int32 +} + +// Save validates u and writes it to code's row. +func (s *Store) Save(ctx context.Context, code string, u Update) (dbq.AcquisitionService, error) { + u.Label = strings.TrimSpace(u.Label) + u.BaseURL = strings.TrimSpace(u.BaseURL) + u.Username = strings.TrimSpace(u.Username) + u.Category = strings.TrimSpace(u.Category) + u.LocalPath = strings.TrimSpace(u.LocalPath) + u.RemotePath = strings.TrimSpace(u.RemotePath) + if err := u.validate(); err != nil { + return dbq.AcquisitionService{}, err + } + row, err := dbq.New(s.pool).UpdateAcquisitionService(ctx, dbq.UpdateAcquisitionServiceParams{ + Code: code, Label: u.Label, Enabled: u.Enabled, BaseUrl: u.BaseURL, + Username: u.Username, Category: u.Category, TimeoutMs: u.TimeoutMs, + ServesSearch: u.ServesSearch, ServesDownloads: u.ServesDownloads, + ServesEnrichment: u.ServesEnrichment, LocalPath: u.LocalPath, + RemotePath: u.RemotePath, SearchesPerHour: u.SearchesPerHour, + }) + if errors.Is(err, pgx.ErrNoRows) { + return row, ErrNotFound + } + return row, err +} + +// validate mirrors the table's CHECKs, so a bad value is named in the card's +// own terms before Postgres refuses it. +func (u Update) validate() error { + if u.Label == "" { + return invalid("label", "Name: give it a name.") + } + if u.Enabled && u.BaseURL == "" { + return invalid("base_url", "Address: give the address before switching this on.") + } + if u.BaseURL != "" { + parsed, err := url.Parse(u.BaseURL) + if err != nil || (parsed.Scheme != "http" && parsed.Scheme != "https") || parsed.Host == "" { + return invalid("base_url", "Address: start with http:// or https://.") + } + } + if u.TimeoutMs < 1000 || u.TimeoutMs > 300000 { + return invalid("timeout_ms", "Give up after: between 1000 and 300000 milliseconds.") + } + if !strings.HasPrefix(u.LocalPath, "/") { + return invalid("local_path", "Downloads folder as Minstrel sees it: a full path, starting with /.") + } + if u.RemotePath != "" && !strings.HasPrefix(u.RemotePath, "/") { + return invalid("remote_path", "Downloads folder as the client sees it: a full path starting with /, or empty when it is the same path.") + } + if u.SearchesPerHour < 0 || u.SearchesPerHour > 3600 { + return invalid("searches_per_hour", "Searches per hour: between 0 and 3600. 0 means no limit.") + } + return nil +} + +// Delete removes code's row. +func (s *Store) Delete(ctx context.Context, code string) error { + n, err := dbq.New(s.pool).DeleteAcquisitionService(ctx, code) + if err != nil { + return err + } + if n == 0 { + return ErrNotFound + } + return nil +} + +// SetAPIKey stores a key, trimmed; empty clears it. +func (s *Store) SetAPIKey(ctx context.Context, code, key string) error { + n, err := dbq.New(s.pool).SetAcquisitionServiceAPIKey(ctx, dbq.SetAcquisitionServiceAPIKeyParams{ + Code: code, ApiKey: strings.TrimSpace(key), + }) + return affected(n, err) +} + +// SetPassword stores a password as typed (spaces are part of a password); +// empty clears it. +func (s *Store) SetPassword(ctx context.Context, code, password string) error { + n, err := dbq.New(s.pool).SetAcquisitionServicePassword(ctx, dbq.SetAcquisitionServicePasswordParams{ + Code: code, Password: password, + }) + return affected(n, err) +} + +func affected(n int64, err error) error { + if err != nil { + return err + } + if n == 0 { + return ErrNotFound + } + return nil +} diff --git a/internal/acquire/store_test.go b/internal/acquire/store_test.go new file mode 100644 index 00000000..37c369da --- /dev/null +++ b/internal/acquire/store_test.go @@ -0,0 +1,108 @@ +package acquire + +import ( + "context" + "errors" + "io" + "log/slog" + "os" + "testing" + + "github.com/jackc/pgx/v5/pgxpool" + + "git.fabledsword.com/bvandeusen/minstrel/internal/db" + "git.fabledsword.com/bvandeusen/minstrel/internal/dbtest" +) + +func newPool(t *testing.T) *pgxpool.Pool { + t.Helper() + if testing.Short() { + t.Skip("skipping integration test in -short mode") + } + dsn := os.Getenv("MINSTREL_TEST_DATABASE_URL") + if dsn == "" { + t.Skip("MINSTREL_TEST_DATABASE_URL not set") + } + if err := db.Migrate(dsn, slog.New(slog.NewTextHandler(io.Discard, nil))); err != nil { + t.Fatalf("migrate: %v", err) + } + pool, err := pgxpool.New(context.Background(), dsn) + if err != nil { + t.Fatalf("pool: %v", err) + } + t.Cleanup(pool.Close) + dbtest.ResetDB(t, pool) + return pool +} + +func TestStore_AddsConfiguresAndRemoves_Integration(t *testing.T) { + ctx := context.Background() + s := NewStore(newPool(t)) + + first, err := s.Create(ctx, ProtocolExchange) + if err != nil { + t.Fatalf("create: %v", err) + } + second, err := s.Create(ctx, ProtocolExchange) + if err != nil { + t.Fatalf("create a second: %v", err) + } + if first.Code != "exchange" || second.Code != "exchange-2" || second.Label != "FabledExchange 2" { + t.Errorf("codes = %q, %q (%q); want exchange, exchange-2", first.Code, second.Code, second.Label) + } + // A new exchange is off, does every job, files under minstrel and is + // unpaced. + if first.Enabled || first.Kind != KindDownload || !first.ServesSearch || !first.ServesDownloads || + !first.ServesEnrichment || first.Category != DefaultCategory || first.SearchesPerHour != 0 || first.LocalPath != "/downloads" { + t.Errorf("a new exchange = %+v", first) + } + if _, err := s.Create(ctx, "transmission"); !errors.As(err, new(*InvalidError)) { + t.Errorf("an unknown protocol: err = %v", err) + } + + u := Update{ + Label: "Exchange", Enabled: true, BaseURL: "http://exchange:8080", Category: "minstrel", + TimeoutMs: 20000, ServesSearch: true, LocalPath: "/downloads", RemotePath: "/data/torrents", + } + saved, err := s.Save(ctx, first.Code, u) + if err != nil { + t.Fatalf("save: %v", err) + } + if !saved.Enabled || saved.ServesDownloads || saved.RemotePath != "/data/torrents" { + t.Errorf("saved = %+v", saved) + } + if _, err := s.Save(ctx, "nope", u); !errors.Is(err, ErrNotFound) { + t.Errorf("saving a missing row: err = %v", err) + } + + // Secrets are write-only: the list says whether one is set, never what. + if err := s.SetAPIKey(ctx, first.Code, " key "); err != nil { + t.Fatalf("set key: %v", err) + } + full, err := s.Get(ctx, first.Code) + if err != nil || full.ApiKey != "key" { + t.Errorf("stored key = %q (err %v), want it trimmed", full.ApiKey, err) + } + rows, err := s.List(ctx) + if err != nil || len(rows) != 2 { + t.Fatalf("list: %d rows, err %v", len(rows), err) + } + for _, r := range rows { + if r.Code == first.Code && !r.HasApiKey { + t.Error("the list should say a key is set") + } + } + + // Only the exchange with search switched on is a search source. + srcs, err := Sources(ctx, s.pool) + if err != nil || len(srcs) != 1 || srcs[0].Code != first.Code { + t.Errorf("sources = %+v, err %v; want only the enabled exchange", srcs, err) + } + + if err := s.Delete(ctx, second.Code); err != nil { + t.Fatalf("delete: %v", err) + } + if err := s.Delete(ctx, second.Code); !errors.Is(err, ErrNotFound) { + t.Errorf("deleting twice: err = %v", err) + } +} diff --git a/internal/acquire/test.go b/internal/acquire/test.go new file mode 100644 index 00000000..9ec6d07b --- /dev/null +++ b/internal/acquire/test.go @@ -0,0 +1,134 @@ +package acquire + +import ( + "context" + "errors" + "fmt" + "os" + "strings" + + "git.fabledsword.com/bvandeusen/minstrel/internal/db/dbq" + "git.fabledsword.com/bvandeusen/minstrel/internal/exchange" + "git.fabledsword.com/bvandeusen/minstrel/internal/indexer" +) + +// TestResult is what the settings card says after Test. +type TestResult struct { + OK bool `json:"ok"` + Detail string `json:"detail"` +} + +// testQuery is a search every music indexer should answer. +const testQuery = "Radiohead" + +// Test reaches the service a row describes, through the same builders real +// traffic uses. It runs whether or not the row is switched on: paste the +// address, test it, then switch it on. +func Test(ctx context.Context, row dbq.AcquisitionService) TestResult { + if strings.TrimSpace(row.BaseUrl) == "" { + return TestResult{Detail: "No address is set, so there is nothing to reach yet."} + } + if row.Protocol == ProtocolExchange { + return testExchange(ctx, row) + } + if row.Kind == KindSearch { + return testSearch(ctx, row) + } + return testDownload(ctx, row) +} + +// testExchange runs each job the row has switched on and reports them all. +func testExchange(ctx context.Context, row dbq.AcquisitionService) TestResult { + var parts []string + ok := true + run := func(name string, on bool, test func() TestResult) { + if !on { + return + } + r := test() + ok = ok && r.OK + parts = append(parts, name+": "+r.Detail) + } + run("Downloads", row.ServesDownloads, func() TestResult { return testDownload(ctx, row) }) + run("Search", row.ServesSearch, func() TestResult { return testSearch(ctx, row) }) + run("Release details", row.ServesEnrichment, func() TestResult { return testEnrichment(ctx, row) }) + if len(parts) == 0 { + return TestResult{Detail: "Every job of this FabledExchange is switched off, so there is nothing to test."} + } + return TestResult{OK: ok, Detail: strings.Join(parts, " ")} +} + +// testSearch runs a real search. +func testSearch(ctx context.Context, row dbq.AcquisitionService) TestResult { + src := SourceFor(row) + results, err := src.Search.Search(ctx, indexer.Query{Text: testQuery, Categories: indexer.MusicCategories}) + switch { + case errors.Is(err, indexer.ErrUnconfigured): + return TestResult{Detail: "Add this service's address, and the API key it shows on its own settings page."} + case errors.Is(err, indexer.ErrAuth): + return TestResult{Detail: indexer.FirstLine(err.Error()) + ". Check it on the service's own settings page."} + case err != nil: + return TestResult{Detail: indexer.FirstLine(err.Error())} + case len(results) == 0: + return TestResult{OK: true, Detail: "Connected, but a test search for music found nothing. Check that it has an indexer with music categories."} + } + return TestResult{OK: true, Detail: fmt.Sprintf("Connected. %d results for a test search.", len(results))} +} + +// testDownload lists the client's torrents, which proves the session and not +// only the login, then checks the downloads folder from Minstrel's side. +func testDownload(ctx context.Context, row dbq.AcquisitionService) TestResult { + c, err := DownloadClient(row) + if err != nil { + return TestResult{Detail: err.Error()} + } + ts, err := c.Torrents(ctx) + if err != nil { + return TestResult{Detail: indexer.FirstLine(err.Error())} + } + detail := fmt.Sprintf("Connected. %d torrents.", len(ts)) + if row.Category != "" { + detail = fmt.Sprintf("Connected. %d torrents in the %q category.", len(ts), row.Category) + } + if up, err := c.Connected(ctx); err == nil && !up { + detail += " The client reports it is not connected to any peers, so downloads will stall." + } + if problem := landingProblem(row.LocalPath); problem != "" { + return TestResult{Detail: detail + " " + problem} + } + return TestResult{OK: true, Detail: detail} +} + +// landingProblem checks Minstrel can see and write the downloads folder: it +// moves finished downloads out of it. Only the local path can be checked; +// the client's own name for it cannot be seen from here. +func landingProblem(dir string) string { + info, err := os.Stat(dir) + switch { + case err != nil: + return fmt.Sprintf("But Minstrel cannot see its downloads folder at %s. Mount the download client's downloads folder there, or change where Minstrel looks for it.", dir) + case !info.IsDir(): + return fmt.Sprintf("But %s is a file, not the downloads folder.", dir) + } + probe, err := os.CreateTemp(dir, ".minstrel-probe-*") + if err != nil { + return fmt.Sprintf("But Minstrel cannot write to %s, so it could not move a finished download out of it. Check the mount is not read-only and Minstrel's user can write there.", dir) + } + name := probe.Name() + _ = probe.Close() + _ = os.Remove(name) + return "" +} + +// testEnrichment asks about a hash no torrent has. A JSON "not crawled" is a +// healthy answer; a 404 from something that is not Exchange is not. +func testEnrichment(ctx context.Context, row dbq.AcquisitionService) TestResult { + _, err := ExchangeFor(row).Torrent(ctx, strings.Repeat("0", 40)) + switch { + case err == nil, errors.Is(err, exchange.ErrUnknown): + return TestResult{OK: true, Detail: "Connected. The index answered."} + case errors.Is(err, exchange.ErrUnconfigured): + return TestResult{Detail: "No address is set, so there is nothing to reach yet."} + } + return TestResult{Detail: indexer.FirstLine(err.Error())} +} diff --git a/internal/api/admin_acquisition.go b/internal/api/admin_acquisition.go new file mode 100644 index 00000000..e857d24a --- /dev/null +++ b/internal/api/admin_acquisition.go @@ -0,0 +1,209 @@ +package api + +import ( + "encoding/json" + "errors" + "io" + "net/http" + + "github.com/go-chi/chi/v5" + + "git.fabledsword.com/bvandeusen/minstrel/internal/acquire" + "git.fabledsword.com/bvandeusen/minstrel/internal/apierror" + "git.fabledsword.com/bvandeusen/minstrel/internal/db/dbq" +) + +// Indexers and download clients, admin side (M502 #5518). Rows the operator +// adds and configures; nothing is seeded. Secrets are write-only: no response +// carries one, only whether it is set. + +// acquisitionBodyLimit caps a settings body. The largest is a full form of +// short strings. +const acquisitionBodyLimit = 64 << 10 + +type acquisitionServiceResp struct { + Code string `json:"code"` + Kind string `json:"kind"` + Protocol string `json:"protocol"` + Label string `json:"label"` + Enabled bool `json:"enabled"` + BaseURL string `json:"base_url"` + HasAPIKey bool `json:"has_api_key"` + HasPassword bool `json:"has_password"` + Username string `json:"username"` + Category string `json:"category"` + TimeoutMs int32 `json:"timeout_ms"` + ServesSearch bool `json:"serves_search"` + ServesDownloads bool `json:"serves_downloads"` + ServesEnrichment bool `json:"serves_enrichment"` + LocalPath string `json:"local_path"` + RemotePath string `json:"remote_path"` + SearchesPerHour int32 `json:"searches_per_hour"` + // Usable is switched on and addressed: the state the jobs read. + Usable bool `json:"usable"` +} + +func acquisitionRespOf(r dbq.AcquisitionService) acquisitionServiceResp { + return acquisitionServiceResp{ + Code: r.Code, Kind: r.Kind, Protocol: r.Protocol, Label: r.Label, + Enabled: r.Enabled, BaseURL: r.BaseUrl, + HasAPIKey: r.ApiKey != "", HasPassword: r.Password != "", + Username: r.Username, Category: r.Category, TimeoutMs: r.TimeoutMs, + ServesSearch: r.ServesSearch, ServesDownloads: r.ServesDownloads, + ServesEnrichment: r.ServesEnrichment, + LocalPath: r.LocalPath, RemotePath: r.RemotePath, + SearchesPerHour: r.SearchesPerHour, + Usable: r.Enabled && r.BaseUrl != "", + } +} + +// handleListAcquisitionServices implements GET /api/admin/acquisition/services. +func (h *handlers) handleListAcquisitionServices(w http.ResponseWriter, r *http.Request) { + rows, err := h.acquisition.List(r.Context()) + if err != nil { + writeErrWithLog(w, h.logger, "admin: list acquisition services", apierror.Internal(err)) + return + } + out := make([]acquisitionServiceResp, len(rows)) + for i, row := range rows { + out[i] = acquisitionRespOf(dbq.AcquisitionService{ + Code: row.Code, Kind: row.Kind, Protocol: row.Protocol, Label: row.Label, + Enabled: row.Enabled, BaseUrl: row.BaseUrl, Username: row.Username, + Category: row.Category, TimeoutMs: row.TimeoutMs, + ServesSearch: row.ServesSearch, ServesDownloads: row.ServesDownloads, + ServesEnrichment: row.ServesEnrichment, LocalPath: row.LocalPath, + RemotePath: row.RemotePath, SearchesPerHour: row.SearchesPerHour, + }) + out[i].HasAPIKey, out[i].HasPassword = row.HasApiKey, row.HasPassword + } + writeJSON(w, http.StatusOK, map[string]any{"services": out}) +} + +// handleCreateAcquisitionService implements POST /api/admin/acquisition/services +// with {"protocol": "prowlarr"|"torznab"|"qbittorrent"|"exchange"}. The row +// starts switched off, with no address. +func (h *handlers) handleCreateAcquisitionService(w http.ResponseWriter, r *http.Request) { + var body struct { + Protocol string `json:"protocol"` + } + if !decodeAcquisitionBody(w, r, &body) { + return + } + row, err := h.acquisition.Create(r.Context(), body.Protocol) + if err != nil { + h.writeAcquisitionErr(w, "admin: create acquisition service", err) + return + } + writeJSON(w, http.StatusCreated, acquisitionRespOf(row)) +} + +type updateAcquisitionBody struct { + Label string `json:"label"` + Enabled bool `json:"enabled"` + BaseURL string `json:"base_url"` + Username string `json:"username"` + Category string `json:"category"` + TimeoutMs int32 `json:"timeout_ms"` + ServesSearch bool `json:"serves_search"` + ServesDownloads bool `json:"serves_downloads"` + ServesEnrichment bool `json:"serves_enrichment"` + LocalPath string `json:"local_path"` + RemotePath string `json:"remote_path"` + SearchesPerHour int32 `json:"searches_per_hour"` +} + +// handleUpdateAcquisitionService implements PUT +// /api/admin/acquisition/services/{code}: the whole form except credentials. +func (h *handlers) handleUpdateAcquisitionService(w http.ResponseWriter, r *http.Request) { + var body updateAcquisitionBody + if !decodeAcquisitionBody(w, r, &body) { + return + } + row, err := h.acquisition.Save(r.Context(), chi.URLParam(r, "code"), acquire.Update(body)) + if err != nil { + h.writeAcquisitionErr(w, "admin: update acquisition service", err) + return + } + writeJSON(w, http.StatusOK, acquisitionRespOf(row)) +} + +// handleDeleteAcquisitionService implements DELETE +// /api/admin/acquisition/services/{code}. +func (h *handlers) handleDeleteAcquisitionService(w http.ResponseWriter, r *http.Request) { + if err := h.acquisition.Delete(r.Context(), chi.URLParam(r, "code")); err != nil { + h.writeAcquisitionErr(w, "admin: delete acquisition service", err) + return + } + w.WriteHeader(http.StatusNoContent) +} + +// handleTestAcquisitionService implements POST +// /api/admin/acquisition/services/{code}/test. POST because it reaches +// another machine, which a prefetch must not trigger. Always 200 with +// {ok, detail}: a failed connection is the answer, not an error. +func (h *handlers) handleTestAcquisitionService(w http.ResponseWriter, r *http.Request) { + row, err := h.acquisition.Get(r.Context(), chi.URLParam(r, "code")) + if err != nil { + h.writeAcquisitionErr(w, "admin: test acquisition service", err) + return + } + writeJSON(w, http.StatusOK, acquire.Test(r.Context(), row)) +} + +// handleSetAcquisitionSecret implements PUT and DELETE on +// /api/admin/acquisition/services/{code}/api-key and .../password. PUT takes +// {"value": "..."}; DELETE clears it. 204 with no body either way: echoing +// anything here is how a secret ends up in a response. +func (h *handlers) handleSetAcquisitionSecret(field string) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + value := "" + if r.Method == http.MethodPut { + var body struct { + Value string `json:"value"` + } + if !decodeAcquisitionBody(w, r, &body) { + return + } + if body.Value == "" { + writeErr(w, apierror.BadRequest("invalid_setting", "Paste a value, or use Remove to clear the one that is set.")) + return + } + value = body.Value + } + code := chi.URLParam(r, "code") + var err error + if field == "password" { + err = h.acquisition.SetPassword(r.Context(), code, value) + } else { + err = h.acquisition.SetAPIKey(r.Context(), code, value) + } + if err != nil { + h.writeAcquisitionErr(w, "admin: set acquisition "+field, err) + return + } + w.WriteHeader(http.StatusNoContent) + } +} + +func decodeAcquisitionBody(w http.ResponseWriter, r *http.Request, into any) bool { + err := json.NewDecoder(http.MaxBytesReader(w, r.Body, acquisitionBodyLimit)).Decode(into) + if err != nil && !errors.Is(err, io.EOF) { + writeErr(w, apierror.BadRequest("invalid_body", "malformed JSON")) + return false + } + return true +} + +func (h *handlers) writeAcquisitionErr(w http.ResponseWriter, msg string, err error) { + var invalid *acquire.InvalidError + switch { + case errors.As(err, &invalid): + writeErr(w, apierror.BadRequest("invalid_setting", invalid.Message)) + case errors.Is(err, acquire.ErrNotFound): + writeErr(w, apierror.NotFound("service")) + case errors.Is(err, acquire.ErrTooMany): + writeErr(w, apierror.Conflict("too_many_services", "There are already twenty of these. Remove one you no longer use first.")) + default: + writeErrWithLog(w, h.logger, msg, apierror.Internal(err)) + } +} diff --git a/internal/api/api.go b/internal/api/api.go index 1162ec25..ec2419bf 100644 --- a/internal/api/api.go +++ b/internal/api/api.go @@ -13,6 +13,7 @@ import ( "github.com/go-chi/chi/v5" "github.com/jackc/pgx/v5/pgxpool" + "git.fabledsword.com/bvandeusen/minstrel/internal/acquire" "git.fabledsword.com/bvandeusen/minstrel/internal/auth" "git.fabledsword.com/bvandeusen/minstrel/internal/config" "git.fabledsword.com/bvandeusen/minstrel/internal/coverart" @@ -70,6 +71,7 @@ func Mount(r chi.Router, pool *pgxpool.Pool, logger *slog.Logger, events *playev fingerprintSettings: fpSettings, loudnessSettings: loudSettings, acoustIDLookup: acoustIDLookup, + acquisition: acquire.NewStore(pool), leveled: newLeveledRenderer(dataDir, loudSettings, logger), librarySize: recommendation.NewLibrarySize(nil), loginGuard: auth.NewLoginGuard(), @@ -214,6 +216,15 @@ func Mount(r chi.Router, pool *pgxpool.Pool, logger *slog.Logger, events *playev admin.Get("/lidarr/config", h.handleGetLidarrConfig) admin.Put("/lidarr/config", h.handlePutLidarrConfig) admin.Post("/lidarr/test", h.handleTestLidarrConnection) + admin.Get("/acquisition/services", h.handleListAcquisitionServices) + admin.Post("/acquisition/services", h.handleCreateAcquisitionService) + admin.Put("/acquisition/services/{code}", h.handleUpdateAcquisitionService) + admin.Delete("/acquisition/services/{code}", h.handleDeleteAcquisitionService) + admin.Post("/acquisition/services/{code}/test", h.handleTestAcquisitionService) + admin.Put("/acquisition/services/{code}/api-key", h.handleSetAcquisitionSecret("api_key")) + admin.Delete("/acquisition/services/{code}/api-key", h.handleSetAcquisitionSecret("api_key")) + admin.Put("/acquisition/services/{code}/password", h.handleSetAcquisitionSecret("password")) + admin.Delete("/acquisition/services/{code}/password", h.handleSetAcquisitionSecret("password")) admin.Get("/lidarr/quality-profiles", h.handleListQualityProfiles) admin.Get("/lidarr/metadata-profiles", h.handleListMetadataProfiles) admin.Get("/lidarr/root-folders", h.handleListRootFolders) @@ -372,6 +383,8 @@ type handlers struct { // acoustIDLookup is the AcoustID lookup worker (M401): its settings, its // status and "look up now". Nil serves the defaults (off). acoustIDLookup *library.AcoustIDLookupWorker + // acquisition is the operator's indexers and download clients (M502). + acquisition *acquire.Store // leveled renders the leveled streams handed to Sonos/UPnP speakers // (M464 #5001). Nil when its cache directory cannot be made: a level // request then gets the plain stream. diff --git a/internal/db/dbq/acquisition.sql.go b/internal/db/dbq/acquisition.sql.go new file mode 100644 index 00000000..2ab1b648 --- /dev/null +++ b/internal/db/dbq/acquisition.sql.go @@ -0,0 +1,362 @@ +// Code generated by sqlc. DO NOT EDIT. +// versions: +// sqlc v1.31.1 +// source: acquisition.sql + +package dbq + +import ( + "context" + + "github.com/jackc/pgx/v5/pgtype" +) + +const createAcquisitionService = `-- name: CreateAcquisitionService :one +INSERT INTO acquisition_services + (code, kind, protocol, label, category, timeout_ms, searches_per_hour) +VALUES + ($1, $2, $3, $4, + $5, $6, $7) +ON CONFLICT (code) DO NOTHING +RETURNING code, kind, protocol, label, enabled, base_url, api_key, username, password, category, timeout_ms, serves_search, serves_downloads, serves_enrichment, local_path, remote_path, searches_per_hour, created_at, updated_at +` + +type CreateAcquisitionServiceParams struct { + Code string + Kind string + Protocol string + Label string + Category string + TimeoutMs int32 + SearchesPerHour int32 +} + +// ON CONFLICT so the caller can try the next code when this one is taken. +func (q *Queries) CreateAcquisitionService(ctx context.Context, arg CreateAcquisitionServiceParams) (AcquisitionService, error) { + row := q.db.QueryRow(ctx, createAcquisitionService, + arg.Code, + arg.Kind, + arg.Protocol, + arg.Label, + arg.Category, + arg.TimeoutMs, + arg.SearchesPerHour, + ) + var i AcquisitionService + err := row.Scan( + &i.Code, + &i.Kind, + &i.Protocol, + &i.Label, + &i.Enabled, + &i.BaseUrl, + &i.ApiKey, + &i.Username, + &i.Password, + &i.Category, + &i.TimeoutMs, + &i.ServesSearch, + &i.ServesDownloads, + &i.ServesEnrichment, + &i.LocalPath, + &i.RemotePath, + &i.SearchesPerHour, + &i.CreatedAt, + &i.UpdatedAt, + ) + return i, err +} + +const deleteAcquisitionService = `-- name: DeleteAcquisitionService :execrows +DELETE FROM acquisition_services WHERE code = $1 +` + +func (q *Queries) DeleteAcquisitionService(ctx context.Context, code string) (int64, error) { + result, err := q.db.Exec(ctx, deleteAcquisitionService, code) + if err != nil { + return 0, err + } + return result.RowsAffected(), nil +} + +const getAcquisitionService = `-- name: GetAcquisitionService :one +SELECT code, kind, protocol, label, enabled, base_url, api_key, username, password, category, timeout_ms, serves_search, serves_downloads, serves_enrichment, local_path, remote_path, searches_per_hour, created_at, updated_at FROM acquisition_services WHERE code = $1 +` + +// The full row, credentials included. For building a client, never for a +// response. +func (q *Queries) GetAcquisitionService(ctx context.Context, code string) (AcquisitionService, error) { + row := q.db.QueryRow(ctx, getAcquisitionService, code) + var i AcquisitionService + err := row.Scan( + &i.Code, + &i.Kind, + &i.Protocol, + &i.Label, + &i.Enabled, + &i.BaseUrl, + &i.ApiKey, + &i.Username, + &i.Password, + &i.Category, + &i.TimeoutMs, + &i.ServesSearch, + &i.ServesDownloads, + &i.ServesEnrichment, + &i.LocalPath, + &i.RemotePath, + &i.SearchesPerHour, + &i.CreatedAt, + &i.UpdatedAt, + ) + return i, err +} + +const listAcquisitionServices = `-- name: ListAcquisitionServices :many + +SELECT code, kind, protocol, label, enabled, base_url, + (api_key <> '')::boolean AS has_api_key, + (password <> '')::boolean AS has_password, + username, category, timeout_ms, + serves_search, serves_downloads, serves_enrichment, + local_path, remote_path, searches_per_hour, + created_at, updated_at +FROM acquisition_services +ORDER BY kind, label, code +` + +type ListAcquisitionServicesRow struct { + Code string + Kind string + Protocol string + Label string + Enabled bool + BaseUrl string + HasApiKey bool + HasPassword bool + Username string + Category string + TimeoutMs int32 + ServesSearch bool + ServesDownloads bool + ServesEnrichment bool + LocalPath string + RemotePath string + SearchesPerHour int32 + CreatedAt pgtype.Timestamptz + UpdatedAt pgtype.Timestamptz +} + +// Indexers and download clients (M502 #5518). +// Returns no secret, only whether one is set: this feeds the settings page. +func (q *Queries) ListAcquisitionServices(ctx context.Context) ([]ListAcquisitionServicesRow, error) { + rows, err := q.db.Query(ctx, listAcquisitionServices) + if err != nil { + return nil, err + } + defer rows.Close() + var items []ListAcquisitionServicesRow + for rows.Next() { + var i ListAcquisitionServicesRow + if err := rows.Scan( + &i.Code, + &i.Kind, + &i.Protocol, + &i.Label, + &i.Enabled, + &i.BaseUrl, + &i.HasApiKey, + &i.HasPassword, + &i.Username, + &i.Category, + &i.TimeoutMs, + &i.ServesSearch, + &i.ServesDownloads, + &i.ServesEnrichment, + &i.LocalPath, + &i.RemotePath, + &i.SearchesPerHour, + &i.CreatedAt, + &i.UpdatedAt, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const listUsableAcquisitionServices = `-- name: ListUsableAcquisitionServices :many +SELECT code, kind, protocol, label, enabled, base_url, api_key, username, password, category, timeout_ms, serves_search, serves_downloads, serves_enrichment, local_path, remote_path, searches_per_hour, created_at, updated_at FROM acquisition_services +WHERE enabled AND base_url <> '' + AND CASE WHEN protocol = 'exchange' THEN + CASE $1::text + WHEN 'search' THEN serves_search + WHEN 'download' THEN serves_downloads + WHEN 'enrichment' THEN serves_enrichment + ELSE false + END + ELSE kind = $1::text + END +ORDER BY code +` + +// Every enabled, addressed service doing one job. An exchange row does a job +// when its switch for it is on; any other row does the job its kind names. +// Ordered by code so a fan-out is deterministic. +func (q *Queries) ListUsableAcquisitionServices(ctx context.Context, job string) ([]AcquisitionService, error) { + rows, err := q.db.Query(ctx, listUsableAcquisitionServices, job) + if err != nil { + return nil, err + } + defer rows.Close() + var items []AcquisitionService + for rows.Next() { + var i AcquisitionService + if err := rows.Scan( + &i.Code, + &i.Kind, + &i.Protocol, + &i.Label, + &i.Enabled, + &i.BaseUrl, + &i.ApiKey, + &i.Username, + &i.Password, + &i.Category, + &i.TimeoutMs, + &i.ServesSearch, + &i.ServesDownloads, + &i.ServesEnrichment, + &i.LocalPath, + &i.RemotePath, + &i.SearchesPerHour, + &i.CreatedAt, + &i.UpdatedAt, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const setAcquisitionServiceAPIKey = `-- name: SetAcquisitionServiceAPIKey :execrows +UPDATE acquisition_services SET api_key = $1, updated_at = now() +WHERE code = $2 +` + +type SetAcquisitionServiceAPIKeyParams struct { + ApiKey string + Code string +} + +func (q *Queries) SetAcquisitionServiceAPIKey(ctx context.Context, arg SetAcquisitionServiceAPIKeyParams) (int64, error) { + result, err := q.db.Exec(ctx, setAcquisitionServiceAPIKey, arg.ApiKey, arg.Code) + if err != nil { + return 0, err + } + return result.RowsAffected(), nil +} + +const setAcquisitionServicePassword = `-- name: SetAcquisitionServicePassword :execrows +UPDATE acquisition_services SET password = $1, updated_at = now() +WHERE code = $2 +` + +type SetAcquisitionServicePasswordParams struct { + Password string + Code string +} + +func (q *Queries) SetAcquisitionServicePassword(ctx context.Context, arg SetAcquisitionServicePasswordParams) (int64, error) { + result, err := q.db.Exec(ctx, setAcquisitionServicePassword, arg.Password, arg.Code) + if err != nil { + return 0, err + } + return result.RowsAffected(), nil +} + +const updateAcquisitionService = `-- name: UpdateAcquisitionService :one +UPDATE acquisition_services SET + label = $1, + enabled = $2, + base_url = $3, + username = $4, + category = $5, + timeout_ms = $6, + serves_search = $7, + serves_downloads = $8, + serves_enrichment = $9, + local_path = $10, + remote_path = $11, + searches_per_hour = $12, + updated_at = now() +WHERE code = $13 +RETURNING code, kind, protocol, label, enabled, base_url, api_key, username, password, category, timeout_ms, serves_search, serves_downloads, serves_enrichment, local_path, remote_path, searches_per_hour, created_at, updated_at +` + +type UpdateAcquisitionServiceParams struct { + Label string + Enabled bool + BaseUrl string + Username string + Category string + TimeoutMs int32 + ServesSearch bool + ServesDownloads bool + ServesEnrichment bool + LocalPath string + RemotePath string + SearchesPerHour int32 + Code string +} + +// Everything on the settings form except the credentials, which have their +// own write-only calls. +func (q *Queries) UpdateAcquisitionService(ctx context.Context, arg UpdateAcquisitionServiceParams) (AcquisitionService, error) { + row := q.db.QueryRow(ctx, updateAcquisitionService, + arg.Label, + arg.Enabled, + arg.BaseUrl, + arg.Username, + arg.Category, + arg.TimeoutMs, + arg.ServesSearch, + arg.ServesDownloads, + arg.ServesEnrichment, + arg.LocalPath, + arg.RemotePath, + arg.SearchesPerHour, + arg.Code, + ) + var i AcquisitionService + err := row.Scan( + &i.Code, + &i.Kind, + &i.Protocol, + &i.Label, + &i.Enabled, + &i.BaseUrl, + &i.ApiKey, + &i.Username, + &i.Password, + &i.Category, + &i.TimeoutMs, + &i.ServesSearch, + &i.ServesDownloads, + &i.ServesEnrichment, + &i.LocalPath, + &i.RemotePath, + &i.SearchesPerHour, + &i.CreatedAt, + &i.UpdatedAt, + ) + return i, err +} diff --git a/internal/db/dbq/models.go b/internal/db/dbq/models.go index bcaa80ca..64c066c6 100644 --- a/internal/db/dbq/models.go +++ b/internal/db/dbq/models.go @@ -195,6 +195,28 @@ type AcoustidSetting struct { UpdatedAt pgtype.Timestamptz } +type AcquisitionService struct { + Code string + Kind string + Protocol string + Label string + Enabled bool + BaseUrl string + ApiKey string + Username string + Password string + Category string + TimeoutMs int32 + ServesSearch bool + ServesDownloads bool + ServesEnrichment bool + LocalPath string + RemotePath string + SearchesPerHour int32 + CreatedAt pgtype.Timestamptz + UpdatedAt pgtype.Timestamptz +} + type Album struct { ID pgtype.UUID Title string diff --git a/internal/db/migrations/0079_acquisition_services.down.sql b/internal/db/migrations/0079_acquisition_services.down.sql new file mode 100644 index 00000000..49d5d65c --- /dev/null +++ b/internal/db/migrations/0079_acquisition_services.down.sql @@ -0,0 +1 @@ +DROP TABLE IF EXISTS acquisition_services; diff --git a/internal/db/migrations/0079_acquisition_services.up.sql b/internal/db/migrations/0079_acquisition_services.up.sql new file mode 100644 index 00000000..b3b1bc66 --- /dev/null +++ b/internal/db/migrations/0079_acquisition_services.up.sql @@ -0,0 +1,63 @@ +-- 0079_acquisition_services.up.sql — the indexers and download clients the +-- operator adds in Admin → Integrations (Scribe milestone #502, #5518). +-- +-- Minstrel replaces Lidarr, so it searches and downloads itself. Nothing is +-- seeded and nothing is copied from Lidarr: a connection is a row the operator +-- adds, from a list of what Minstrel can speak. Ported from Fabled +-- Librarian's acquisition_services (its migrations 0044, 0055, 0062, 0064, +-- 0070), which settled the shape: +-- +-- kind which list the row appears in: search (Indexers) or download +-- (Download clients). +-- protocol which client is built: prowlarr (its native /api/v1/search, +-- because Prowlarr has no aggregate Torznab feed), torznab (any +-- single feed), qbittorrent (the WebUI v2), exchange. +-- +-- FabledExchange is ONE row under Download clients with a switch per job: +-- downloading, searching its DHT index, and describing a torrent's files and +-- swarm before a grab. The serves_* switches mean something on an exchange +-- row only; every other row does the one job its kind names. +-- +-- Secrets (api_key, password) are write-only: no query the API answers with +-- returns them, only whether one is set. +CREATE TABLE acquisition_services ( + -- code is the stable identity: the protocol for the first of its kind, + -- then protocol-2, protocol-3. Legible in a log, never typed. + code text PRIMARY KEY, + kind text NOT NULL + CONSTRAINT acquisition_services_kind_check CHECK (kind IN ('search', 'download')), + protocol text NOT NULL + CONSTRAINT acquisition_services_protocol_check + CHECK (protocol IN ('prowlarr', 'torznab', 'qbittorrent', 'exchange')), + label text NOT NULL, + -- A new row is off: it has no address yet. + enabled boolean NOT NULL DEFAULT false, + -- The address as the operator's browser opens it. For a Torznab feed it + -- is the whole feed URL up to and including /api. + base_url text NOT NULL DEFAULT '', + api_key text NOT NULL DEFAULT '', + -- qBittorrent only. Both may be empty: a WebUI with auth bypassed for + -- local addresses takes neither. + username text NOT NULL DEFAULT '', + password text NOT NULL DEFAULT '', + -- Download clients: everything Minstrel adds is filed under it. + category text NOT NULL DEFAULT '', + timeout_ms integer NOT NULL DEFAULT 30000 + CONSTRAINT acquisition_services_timeout_check CHECK (timeout_ms BETWEEN 1000 AND 300000), + serves_search boolean NOT NULL DEFAULT true, + serves_downloads boolean NOT NULL DEFAULT true, + serves_enrichment boolean NOT NULL DEFAULT true, + -- One downloads folder, two names: where Minstrel has it mounted, and + -- where the client sees it (empty = the same path, nothing translated). + local_path text NOT NULL DEFAULT '/downloads' + CONSTRAINT acquisition_services_local_path_check CHECK (local_path LIKE '/%'), + remote_path text NOT NULL DEFAULT '' + CONSTRAINT acquisition_services_remote_path_check CHECK (remote_path = '' OR remote_path LIKE '/%'), + -- The most searches the schedule sends this source an hour; 0 = unpaced. + -- Prowlarr's indexers answer 429 when asked too often; an Exchange + -- searches its own database and starts unpaced. + searches_per_hour integer NOT NULL DEFAULT 120 + CONSTRAINT acquisition_services_searches_per_hour_check CHECK (searches_per_hour BETWEEN 0 AND 3600), + created_at timestamptz NOT NULL DEFAULT now(), + updated_at timestamptz NOT NULL DEFAULT now() +); diff --git a/internal/db/queries/acquisition.sql b/internal/db/queries/acquisition.sql new file mode 100644 index 00000000..139b4db5 --- /dev/null +++ b/internal/db/queries/acquisition.sql @@ -0,0 +1,76 @@ +-- Indexers and download clients (M502 #5518). + +-- name: ListAcquisitionServices :many +-- Returns no secret, only whether one is set: this feeds the settings page. +SELECT code, kind, protocol, label, enabled, base_url, + (api_key <> '')::boolean AS has_api_key, + (password <> '')::boolean AS has_password, + username, category, timeout_ms, + serves_search, serves_downloads, serves_enrichment, + local_path, remote_path, searches_per_hour, + created_at, updated_at +FROM acquisition_services +ORDER BY kind, label, code; + +-- name: GetAcquisitionService :one +-- The full row, credentials included. For building a client, never for a +-- response. +SELECT * FROM acquisition_services WHERE code = sqlc.arg(code); + +-- name: ListUsableAcquisitionServices :many +-- Every enabled, addressed service doing one job. An exchange row does a job +-- when its switch for it is on; any other row does the job its kind names. +-- Ordered by code so a fan-out is deterministic. +SELECT * FROM acquisition_services +WHERE enabled AND base_url <> '' + AND CASE WHEN protocol = 'exchange' THEN + CASE sqlc.arg(job)::text + WHEN 'search' THEN serves_search + WHEN 'download' THEN serves_downloads + WHEN 'enrichment' THEN serves_enrichment + ELSE false + END + ELSE kind = sqlc.arg(job)::text + END +ORDER BY code; + +-- name: CreateAcquisitionService :one +-- ON CONFLICT so the caller can try the next code when this one is taken. +INSERT INTO acquisition_services + (code, kind, protocol, label, category, timeout_ms, searches_per_hour) +VALUES + (sqlc.arg(code), sqlc.arg(kind), sqlc.arg(protocol), sqlc.arg(label), + sqlc.arg(category), sqlc.arg(timeout_ms), sqlc.arg(searches_per_hour)) +ON CONFLICT (code) DO NOTHING +RETURNING *; + +-- name: UpdateAcquisitionService :one +-- Everything on the settings form except the credentials, which have their +-- own write-only calls. +UPDATE acquisition_services SET + label = sqlc.arg(label), + enabled = sqlc.arg(enabled), + base_url = sqlc.arg(base_url), + username = sqlc.arg(username), + category = sqlc.arg(category), + timeout_ms = sqlc.arg(timeout_ms), + serves_search = sqlc.arg(serves_search), + serves_downloads = sqlc.arg(serves_downloads), + serves_enrichment = sqlc.arg(serves_enrichment), + local_path = sqlc.arg(local_path), + remote_path = sqlc.arg(remote_path), + searches_per_hour = sqlc.arg(searches_per_hour), + updated_at = now() +WHERE code = sqlc.arg(code) +RETURNING *; + +-- name: DeleteAcquisitionService :execrows +DELETE FROM acquisition_services WHERE code = sqlc.arg(code); + +-- name: SetAcquisitionServiceAPIKey :execrows +UPDATE acquisition_services SET api_key = sqlc.arg(api_key), updated_at = now() +WHERE code = sqlc.arg(code); + +-- name: SetAcquisitionServicePassword :execrows +UPDATE acquisition_services SET password = sqlc.arg(password), updated_at = now() +WHERE code = sqlc.arg(code); diff --git a/internal/dbtest/reset.go b/internal/dbtest/reset.go index 2d80157a..7824ccf0 100644 --- a/internal/dbtest/reset.go +++ b/internal/dbtest/reset.go @@ -102,6 +102,7 @@ var dataTables = []string{ "user_notifications", "user_notification_prefs", "user_notification_email_state", // #5346 + "acquisition_services", // M502 #5518 "tracks", "albums", "artists", diff --git a/internal/downloader/downloader_test.go b/internal/downloader/downloader_test.go new file mode 100644 index 00000000..5af62b57 --- /dev/null +++ b/internal/downloader/downloader_test.go @@ -0,0 +1,156 @@ +package downloader + +import ( + "context" + "errors" + "io" + "net/http" + "net/http/httptest" + "strings" + "sync/atomic" + "testing" +) + +func TestPathMap(t *testing.T) { + m := PathMap{Local: "/downloads", Remote: "/data/torrents/"} + for in, want := range map[string]string{ + "/downloads": "/data/torrents", + "/downloads/abc/x.mp3": "/data/torrents/abc/x.mp3", + "/downloads2/abc": "/downloads2/abc", + "/music/a": "/music/a", + "": "", + } { + if got := m.ToRemote(in); got != want { + t.Errorf("ToRemote(%q) = %q, want %q", in, got, want) + } + } + if got := m.ToLocal("/data/torrents/abc"); got != "/downloads/abc" { + t.Errorf("ToLocal = %q", got) + } + if (PathMap{Local: "/downloads", Remote: "/downloads/"}).Active() { + t.Error("the same folder under one name is not a mapping") + } +} + +// fakeQbit is a qBittorrent WebUI that wants a login, and that expires the +// session once to make the client log in again. +func fakeQbit(t *testing.T, expireOnce bool) (*httptest.Server, *atomic.Int32) { + t.Helper() + var logins atomic.Int32 + var expired atomic.Bool + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Header.Get("Referer") == "" { + w.WriteHeader(http.StatusForbidden) + return + } + switch r.URL.Path { + case "/api/v2/auth/login": + _ = r.ParseForm() + if r.Form.Get("password") != "pw" { + _, _ = w.Write([]byte("Fails.")) + return + } + logins.Add(1) + http.SetCookie(w, &http.Cookie{Name: "SID", Value: "s", Path: "/"}) + _, _ = w.Write([]byte("Ok.")) + case "/api/v2/torrents/info": + if _, err := r.Cookie("SID"); err != nil { + w.WriteHeader(http.StatusForbidden) + return + } + if expireOnce && !expired.Swap(true) { + w.WriteHeader(http.StatusForbidden) + return + } + if r.URL.Query().Get("category") != "minstrel" { + t.Errorf("category = %q", r.URL.Query().Get("category")) + } + _, _ = w.Write([]byte(`[{"hash": "ABC", "name": "x", "save_path": "/data/torrents/abc", "progress": 1}]`)) + case "/api/v2/torrents/add": + body, _ := io.ReadAll(r.Body) + if !strings.Contains(string(body), "/data/torrents/new") { + t.Errorf("add body has no translated save path: %s", body) + } + _, _ = w.Write([]byte("Ok.")) + default: + w.WriteHeader(http.StatusNotFound) + } + })) + t.Cleanup(srv.Close) + return srv, &logins +} + +func TestQBittorrent_LogsInAndLogsInAgainWhenTheSessionExpires(t *testing.T) { + srv, logins := fakeQbit(t, true) + d := Mapped(&Client{Config: Config{BaseURL: srv.URL, Username: "u", Password: "pw", Category: "minstrel"}}, + PathMap{Local: "/downloads", Remote: "/data/torrents"}) + ts, err := d.Torrents(context.Background()) + if err != nil { + t.Fatalf("torrents: %v", err) + } + if logins.Load() != 2 { + t.Errorf("logins = %d, want 2 (the second after the 403)", logins.Load()) + } + if len(ts) != 1 || ts[0].Hash != "abc" || ts[0].SavePath != "/downloads/abc" || !ts[0].Done() { + t.Errorf("torrents = %+v, want one, hash lowercased, path in Minstrel's terms", ts) + } + if err := d.AddTorrent(context.Background(), Add{Magnet: "magnet:?xt=urn:btih:abc", SavePath: "/downloads/new"}); err != nil { + t.Errorf("add: %v", err) + } +} + +func TestQBittorrent_AWrongPasswordIsAuth(t *testing.T) { + srv, _ := fakeQbit(t, false) + c := &Client{Config: Config{BaseURL: srv.URL, Password: "nope"}} + if _, err := c.Torrents(context.Background()); !errors.Is(err, ErrAuth) { + t.Fatalf("err = %v, want ErrAuth", err) + } +} + +func TestExchange_MapsDownloadsAndRefusesWhileRestoring(t *testing.T) { + var restoring, sentKey atomic.Bool + restoring.Store(true) + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Header.Get("Authorization") != "" { + sentKey.Store(true) + } + pending := "0" + if restoring.Load() { + pending = "3" + } + _, _ = io.WriteString(w, `{"enabled": true, "running": false, "held_reason": "disk full", "restore_pending": `+pending+`, "downloads": [ + {"infohash": "ABC", "name": "Album", "save_path": "/dl", "state": "seeding", "progress": 1, "length": 10, "bytes": 10, "category": "minstrel"}, + {"infohash": "DEF", "name": "Other", "save_path": "/dl", "state": "fast", "category": "someone-else"}, + {"infohash": "123", "name": "Broken", "save_path": "/dl", "failed": true, "fail_reason": "tracker gone", "category": "minstrel"} + ]}`) + })) + defer srv.Close() + + c := &Exchange{Config: Config{BaseURL: srv.URL, Category: "minstrel"}} + if _, err := c.Torrents(context.Background()); !errors.Is(err, ErrRestoring) { + t.Fatalf("while restoring: err = %v, want ErrRestoring", err) + } + if sentKey.Load() { + t.Error("an Authorization header was sent with no key set") + } + restoring.Store(false) + ts, err := c.Torrents(context.Background()) + if err != nil { + t.Fatalf("torrents: %v", err) + } + if len(ts) != 2 { + t.Fatalf("torrents = %+v, want the two in Minstrel's category", ts) + } + if ts[0].Hash != "abc" || ts[0].State != "uploading" || ts[0].ContentPath != "/dl/Album" || !strings.Contains(ts[0].EngineDown, "disk full") { + t.Errorf("first = %+v", ts[0]) + } + if !ts[1].Failed() || ts[1].FailReason != "tracker gone" { + t.Errorf("second = %+v, want failed with Exchange's reason", ts[1]) + } +} + +func TestFor_RefusesAnUnknownProtocol(t *testing.T) { + if _, err := For("transmission", Config{}); err == nil { + t.Fatal("want an error for a client Minstrel cannot drive") + } +} diff --git a/internal/downloader/driver.go b/internal/downloader/driver.go new file mode 100644 index 00000000..90e3bf9d --- /dev/null +++ b/internal/downloader/driver.go @@ -0,0 +1,198 @@ +// Package downloader drives the download clients Minstrel hands a grab to +// (M502 #5518): qBittorrent's WebUI v2, and FabledExchange's native +// /api/downloads. Ported from Fabled Librarian's internal/downloader. +// +// Both answer through one Driver, and Torrent is qBittorrent's shape: the +// Exchange driver maps its own downloads onto it, so the grab, the importer +// and the stall watchdog read one kind of record whichever client holds the +// torrent. +package downloader + +import ( + "context" + "errors" + "fmt" + "time" +) + +// Protocols a download client row may name. +const ( + KindQBittorrent = "qbittorrent" + KindExchange = "exchange" +) + +// ErrUnconfigured is a client with no address. +var ErrUnconfigured = errors.New("download client is not configured") + +// ErrUnreachable is a call that could not be made, or a refusal that is not +// about credentials. +var ErrUnreachable = errors.New("download client unreachable") + +// ErrAuth is credentials the client refused. +var ErrAuth = errors.New("download client refused the credentials") + +// ErrRestoring is an Exchange still bringing its downloads back after a +// restart: its list is incomplete, so nothing may be concluded from it. +var ErrRestoring = errors.New("download client is still restoring its downloads after a restart") + +// ErrDuplicate is a torrent the client already holds. +var ErrDuplicate = errors.New("download client already holds this torrent") + +// ErrUnsupported is a call this client has no equivalent for. +var ErrUnsupported = errors.New("download client cannot do that") + +// UserAgent names Minstrel in the client's logs. +const UserAgent = "Minstrel (self-hosted music server; download client)" + +// DefaultTimeout is what an unset Timeout means: a download client answers +// from its own state. +const DefaultTimeout = 15 * time.Second + +// Config is one download client. +type Config struct { + BaseURL string + // Username and Password are qBittorrent's. Both may be empty for a WebUI + // that bypasses auth for local addresses. + Username string + Password string + // APIKey is Exchange's, sent as a bearer only when set. + APIKey string + // Category files everything Minstrel adds, so its torrents can be told + // from the operator's own. + Category string + Timeout time.Duration +} + +// Driver is what Minstrel needs from a download client. +type Driver interface { + AddTorrent(ctx context.Context, a Add) error + // Torrents lists this client's torrents in Config.Category, or only the + // given hashes. + Torrents(ctx context.Context, hashes ...string) ([]Torrent, error) + // Connected says whether the client can reach any peer at all. + Connected(ctx context.Context) (bool, error) + Stop(ctx context.Context, hashes ...string) error + Start(ctx context.Context, hashes ...string) error + Reannounce(ctx context.Context, hashes ...string) error + Recheck(ctx context.Context, hashes ...string) error + Delete(ctx context.Context, deleteFiles bool, hashes ...string) error +} + +// FileSkipper sets which files of a torrent are downloaded. +type FileSkipper interface { + SkipFiles(ctx context.Context, hash string, indices []int) error + KeepFiles(ctx context.Context, hash string, indices []int) error +} + +// File is one file inside a torrent. +type File struct { + Index int + Path string + Size int64 +} + +// FileLister lists a torrent's files. +type FileLister interface { + Files(ctx context.Context, hash string) ([]File, error) +} + +// FileState is one file's progress. +type FileState struct { + Path string + Size int64 + Done int64 + Skipped bool +} + +// Unfinished says whether the file is wanted and not yet whole. +func (f FileState) Unfinished() bool { return !f.Skipped && f.Done < f.Size } + +// FileProgress reports per-file progress. +type FileProgress interface { + FileStates(ctx context.Context, hash string) ([]FileState, error) +} + +var ( + _ Driver = (*Client)(nil) + _ Driver = (*Exchange)(nil) + _ FileSkipper = (*Client)(nil) + _ FileSkipper = (*Exchange)(nil) + _ FileSkipper = (*mapped)(nil) + _ FileLister = (*Client)(nil) + _ FileLister = (*Exchange)(nil) + _ FileLister = (*mapped)(nil) + _ FileProgress = (*Client)(nil) + _ FileProgress = (*Exchange)(nil) + _ FileProgress = (*mapped)(nil) +) + +// For builds the driver a protocol names. +func For(protocol string, cfg Config) (Driver, error) { + switch protocol { + case KindQBittorrent: + return &Client{Config: cfg}, nil + case KindExchange: + return &Exchange{Config: cfg}, nil + } + return nil, fmt.Errorf("download client %q is not one Minstrel can drive", protocol) +} + +// Torrent is one torrent in qBittorrent's JSON shape. +type Torrent struct { + Hash string `json:"hash"` + Name string `json:"name"` + Size int64 `json:"size"` + Progress float64 `json:"progress"` + State string `json:"state"` + Category string `json:"category"` + Tags string `json:"tags"` + SavePath string `json:"save_path"` + ContentPath string `json:"content_path"` + AddedOn int64 `json:"added_on"` + NumSeeds int `json:"num_seeds"` + NumLeechs int `json:"num_leechs"` + AmountLeft int64 `json:"amount_left"` + Ratio float64 `json:"ratio"` + DLSpeed int64 `json:"dlspeed"` + ETA int64 `json:"eta"` + // EngineDown is why the client's whole engine is not moving anything, in + // words; empty when it is running. Exchange says so; qBittorrent does not. + EngineDown string `json:"-"` + // FailReason is the client's own reason for a failed torrent. + FailReason string `json:"-"` +} + +// Done says the torrent is whole. +func (t Torrent) Done() bool { return t.Progress >= 1 } + +// Failed says the client gave up on it. +func (t Torrent) Failed() bool { return t.State == "error" || t.State == "missingFiles" } + +// Fetching says it is still fetching metadata. +func (t Torrent) Fetching() bool { return t.State == "metaDL" || t.State == "forcedMetaDL" } + +// Alone says nobody is sharing an unfinished torrent. +func (t Torrent) Alone() bool { return !t.Done() && t.NumSeeds == 0 && t.NumLeechs == 0 } + +// Added is when the client took it. +func (t Torrent) Added() time.Time { + if t.AddedOn <= 0 { + return time.Time{} + } + return time.Unix(t.AddedOn, 0) +} + +// Add is one torrent to hand a client: a magnet or the .torrent bytes. +type Add struct { + Magnet string + Torrent []byte + // SavePath is where it lands, as Minstrel sees it; Mapped translates it. + // Required: a download with no save path lands outside the folder + // Minstrel imports from. + SavePath string + Tags []string + Paused bool + // StopAtMetadata asks qBittorrent to stop once it has the file list, so + // unwanted files can be skipped before any data moves. + StopAtMetadata bool +} diff --git a/internal/downloader/exchange.go b/internal/downloader/exchange.go new file mode 100644 index 00000000..e1b04fd0 --- /dev/null +++ b/internal/downloader/exchange.go @@ -0,0 +1,340 @@ +package downloader + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "net/url" + "path" + "strings" + "time" +) + +// Exchange drives FabledExchange's native /api/downloads: not its qBittorrent +// facade, so its own states, failure reasons and engine status come through. +type Exchange struct { + Config Config + // HTTP is injectable for tests. Nil takes a plain client. + HTTP *http.Client +} + +type exchangeItem struct { + InfoHash string `json:"infohash"` + Name string `json:"name"` + Bytes int64 `json:"bytes"` + Length int64 `json:"length"` + Progress float64 `json:"progress"` + State string `json:"state"` + Peers int `json:"peers"` + Category string `json:"category"` + Tags []string `json:"tags"` + SavePath string `json:"save_path"` + AddedAt time.Time `json:"added_at"` + Failed bool `json:"failed"` + FailReason string `json:"fail_reason"` + Ratio float64 `json:"ratio"` +} + +type exchangeSnapshot struct { + Enabled bool `json:"enabled"` + Running bool `json:"running"` + HeldReason string `json:"held_reason"` + InboundConns int64 `json:"inbound_conns"` + RestorePending int `json:"restore_pending"` + Downloads []exchangeItem `json:"downloads"` +} + +func (c *Exchange) AddTorrent(ctx context.Context, a Add) error { + if a.Magnet == "" && len(a.Torrent) == 0 { + return fmt.Errorf("add: neither a magnet nor torrent bytes") + } + if a.SavePath == "" { + return fmt.Errorf("add: no save path, which would land the download outside the folder Minstrel imports from") + } + if len(a.Torrent) > 0 { + q := url.Values{"save_path": {a.SavePath}} + if c.Config.Category != "" { + q.Set("category", c.Config.Category) + } + for _, t := range a.Tags { + q.Add("tag", t) + } + if a.Paused { + q.Set("paused", "true") + } + _, err := c.call(ctx, http.MethodPost, "/api/downloads/torrent?"+q.Encode(), "application/x-bittorrent", a.Torrent) + return err + } + body, err := json.Marshal(map[string]any{ + "magnet": a.Magnet, "category": c.Config.Category, "save_path": a.SavePath, + "tags": a.Tags, "paused": a.Paused, + }) + if err != nil { + return fmt.Errorf("add: %w", err) + } + _, err = c.call(ctx, http.MethodPost, "/api/downloads", "application/json", body) + return err +} + +// Torrents maps Exchange's downloads onto qBittorrent's shape. While Exchange +// is still restoring after a restart its list is incomplete, so it answers +// ErrRestoring rather than a list that looks like downloads vanished. +func (c *Exchange) Torrents(ctx context.Context, hashes ...string) ([]Torrent, error) { + snap, err := c.snapshot(ctx) + if err != nil { + return nil, err + } + if snap.RestorePending > 0 { + return nil, fmt.Errorf("%w (%d still to come back)", ErrRestoring, snap.RestorePending) + } + want := map[string]bool{} + for _, h := range hashes { + want[strings.ToLower(strings.TrimSpace(h))] = true + } + down := "" + switch { + case snap.Running: + case !snap.Enabled: + down = "the download engine is switched off in FabledExchange" + case snap.HeldReason != "": + down = "the download engine is held (" + snap.HeldReason + ") and resumes by itself" + default: + down = "the download engine is enabled but not running; FabledExchange is retrying it" + } + out := make([]Torrent, 0, len(snap.Downloads)) + for _, it := range snap.Downloads { + if c.Config.Category != "" && it.Category != c.Config.Category { + continue + } + t := asTorrent(it) + t.EngineDown = down + if len(want) > 0 && !want[t.Hash] { + continue + } + out = append(out, t) + } + return out, nil +} + +func asTorrent(it exchangeItem) Torrent { + content := it.SavePath + if it.Name != "" { + content = path.Join(it.SavePath, it.Name) + } + left := max(it.Length-it.Bytes, 0) + state := exchangeState(it.State, it.Progress) + if it.Failed { + state = "error" + } + var added int64 + if !it.AddedAt.IsZero() { + added = it.AddedAt.Unix() + } + return Torrent{ + Hash: strings.ToLower(it.InfoHash), Name: it.Name, Size: it.Length, + Progress: it.Progress, State: state, Category: it.Category, + Tags: strings.Join(it.Tags, ","), SavePath: it.SavePath, ContentPath: content, + AddedOn: added, NumLeechs: it.Peers, AmountLeft: left, + FailReason: it.FailReason, Ratio: it.Ratio, + } +} + +// exchangeState names Exchange's state in qBittorrent's vocabulary. +func exchangeState(tier string, progress float64) string { + switch tier { + case "fast", "slow": + return "downloading" + case "metadata": + return "metaDL" + case "stalled": + return "stalledDL" + case "queued": + return "queuedDL" + case "seeding": + return "uploading" + case "relocating": + return "moving" + case "paused": + if progress >= 1 { + return "pausedUP" + } + return "pausedDL" + } + return "unknown" +} + +// Connected is a running engine that has had an inbound connection: the +// nearest Exchange comes to qBittorrent's "connected". +func (c *Exchange) Connected(ctx context.Context) (bool, error) { + snap, err := c.snapshot(ctx) + if err != nil { + return false, err + } + return snap.Running && snap.InboundConns > 0, nil +} + +func (c *Exchange) Stop(ctx context.Context, hashes ...string) error { + return c.each(ctx, hashes, func(h string) error { + _, err := c.call(ctx, http.MethodPost, "/api/downloads/"+url.PathEscape(h)+"/pause", "", nil) + return err + }) +} + +func (c *Exchange) Start(ctx context.Context, hashes ...string) error { + return c.each(ctx, hashes, func(h string) error { + _, err := c.call(ctx, http.MethodPost, "/api/downloads/"+url.PathEscape(h)+"/resume", "", nil) + return err + }) +} + +func (c *Exchange) SkipFiles(ctx context.Context, hash string, indices []int) error { + return c.filePriority(ctx, hash, indices, "skip") +} + +func (c *Exchange) KeepFiles(ctx context.Context, hash string, indices []int) error { + return c.filePriority(ctx, hash, indices, "normal") +} + +func (c *Exchange) filePriority(ctx context.Context, hash string, indices []int, priority string) error { + if len(indices) == 0 { + return nil + } + body, err := json.Marshal(map[string]any{"indices": indices, "priority": priority}) + if err != nil { + return err + } + _, err = c.call(ctx, http.MethodPatch, "/api/downloads/"+url.PathEscape(hash)+"/files", "application/json", body) + return err +} + +func (c *Exchange) Files(ctx context.Context, hash string) ([]File, error) { + raw, err := c.files(ctx, hash) + if err != nil { + return nil, err + } + out := make([]File, len(raw)) + for i, f := range raw { + out[i] = File{Index: f.Index, Path: f.Path, Size: f.Length} + } + return out, nil +} + +func (c *Exchange) FileStates(ctx context.Context, hash string) ([]FileState, error) { + raw, err := c.files(ctx, hash) + if err != nil { + return nil, err + } + out := make([]FileState, len(raw)) + for i, f := range raw { + if f.Bytes == nil { + return nil, fmt.Errorf("%w: FabledExchange does not say how much of each file it has", ErrUnsupported) + } + out[i] = FileState{Path: f.Path, Size: f.Length, Done: *f.Bytes, Skipped: f.Priority == "skip"} + } + return out, nil +} + +type exchangeFile struct { + Index int `json:"index"` + Path string `json:"path"` + Length int64 `json:"length"` + Bytes *int64 `json:"bytes"` + Priority string `json:"priority"` +} + +func (c *Exchange) files(ctx context.Context, hash string) ([]exchangeFile, error) { + body, err := c.call(ctx, http.MethodGet, "/api/downloads/"+url.PathEscape(hash), "", nil) + if err != nil { + return nil, err + } + var detail struct { + Files []exchangeFile `json:"files"` + } + if err := json.Unmarshal(body, &detail); err != nil { + return nil, fmt.Errorf("%w: parse download detail: %v", ErrUnreachable, err) + } + return detail.Files, nil +} + +func (c *Exchange) Reannounce(context.Context, ...string) error { + return fmt.Errorf("%w: FabledExchange schedules its own announces", ErrUnsupported) +} + +func (c *Exchange) Recheck(context.Context, ...string) error { + return fmt.Errorf("%w: FabledExchange has no recheck", ErrUnsupported) +} + +func (c *Exchange) Delete(ctx context.Context, deleteFiles bool, hashes ...string) error { + return c.each(ctx, hashes, func(h string) error { + p := "/api/downloads/" + url.PathEscape(h) + if deleteFiles { + p += "?delete_files=true" + } + _, err := c.call(ctx, http.MethodDelete, p, "", nil) + return err + }) +} + +func (c *Exchange) each(ctx context.Context, hashes []string, do func(string) error) error { + for _, h := range hashes { + if ctx.Err() != nil { + return ctx.Err() + } + if err := do(strings.ToLower(strings.TrimSpace(h))); err != nil { + return err + } + } + return nil +} + +func (c *Exchange) snapshot(ctx context.Context) (exchangeSnapshot, error) { + var snap exchangeSnapshot + body, err := c.call(ctx, http.MethodGet, "/api/downloads", "", nil) + if err != nil { + return snap, err + } + if err := json.Unmarshal(body, &snap); err != nil { + return snap, fmt.Errorf("%w: parse download list: %v", ErrUnreachable, err) + } + return snap, nil +} + +func (c *Exchange) call(ctx context.Context, method, p, contentType string, body []byte) ([]byte, error) { + if c.Config.BaseURL == "" { + return nil, ErrUnconfigured + } + httpc := c.HTTP + if httpc == nil { + httpc = &http.Client{} + } + status, out, err := send(ctx, httpc, c.Config.Timeout, method, + strings.TrimRight(c.Config.BaseURL, "/")+p, contentType, body, func(h http.Header) { + h.Set("Accept", "application/json") + if c.Config.APIKey != "" { + h.Set("Authorization", "Bearer "+c.Config.APIKey) + } + }) + if err != nil { + return nil, err + } + if status >= 200 && status < 300 { + return out, nil + } + var e struct { + Error string `json:"error"` + Reason string `json:"reason"` + } + _ = json.Unmarshal(out, &e) + said := strings.TrimSpace(e.Error) + if said == "" { + said = fmt.Sprintf("HTTP %d", status) + } + switch { + case status == http.StatusUnauthorized || status == http.StatusForbidden: + return nil, fmt.Errorf("%w: FabledExchange refused the API key (%s)", ErrAuth, said) + case status == http.StatusConflict && e.Reason == "duplicate": + return nil, fmt.Errorf("%w: %s", ErrDuplicate, said) + } + return nil, fmt.Errorf("%w: %s", ErrUnreachable, said) +} diff --git a/internal/downloader/pathmap.go b/internal/downloader/pathmap.go new file mode 100644 index 00000000..be43772e --- /dev/null +++ b/internal/downloader/pathmap.go @@ -0,0 +1,107 @@ +package downloader + +import ( + "context" + "path" + "strings" +) + +// PathMap is one downloads folder under two names: where Minstrel has it +// mounted (Local) and where the client sees it (Remote). The two usually run +// in different containers. Every path crossing between them swaps one prefix +// for the other. +type PathMap struct { + Local string + Remote string +} + +// Active says the two names differ, so there is something to translate. +func (m PathMap) Active() bool { + return m.Remote != "" && m.Local != "" && path.Clean(m.Remote) != path.Clean(m.Local) +} + +// ToRemote is a Minstrel path as the client names it. +func (m PathMap) ToRemote(p string) string { return swap(p, m.Local, m.Remote) } + +// ToLocal is a client's path as Minstrel names it. +func (m PathMap) ToLocal(p string) string { return swap(p, m.Remote, m.Local) } + +// swap replaces the prefix from with to on segment boundaries +// (/downloads2 is not inside /downloads). A path outside from is unchanged. +func swap(p, from, to string) string { + if p == "" || from == "" || to == "" { + return p + } + from, to = path.Clean(from), path.Clean(to) + clean := path.Clean(p) + if clean == from { + return to + } + if from == "/" { + return path.Join(to, clean) + } + if rest, ok := strings.CutPrefix(clean, from+"/"); ok { + return path.Join(to, rest) + } + return p +} + +// Mapped wraps d so every path it is given or answers with is translated. +// File indices and in-torrent paths need no translation. +func Mapped(d Driver, m PathMap) Driver { + if !m.Active() { + return d + } + return &mapped{Driver: d, m: m} +} + +type mapped struct { + Driver + m PathMap +} + +func (c *mapped) AddTorrent(ctx context.Context, a Add) error { + a.SavePath = c.m.ToRemote(a.SavePath) + return c.Driver.AddTorrent(ctx, a) +} + +func (c *mapped) Torrents(ctx context.Context, hashes ...string) ([]Torrent, error) { + ts, err := c.Driver.Torrents(ctx, hashes...) + for i := range ts { + ts[i].SavePath = c.m.ToLocal(ts[i].SavePath) + ts[i].ContentPath = c.m.ToLocal(ts[i].ContentPath) + } + return ts, err +} + +func (c *mapped) SkipFiles(ctx context.Context, hash string, indices []int) error { + fs, ok := c.Driver.(FileSkipper) + if !ok { + return ErrUnsupported + } + return fs.SkipFiles(ctx, hash, indices) +} + +func (c *mapped) KeepFiles(ctx context.Context, hash string, indices []int) error { + fs, ok := c.Driver.(FileSkipper) + if !ok { + return ErrUnsupported + } + return fs.KeepFiles(ctx, hash, indices) +} + +func (c *mapped) Files(ctx context.Context, hash string) ([]File, error) { + fl, ok := c.Driver.(FileLister) + if !ok { + return nil, ErrUnsupported + } + return fl.Files(ctx, hash) +} + +func (c *mapped) FileStates(ctx context.Context, hash string) ([]FileState, error) { + fp, ok := c.Driver.(FileProgress) + if !ok { + return nil, ErrUnsupported + } + return fp.FileStates(ctx, hash) +} diff --git a/internal/downloader/qbittorrent.go b/internal/downloader/qbittorrent.go new file mode 100644 index 00000000..73d1eb6b --- /dev/null +++ b/internal/downloader/qbittorrent.go @@ -0,0 +1,367 @@ +package downloader + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "mime/multipart" + "net/http" + "net/http/cookiejar" + "net/url" + "strconv" + "strings" + "sync" + "time" +) + +// Client drives one qBittorrent through its WebUI API v2. +type Client struct { + Config Config + + mu sync.Mutex + // http carries the SID cookie once logged in; nil until then. + http *http.Client +} + +// AddTorrent hands qBittorrent a magnet or .torrent bytes, filed under the +// configured category with the save path already set. +func (c *Client) AddTorrent(ctx context.Context, a Add) error { + if a.Magnet == "" && len(a.Torrent) == 0 { + return fmt.Errorf("add: neither a magnet nor torrent bytes") + } + if a.SavePath == "" { + return fmt.Errorf("add: no save path, which would land the download outside the folder Minstrel imports from") + } + body := &bytes.Buffer{} + form := multipart.NewWriter(body) + fields := map[string]string{ + "savepath": a.SavePath, + "category": c.Config.Category, + // Both, because qBittorrent 5 renamed paused to stopped and an older + // one ignores the field it does not know. + "paused": strconv.FormatBool(a.Paused), + "stopped": strconv.FormatBool(a.Paused), + } + if len(a.Tags) > 0 { + fields["tags"] = strings.Join(a.Tags, ",") + } + if a.StopAtMetadata && !a.Paused { + fields["stopCondition"] = "MetadataReceived" + } + if a.Magnet != "" { + fields["urls"] = a.Magnet + } + for k, v := range fields { + if v == "" { + continue + } + if err := form.WriteField(k, v); err != nil { + return fmt.Errorf("add: build form: %w", err) + } + } + if len(a.Torrent) > 0 { + part, err := form.CreateFormFile("torrents", "release.torrent") + if err != nil { + return fmt.Errorf("add: build form: %w", err) + } + if _, err := part.Write(a.Torrent); err != nil { + return fmt.Errorf("add: build form: %w", err) + } + } + if err := form.Close(); err != nil { + return fmt.Errorf("add: build form: %w", err) + } + resp, err := c.post(ctx, "/api/v2/torrents/add", form.FormDataContentType(), body.Bytes()) + if err != nil { + return err + } + // qBittorrent refuses with HTTP 200 and the body "Fails.". + if strings.HasPrefix(strings.TrimSpace(string(resp)), "Fails") { + return fmt.Errorf("%w: the client refused the torrent", ErrUnreachable) + } + return nil +} + +// Torrents lists the configured category's torrents, or the given hashes. +func (c *Client) Torrents(ctx context.Context, hashes ...string) ([]Torrent, error) { + params := url.Values{} + if c.Config.Category != "" { + params.Set("category", c.Config.Category) + } + if len(hashes) > 0 { + params.Set("hashes", strings.Join(hashes, "|")) + } + body, err := c.get(ctx, "/api/v2/torrents/info", params) + if err != nil { + return nil, err + } + var out []Torrent + if err := json.Unmarshal(body, &out); err != nil { + return nil, fmt.Errorf("%w: parse torrent list: %v", ErrUnreachable, err) + } + for i := range out { + out[i].Hash = strings.ToLower(out[i].Hash) + } + return out, nil +} + +// Connected reads qBittorrent's own connection status. +func (c *Client) Connected(ctx context.Context) (bool, error) { + body, err := c.get(ctx, "/api/v2/transfer/info", nil) + if err != nil { + return false, err + } + var info struct { + ConnectionStatus string `json:"connection_status"` + } + if err := json.Unmarshal(body, &info); err != nil { + return false, fmt.Errorf("%w: parse transfer info: %v", ErrUnreachable, err) + } + return info.ConnectionStatus == "connected", nil +} + +// Stop and Start try qBittorrent 5's names first and fall back to 4's. +func (c *Client) Stop(ctx context.Context, hashes ...string) error { + return c.command(ctx, hashes, "/api/v2/torrents/stop", "/api/v2/torrents/pause") +} + +func (c *Client) Start(ctx context.Context, hashes ...string) error { + return c.command(ctx, hashes, "/api/v2/torrents/start", "/api/v2/torrents/resume") +} + +func (c *Client) Reannounce(ctx context.Context, hashes ...string) error { + return c.command(ctx, hashes, "/api/v2/torrents/reannounce", "/api/v2/torrents/reannounce") +} + +func (c *Client) Recheck(ctx context.Context, hashes ...string) error { + return c.command(ctx, hashes, "/api/v2/torrents/recheck", "/api/v2/torrents/recheck") +} + +func (c *Client) Delete(ctx context.Context, deleteFiles bool, hashes ...string) error { + if len(hashes) == 0 { + return nil + } + params := url.Values{ + "hashes": {strings.Join(hashes, "|")}, + "deleteFiles": {strconv.FormatBool(deleteFiles)}, + } + _, err := c.post(ctx, "/api/v2/torrents/delete", "application/x-www-form-urlencoded", []byte(params.Encode())) + return err +} + +func (c *Client) SkipFiles(ctx context.Context, hash string, indices []int) error { + return c.filePriority(ctx, hash, indices, "0") +} + +func (c *Client) KeepFiles(ctx context.Context, hash string, indices []int) error { + return c.filePriority(ctx, hash, indices, "1") +} + +func (c *Client) filePriority(ctx context.Context, hash string, indices []int, priority string) error { + if len(indices) == 0 { + return nil + } + ids := make([]string, len(indices)) + for i, n := range indices { + ids[i] = strconv.Itoa(n) + } + params := url.Values{"hash": {hash}, "id": {strings.Join(ids, "|")}, "priority": {priority}} + _, err := c.post(ctx, "/api/v2/torrents/filePrio", "application/x-www-form-urlencoded", []byte(params.Encode())) + return err +} + +func (c *Client) Files(ctx context.Context, hash string) ([]File, error) { + raw, err := c.files(ctx, hash) + if err != nil { + return nil, err + } + out := make([]File, len(raw)) + for i, f := range raw { + out[i] = File{Index: f.Index, Path: f.Name, Size: f.Size} + } + return out, nil +} + +func (c *Client) FileStates(ctx context.Context, hash string) ([]FileState, error) { + raw, err := c.files(ctx, hash) + if err != nil { + return nil, err + } + out := make([]FileState, len(raw)) + for i, f := range raw { + done := f.Size + if f.Progress < 1 { + // Progress is a float; never round an unfinished file up to whole. + done = min(int64(f.Progress*float64(f.Size)), f.Size-1) + } + out[i] = FileState{Path: f.Name, Size: f.Size, Done: max(done, 0), Skipped: f.Priority == 0} + } + return out, nil +} + +type qbitFile struct { + Index int `json:"index"` + Name string `json:"name"` + Size int64 `json:"size"` + Progress float64 `json:"progress"` + Priority int `json:"priority"` +} + +func (c *Client) files(ctx context.Context, hash string) ([]qbitFile, error) { + body, err := c.get(ctx, "/api/v2/torrents/files", url.Values{"hash": {hash}}) + if err != nil { + return nil, err + } + var raw []qbitFile + if err := json.Unmarshal(body, &raw); err != nil { + return nil, fmt.Errorf("%w: parse file list: %v", ErrUnreachable, err) + } + return raw, nil +} + +func (c *Client) command(ctx context.Context, hashes []string, primary, fallback string) error { + if len(hashes) == 0 { + return nil + } + params := []byte(url.Values{"hashes": {strings.Join(hashes, "|")}}.Encode()) + _, err := c.post(ctx, primary, "application/x-www-form-urlencoded", params) + var status statusError + if errors.As(err, &status) && (status.code == http.StatusNotFound || status.code == http.StatusMethodNotAllowed) { + _, err = c.post(ctx, fallback, "application/x-www-form-urlencoded", params) + } + return err +} + +type statusError struct { + code int + what string +} + +func (e statusError) Error() string { return fmt.Sprintf("%s: HTTP %d", e.what, e.code) } + +func (e statusError) Unwrap() error { return ErrUnreachable } + +func (c *Client) get(ctx context.Context, path string, params url.Values) ([]byte, error) { + endpoint := path + if len(params) > 0 { + endpoint += "?" + params.Encode() + } + return c.do(ctx, http.MethodGet, endpoint, "", nil) +} + +func (c *Client) post(ctx context.Context, path, contentType string, body []byte) ([]byte, error) { + return c.do(ctx, http.MethodPost, path, contentType, body) +} + +// do logs in when there is no session, and once more on a 403, which is how +// qBittorrent says a session expired. +func (c *Client) do(ctx context.Context, method, path, contentType string, body []byte) ([]byte, error) { + if c.Config.BaseURL == "" { + return nil, ErrUnconfigured + } + if err := c.ensureSession(ctx, false); err != nil { + return nil, err + } + out, err := c.call(ctx, method, path, contentType, body) + var status statusError + if errors.As(err, &status) && status.code == http.StatusForbidden { + if err := c.ensureSession(ctx, true); err != nil { + return nil, err + } + return c.call(ctx, method, path, contentType, body) + } + return out, err +} + +func (c *Client) ensureSession(ctx context.Context, force bool) error { + c.mu.Lock() + defer c.mu.Unlock() + if c.http != nil && !force { + return nil + } + jar, err := cookiejar.New(nil) + if err != nil { + return fmt.Errorf("cookie jar: %w", err) + } + httpc := &http.Client{Jar: jar} + params := url.Values{"username": {c.Config.Username}, "password": {c.Config.Password}} + out, err := c.callWith(ctx, httpc, http.MethodPost, "/api/v2/auth/login", + "application/x-www-form-urlencoded", []byte(params.Encode())) + if err != nil { + var status statusError + if errors.As(err, &status) && status.code == http.StatusForbidden { + return fmt.Errorf("%w: too many failed attempts, or this address is banned", ErrAuth) + } + return err + } + if strings.HasPrefix(strings.TrimSpace(string(out)), "Fails") { + return fmt.Errorf("%w: wrong username or password", ErrAuth) + } + c.http = httpc + return nil +} + +func (c *Client) call(ctx context.Context, method, path, contentType string, body []byte) ([]byte, error) { + c.mu.Lock() + httpc := c.http + c.mu.Unlock() + if httpc == nil { + httpc = &http.Client{} + } + return c.callWith(ctx, httpc, method, path, contentType, body) +} + +func (c *Client) callWith(ctx context.Context, httpc *http.Client, method, path, contentType string, body []byte) ([]byte, error) { + base := strings.TrimRight(c.Config.BaseURL, "/") + status, out, err := send(ctx, httpc, c.Config.Timeout, method, base+path, contentType, body, func(h http.Header) { + // qBittorrent's CSRF check refuses a request without a matching + // Referer. + h.Set("Referer", base) + }) + if err != nil { + return nil, err + } + if status < 200 || status >= 300 { + return nil, statusError{code: status, what: path} + } + return out, nil +} + +// send makes one request to a download client under its timeout, lets the +// caller add headers, and reads at most 16 MB. It returns the status for the +// caller to read, because each client's refusals are its own. +func send(ctx context.Context, httpc *http.Client, timeout time.Duration, method, target, contentType string, body []byte, set func(http.Header)) (int, []byte, error) { + if timeout <= 0 { + timeout = DefaultTimeout + } + ctx, cancel := context.WithTimeout(ctx, timeout) + defer cancel() + var reader io.Reader + if body != nil { + reader = bytes.NewReader(body) + } + req, err := http.NewRequestWithContext(ctx, method, target, reader) + if err != nil { + return 0, nil, fmt.Errorf("%w: build request: %v", ErrUnreachable, err) + } + req.Header.Set("User-Agent", UserAgent) + if contentType != "" { + req.Header.Set("Content-Type", contentType) + } + if set != nil { + set(req.Header) + } + resp, err := httpc.Do(req) + if err != nil { + return 0, nil, fmt.Errorf("%w: %v", ErrUnreachable, err) + } + defer func() { _ = resp.Body.Close() }() + out, err := io.ReadAll(io.LimitReader(resp.Body, 16<<20)) + if err != nil { + return 0, nil, fmt.Errorf("%w: read body: %v", ErrUnreachable, err) + } + return resp.StatusCode, out, nil +} diff --git a/internal/exchange/exchange.go b/internal/exchange/exchange.go new file mode 100644 index 00000000..a5417b73 --- /dev/null +++ b/internal/exchange/exchange.go @@ -0,0 +1,182 @@ +// Package exchange talks to FabledExchange's native API for the two jobs that +// are not downloading (M502 #5518): searching its DHT index (/api/search) and +// describing one torrent (/api/torrents/{infohash}). Downloading is +// internal/downloader's. Ported from Fabled Librarian's internal/exchange. +// +// The native search rather than Exchange's Torznab feed, because it reports +// whether a swarm was measured and how many files a torrent holds, which the +// feed cannot say. +package exchange + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "log/slog" + "net/http" + "strings" + "time" + + "git.fabledsword.com/bvandeusen/minstrel/internal/indexer" +) + +// ErrUnconfigured is an Exchange with no address. +var ErrUnconfigured = errors.New("exchange is not configured") + +// ErrUnreachable is a call that could not be made or was not understood. +var ErrUnreachable = errors.New("exchange unreachable") + +// ErrAuth is an API key Exchange refused, or a secured Exchange asked with +// none. +var ErrAuth = errors.New("exchange refused the API key") + +// ErrUnknown is a torrent Exchange has not crawled: a JSON 404, which is an +// answer and not a fault. +var ErrUnknown = errors.New("exchange has not crawled this torrent") + +// UserAgent names Minstrel in Exchange's logs. +const UserAgent = "Minstrel (self-hosted music server; release search)" + +// DefaultTimeout is what an unset Timeout means. Exchange answers from its +// own database. +const DefaultTimeout = 20 * time.Second + +// Config is one Exchange. +type Config struct { + BaseURL string + // APIKey is optional and sent only when set: an empty bearer makes a + // secured Exchange answer 401. + APIKey string + Timeout time.Duration +} + +// Client asks one Exchange. +type Client struct { + Config Config + // HTTP is injectable for tests. Nil takes http.DefaultClient. + HTTP *http.Client + // Code is the acquisition_services row, stamped onto search results. + Code string + Logger *slog.Logger +} + +func (c *Client) log() *slog.Logger { + if c == nil || c.Logger == nil { + return slog.Default() + } + return c.Logger +} + +// Detail is what Exchange knows about one torrent. +type Detail struct { + InfoHash string `json:"infohash"` + Name string `json:"name"` + Size int64 `json:"size"` + FilesCount int `json:"files_count"` + Seeders int `json:"seeders"` + SwarmKnown bool `json:"swarm_known"` + Leechers int `json:"leechers"` + LeechersKnown bool `json:"leechers_known"` + ContentType string `json:"content_type"` + Files []DetailFile `json:"files"` +} + +// DetailFile is one file inside a torrent. +type DetailFile struct { + Index int `json:"index"` + Path string `json:"path"` + Size int64 `json:"size"` +} + +// Torrent asks Exchange about one v1 infohash. +func (c *Client) Torrent(ctx context.Context, infohash string) (Detail, error) { + if strings.TrimSpace(c.Config.BaseURL) == "" { + return Detail{}, ErrUnconfigured + } + hash := indexer.NormalizeHash(infohash) + if len(hash) != 40 || !indexer.IsInfoHash(hash) { + return Detail{}, fmt.Errorf("%w: %q is not a 40-character infohash", ErrUnreachable, infohash) + } + body, err := c.get(ctx, "/api/torrents/"+hash) + if err != nil { + return Detail{}, err + } + var out Detail + if err := json.Unmarshal(body, &out); err != nil { + return Detail{}, fmt.Errorf("%w: parse torrent detail: %v", ErrUnreachable, err) + } + if out.InfoHash == "" { + return Detail{}, ErrUnknown + } + return out, nil +} + +func (c *Client) get(ctx context.Context, path string) ([]byte, error) { + timeout := c.Config.Timeout + if timeout <= 0 { + timeout = DefaultTimeout + } + ctx, cancel := context.WithTimeout(ctx, timeout) + defer cancel() + + target := strings.TrimRight(c.Config.BaseURL, "/") + path + req, err := http.NewRequestWithContext(ctx, http.MethodGet, target, nil) + if err != nil { + return nil, fmt.Errorf("%w: %v", ErrUnreachable, err) + } + req.Header.Set("User-Agent", UserAgent) + req.Header.Set("Accept", "application/json") + if key := strings.TrimSpace(c.Config.APIKey); key != "" { + req.Header.Set("Authorization", "Bearer "+key) + } + client := c.HTTP + if client == nil { + client = http.DefaultClient + } + res, err := client.Do(req) + if err != nil { + return nil, fmt.Errorf("%w: %v", ErrUnreachable, err) + } + defer func() { _ = res.Body.Close() }() + body, err := io.ReadAll(io.LimitReader(res.Body, 1<<20)) + if err != nil { + return nil, fmt.Errorf("%w: read answer: %v", ErrUnreachable, err) + } + switch { + case res.StatusCode == http.StatusNotFound: + // A 404 that is not JSON came from something else at this address, + // which must not look like a healthy Exchange that lacks a torrent. + if !looksLikeJSON(body) { + return nil, fmt.Errorf("%w: something at this address answered 404, "+ + "but it was not FabledExchange", ErrUnreachable) + } + return nil, ErrUnknown + case res.StatusCode == http.StatusUnauthorized || res.StatusCode == http.StatusForbidden: + return nil, fmt.Errorf("%w: %s", ErrAuth, + message(body, "set this FabledExchange's API key in Minstrel's settings")) + case res.StatusCode == http.StatusServiceUnavailable: + return nil, fmt.Errorf("%w: %s", ErrUnreachable, message(body, "no index is configured")) + case res.StatusCode < 200 || res.StatusCode >= 300: + return nil, fmt.Errorf("%w: %s", ErrUnreachable, + message(body, fmt.Sprintf("returned %d", res.StatusCode))) + } + return body, nil +} + +// message is Exchange's own {"error": "..."} text, or fallback. +func message(body []byte, fallback string) string { + var env struct { + Error string `json:"error"` + } + if err := json.Unmarshal(body, &env); err == nil && strings.TrimSpace(env.Error) != "" { + return strings.TrimSpace(env.Error) + } + return fallback +} + +func looksLikeJSON(body []byte) bool { + t := strings.TrimSpace(string(body)) + return strings.HasPrefix(t, "{") || strings.HasPrefix(t, "[") +} diff --git a/internal/exchange/exchange_test.go b/internal/exchange/exchange_test.go new file mode 100644 index 00000000..bfd5aa92 --- /dev/null +++ b/internal/exchange/exchange_test.go @@ -0,0 +1,108 @@ +package exchange + +import ( + "context" + "errors" + "fmt" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "git.fabledsword.com/bvandeusen/minstrel/internal/indexer" +) + +func TestTorrent_SendsTheKeyOnlyWhenSet(t *testing.T) { + auth := make(chan string, 2) + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + auth <- r.Header.Get("Authorization") + _, _ = w.Write([]byte(`{"infohash": "` + strings.Repeat("a", 40) + `", "files_count": 12}`)) + })) + defer srv.Close() + + for _, key := range []string{"", "k"} { + c := &Client{Config: Config{BaseURL: srv.URL, APIKey: key}} + d, err := c.Torrent(context.Background(), strings.Repeat("A", 40)) + if err != nil || d.FilesCount != 12 { + t.Fatalf("key %q: detail %+v err %v", key, d, err) + } + } + if without, with := <-auth, <-auth; without != "" || with != "Bearer k" { + t.Errorf("Authorization = %q then %q, want none without a key and a bearer with one", without, with) + } +} + +func TestTorrent_TellsAnUncrawledTorrentFromAnotherService(t *testing.T) { + for name, tc := range map[string]struct { + status int + body string + want error + }{ + "Exchange has not crawled it": {http.StatusNotFound, `{"error": "unknown torrent"}`, ErrUnknown}, + "not FabledExchange": {http.StatusNotFound, `404`, ErrUnreachable}, + "a refused key": {http.StatusUnauthorized, `{"error": "bad key"}`, ErrAuth}, + } { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(tc.status) + _, _ = w.Write([]byte(tc.body)) + })) + _, err := (&Client{Config: Config{BaseURL: srv.URL}}).Torrent(context.Background(), strings.Repeat("0", 40)) + srv.Close() + if !errors.Is(err, tc.want) { + t.Errorf("%s: err = %v, want %v", name, err, tc.want) + } + } +} + +func TestSearch_PagesAndKeepsMeasuredSwarmsOnly(t *testing.T) { + queries := make(chan string, 4) + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + queries <- r.URL.RawQuery + offset := r.URL.Query().Get("offset") + var rows []string + if offset == "0" { + for i := 0; i < SearchLimit; i++ { + rows = append(rows, fmt.Sprintf(`{"infohash": "%040x", "name": "r%d", "content_type": "music", "seeders": 5, "swarm_known": %t}`, i, i, i == 0)) + } + } else { + // A repeat of page one's first hash, which must not be listed twice. + rows = append(rows, fmt.Sprintf(`{"infohash": "%040x", "name": "again"}`, 0)) + } + _, _ = fmt.Fprintf(w, `{"total": %d, "results": [%s]}`, SearchLimit+1, strings.Join(rows, ",")) + })) + defer srv.Close() + + c := &Client{Code: "exchange", Config: Config{BaseURL: srv.URL}} + results, err := c.Search(context.Background(), indexer.Query{Text: "x", Categories: indexer.MusicCategories}) + if err != nil { + t.Fatalf("search: %v", err) + } + close(queries) + var asked []string + for q := range queries { + asked = append(asked, q) + } + if len(asked) != 2 || !strings.Contains(asked[0], "type=music") { + t.Errorf("queries = %v, want two pages asking for music", asked) + } + if len(results) != SearchLimit { + t.Errorf("results = %d, want %d (the repeat dropped)", len(results), SearchLimit) + } + if r := results[0]; !r.SwarmKnown || r.Seeders != 5 || r.Category != 3000 || r.Source != "exchange" || !strings.HasPrefix(r.Magnet, "magnet:") { + t.Errorf("first = %+v", r) + } + if r := results[1]; r.SwarmKnown || r.Seeders != 0 { + t.Errorf("an unmeasured swarm reported seeders: %+v", r) + } +} + +func TestSearch_ARefusedKeyReadsAsAuth(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusForbidden) + })) + defer srv.Close() + _, err := (&Client{Config: Config{BaseURL: srv.URL}}).Search(context.Background(), indexer.Query{Text: "x"}) + if !errors.Is(err, indexer.ErrAuth) { + t.Fatalf("err = %v, want indexer.ErrAuth", err) + } +} diff --git a/internal/exchange/search.go b/internal/exchange/search.go new file mode 100644 index 00000000..d2fc8724 --- /dev/null +++ b/internal/exchange/search.go @@ -0,0 +1,169 @@ +package exchange + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "net/url" + "strings" + + "git.fabledsword.com/bvandeusen/minstrel/internal/indexer" +) + +// SearchLimit is one page of /api/search; SearchDepth is the most read across +// pages. A query matching more than SearchDepth is too broad to judge anyway. +const ( + SearchLimit = 100 + SearchDepth = 300 +) + +type searchResult struct { + InfoHash string `json:"infohash"` + Name string `json:"name"` + Size int64 `json:"size"` + FilesCount int `json:"files_count"` + ContentType string `json:"content_type"` + Seeders int `json:"seeders"` + SwarmKnown bool `json:"swarm_known"` + Leechers int `json:"leechers"` + LeechersKnown bool `json:"leechers_known"` +} + +type searchPage struct { + Total int `json:"total"` + Results []searchResult `json:"results"` +} + +// Search asks Exchange's DHT index. Every result is a magnet, so there is no +// .torrent to fetch through it. A later page that fails keeps the earlier +// ones; a first page that fails is the search failing. +func (c *Client) Search(ctx context.Context, q indexer.Query) ([]indexer.Result, error) { + if strings.TrimSpace(c.Config.BaseURL) == "" { + return nil, indexer.ErrUnconfigured + } + params := url.Values{"q": {q.Text}, "limit": {fmt.Sprint(SearchLimit)}} + if t := contentType(q.Categories); t != "" { + params.Set("type", t) + } + var ( + out []indexer.Result + seen = map[string]bool{} + ) + for offset := 0; offset < SearchDepth; offset += SearchLimit { + params.Set("offset", fmt.Sprint(offset)) + page, err := c.searchPage(ctx, params) + if err != nil { + if offset == 0 { + return nil, err + } + c.log().Warn("exchange: a later page of a search could not be read; keeping the pages before it", + "q", q.Text, "offset", offset, "err", err) + break + } + for _, r := range page.Results { + res, ok := c.result(r) + if !ok || seen[res.InfoHash] { + continue + } + seen[res.InfoHash] = true + out = append(out, res) + } + if len(page.Results) < SearchLimit || offset+SearchLimit >= page.Total { + break + } + } + return out, nil +} + +func (c *Client) searchPage(ctx context.Context, params url.Values) (searchPage, error) { + var page searchPage + body, err := c.get(ctx, "/api/search?"+params.Encode()) + if err != nil { + return page, searchError(err) + } + if err := json.Unmarshal(body, &page); err != nil { + return page, fmt.Errorf("%w: parse search results: %v", indexer.ErrUnreachable, err) + } + return page, nil +} + +func (c *Client) result(r searchResult) (indexer.Result, bool) { + hash := indexer.NormalizeHash(r.InfoHash) + if hash == "" || strings.TrimSpace(r.Name) == "" { + return indexer.Result{}, false + } + res := indexer.Result{ + Title: r.Name, + InfoHash: hash, + Magnet: magnet(hash, r.Name), + SizeBytes: r.Size, + Category: category(r.ContentType), + Source: c.Code, + FilesCount: r.FilesCount, + SwarmKnown: r.SwarmKnown, + } + // Seeders only when measured: an unmeasured count is not health. + if r.SwarmKnown { + res.Seeders, res.Peers = r.Seeders, r.Seeders + if r.LeechersKnown { + res.Peers = r.Seeders + r.Leechers + } + } + return res, true +} + +// searchError puts Exchange's errors in the vocabulary every search source +// shares, so a refused key reads the same whichever source refused it. +func searchError(err error) error { + switch { + case errors.Is(err, ErrAuth): + return fmt.Errorf("%w: %v", indexer.ErrAuth, err) + case errors.Is(err, ErrUnknown): + return fmt.Errorf("%w: this FabledExchange has no search API", indexer.ErrUnreachable) + } + return fmt.Errorf("%w: %v", indexer.ErrUnreachable, err) +} + +// contentType narrows Exchange's search to music when the query asks only +// for music categories (3000s, not 3030 audiobooks). +func contentType(cats []int) string { + music := false + for _, c := range cats { + if c < 3000 || c >= 4000 || c == 3030 { + return "" + } + music = true + } + if music { + return "music" + } + return "" +} + +func category(contentType string) int { + switch contentType { + case "music": + return 3000 + case "audiobook": + return 3030 + case "ebook": + return 7020 + case "comic": + return 7030 + case "movie": + return 2000 + case "tv": + return 5000 + } + return 8000 +} + +// magnet is a trackerless magnet for a hash Exchange found on the DHT. +func magnet(hash, name string) string { + m := "magnet:?xt=urn:btih:" + hash + if name != "" { + m += "&dn=" + url.QueryEscape(name) + } + return m +} diff --git a/internal/indexer/indexer.go b/internal/indexer/indexer.go new file mode 100644 index 00000000..df554a94 --- /dev/null +++ b/internal/indexer/indexer.go @@ -0,0 +1,157 @@ +// Package indexer is what every search source answers with, and the Torznab +// client (M502 #5518). Ported from Fabled Librarian's internal/indexer. +// +// A client here holds ONE endpoint and knows nothing about the others. The +// fan-out across sources, the merge and what a source being down means belong +// to internal/acquire, because they are decisions about a search rather than +// about one source. +// +// Prowlarr is not served from here: it has no aggregate Torznab feed (`all` +// 404s like any id it does not have, measured by Fabled Librarian, #4304), so +// asking it everything at once means its native API, which internal/prowlarr +// speaks. +package indexer + +import ( + "errors" + "fmt" + "io" + "net/http" + "strings" + "time" +) + +// ErrUnconfigured is a source with no address (or, for Prowlarr, no key): not +// a failure, an instance that does not search there. +var ErrUnconfigured = errors.New("indexer is not configured") + +// ErrUnreachable is a search that could not be made: no network, a timeout, a +// malformed answer, a status outside 2xx. +var ErrUnreachable = errors.New("indexer unreachable") + +// ErrAuth is a key the source refused. Separate from ErrUnreachable because +// it needs a different sentence and a different action: a refused key is a +// field in Settings, an outage is something to wait out. +var ErrAuth = errors.New("the API key was refused") + +// UserAgent names Minstrel in the operator's own Prowlarr and indexer logs. +const UserAgent = "Minstrel (self-hosted music server; release search)" + +// MusicCategories are the Newznab categories a music search asks for: 3000 +// Audio, with its MP3 (3010) and Lossless (3040) children alongside, because +// an indexer that files music under a child answers only for the child. +// 3030 (Audiobook) is deliberately absent. +var MusicCategories = []int{3000, 3010, 3040} + +// Query is one search: the words, and which categories to ask in. Empty +// Categories asks every category. +type Query struct { + Text string + Categories []int +} + +// Result is one search result, as the source described it. +type Result struct { + Title string + // InfoHash is the torrent's identity, lowercased; empty when the source + // gave none. + InfoHash string + // Magnet is a magnet URI when the source published one. + Magnet string + // DownloadURL is a .torrent URL when it did not. It must be fetched + // through the source that offered it (it holds the tracker credentials), + // never handed to the download client. + DownloadURL string + SizeBytes int64 + Category int + // Indexer is the tracker BEHIND the source, as the source named it. + Indexer string + // Source is the acquisition_services code that was asked: where a + // .torrent fetch must be routed back to. + Source string + Published time.Time + // Seeders and Peers as reported. A feed that has not sampled a swarm + // reports a nominal 1, so these are not health unless SwarmKnown. + Seeders int + Peers int + // SwarmKnown says Seeders was measured, which only FabledExchange's own + // search can say. + SwarmKnown bool + // FilesCount is how many files the torrent holds, 0 when not said. + FilesCount int +} + +// Send sends req through c, stopping at a redirect to a magnet (returned as a +// *MagnetLink), and reads at most 8 MB of the answer. A call that never got +// an answer is ErrUnreachable; the status is the caller's to read. +func Send(c *http.Client, req *http.Request) (*http.Response, []byte, error) { + resp, err := StopAtMagnet(c).Do(req) + if m, ok := AsMagnet(err); ok { + return nil, nil, &MagnetLink{URI: m} + } + if err != nil { + return nil, nil, fmt.Errorf("%w: %v", ErrUnreachable, err) + } + defer func() { _ = resp.Body.Close() }() + body, err := io.ReadAll(io.LimitReader(resp.Body, 8<<20)) + if err != nil { + return nil, nil, fmt.Errorf("%w: read body: %v", ErrUnreachable, err) + } + return resp, body, nil +} + +// StatusError is what a non-2xx status means for a search source: a 429 is +// RateLimited naming who, anything else ErrUnreachable. Nil for a 2xx. +func StatusError(resp *http.Response, who string) error { + switch { + case resp.StatusCode == http.StatusTooManyRequests: + return &RateLimited{Who: who, RetryAfter: RetryAfter(resp.Header, time.Now())} + case resp.StatusCode < 200 || resp.StatusCode >= 300: + return fmt.Errorf("%w: HTTP %d", ErrUnreachable, resp.StatusCode) + } + return nil +} + +// NormalizeHash is a torrent's identity in its one canonical spelling. Every +// comparison of infohashes goes through it, so a hash differing only in case +// is never two releases. +func NormalizeHash(s string) string { + return strings.ToLower(strings.TrimSpace(s)) +} + +// IsInfoHash says whether s is shaped like a v1 (40) or v2 (64) hex infohash. +func IsInfoHash(s string) bool { + s = NormalizeHash(s) + if len(s) != 40 && len(s) != 64 { + return false + } + for _, r := range s { + if (r < '0' || r > '9') && (r < 'a' || r > 'f') { + return false + } + } + return true +} + +// MagnetHash pulls the infohash out of a magnet link, or returns "". It does +// not validate it. +func MagnetHash(magnet string) string { + const prefix = "urn:btih:" + at := strings.Index(magnet, prefix) + if at < 0 { + return "" + } + hash := magnet[at+len(prefix):] + if cut := strings.IndexAny(hash, "&/?"); cut >= 0 { + hash = hash[:cut] + } + return strings.ToLower(hash) +} + +// FirstLine is an error's first line, for a sentence on the settings page. +func FirstLine(s string) string { + if i := strings.IndexByte(s, '\n'); i >= 0 { + return s[:i] + } + return s +} diff --git a/internal/indexer/magnet.go b/internal/indexer/magnet.go new file mode 100644 index 00000000..123e61c2 --- /dev/null +++ b/internal/indexer/magnet.go @@ -0,0 +1,58 @@ +package indexer + +import ( + "bytes" + "errors" + "net/http" +) + +// MagnetLink is a download link that turned out to be a magnet. Returned as +// an error by a .torrent fetch so the grab hands the magnet to the download +// client instead. Public trackers do this routinely: Prowlarr's download link +// answers with a redirect to `magnet:?xt=…`, which Go's client would otherwise +// fail on as an unsupported scheme. +type MagnetLink struct{ URI string } + +func (m *MagnetLink) Error() string { return "the download link is a magnet link" } + +// AsMagnet says whether err is a magnet link, and which. +func AsMagnet(err error) (string, bool) { + var m *MagnetLink + if errors.As(err, &m) && m.URI != "" { + return m.URI, true + } + return "", false +} + +// StopAtMagnet returns a copy of c that stops at a redirect to a magnet URI +// and reports it as a *MagnetLink. Every other redirect behaves as c would. +func StopAtMagnet(c *http.Client) *http.Client { + if c == nil { + c = http.DefaultClient + } + out := *c + next := c.CheckRedirect + out.CheckRedirect = func(req *http.Request, via []*http.Request) error { + if req.URL != nil && req.URL.Scheme == "magnet" { + return &MagnetLink{URI: req.URL.String()} + } + if next != nil { + return next(req, via) + } + if len(via) >= 10 { + return errors.New("stopped after 10 redirects") + } + return nil + } + return &out +} + +// MagnetBody is a response whose whole body is a magnet URI, which some +// indexers send with a 200 instead of a redirect. Nil otherwise. +func MagnetBody(body []byte) error { + trimmed := bytes.TrimSpace(body) + if bytes.HasPrefix(trimmed, []byte("magnet:")) && !bytes.ContainsAny(trimmed, " \n<") { + return &MagnetLink{URI: string(trimmed)} + } + return nil +} diff --git a/internal/indexer/ratelimit.go b/internal/indexer/ratelimit.go new file mode 100644 index 00000000..5be3ca9f --- /dev/null +++ b/internal/indexer/ratelimit.go @@ -0,0 +1,50 @@ +package indexer + +import ( + "errors" + "fmt" + "net/http" + "strconv" + "time" +) + +// RateLimited is a source answering 429. Named so the search pacing can hold +// that source back for as long as it asked; still an ErrUnreachable to +// everything else, because this search did not reach it. +type RateLimited struct { + // RetryAfter is what the source asked for; zero when it did not say. + RetryAfter time.Duration + // Who names the source in the sentence, e.g. "Prowlarr". + Who string +} + +func (e *RateLimited) Error() string { + return fmt.Sprintf("%v: HTTP 429, %s is rate-limiting this search", ErrUnreachable, e.Who) +} + +func (e *RateLimited) Unwrap() error { return ErrUnreachable } + +// AsRateLimited finds a RateLimited anywhere in err's chain. +func AsRateLimited(err error) (*RateLimited, bool) { + var rl *RateLimited + if errors.As(err, &rl) { + return rl, true + } + return nil, false +} + +// RetryAfter reads a Retry-After header: seconds, or an HTTP date. Zero when +// absent or unreadable. +func RetryAfter(h http.Header, now time.Time) time.Duration { + v := h.Get("Retry-After") + if v == "" { + return 0 + } + if s, err := strconv.Atoi(v); err == nil && s > 0 { + return time.Duration(s) * time.Second + } + if t, err := http.ParseTime(v); err == nil && t.After(now) { + return t.Sub(now) + } + return 0 +} diff --git a/internal/indexer/torznab.go b/internal/indexer/torznab.go new file mode 100644 index 00000000..03c9a5f4 --- /dev/null +++ b/internal/indexer/torznab.go @@ -0,0 +1,254 @@ +package indexer + +import ( + "context" + "encoding/xml" + "fmt" + "net/http" + "net/url" + "strconv" + "strings" + "time" +) + +// DefaultTimeout is what an unset Timeout means. Longer than a metadata +// call: a feed may itself fan out to several trackers before answering. +const DefaultTimeout = 30 * time.Second + +// TorznabConfig is one Torznab feed. +type TorznabConfig struct { + // Code is the acquisition_services row this client was built from, + // stamped onto every Result. + Code string + // FeedURL is the feed's whole address up to and including /api, as + // Jackett or a Prowlarr indexer shows it — e.g. + // http://prowlarr:9696/12/api. Query parameters are appended to it. + // One field rather than Fabled Librarian's base + path + indexer id: a + // pasted URL is more reliable than Minstrel being right about a third + // party's routes. + FeedURL string + // APIKey is optional: Torznab makes it so. A feed that needs one says so + // with Newznab error 100, which is ErrAuth. + APIKey string + Timeout time.Duration +} + +// Torznab searches one Torznab feed. +type Torznab struct { + Config TorznabConfig + // HTTP is injectable for tests. Nil takes http.DefaultClient. + HTTP *http.Client +} + +// Search asks the feed for q. `t=search` is free text, which every indexer +// supports; `t=music` takes artist and album separately, and a release name +// does not reliably say which of its parts is which. +func (c *Torznab) Search(ctx context.Context, q Query) ([]Result, error) { + params := url.Values{"t": {"search"}, "q": {q.Text}} + for _, cat := range q.Categories { + params.Add("cat", strconv.Itoa(cat)) + } + body, err := c.get(ctx, c.endpoint(params)) + if err != nil { + return nil, err + } + out, err := parseFeed(body) + if err != nil { + return nil, err + } + for i := range out { + out[i].Source = c.Config.Code + } + return out, nil +} + +// Download fetches a result's .torrent bytes through the feed, which holds +// the tracker's credentials. A result with a magnet needs none of this. +func (c *Torznab) Download(ctx context.Context, r Result) ([]byte, error) { + if r.DownloadURL == "" { + return nil, fmt.Errorf("%w: %q has no download URL", ErrUnreachable, r.Title) + } + body, err := c.get(ctx, r.DownloadURL) + if err != nil { + return nil, err + } + return torrentBody(r.Title, body) +} + +// torrentBody checks a fetched .torrent. A torrent is a bencoded dictionary +// and starts with 'd'; an expired session gets an HTML login page with HTTP +// 200, which the download client would otherwise reject as an unsupported +// file from a component that did nothing wrong. +func torrentBody(title string, body []byte) ([]byte, error) { + if err := MagnetBody(body); err != nil { + return nil, err + } + if len(body) == 0 || body[0] != 'd' { + return nil, fmt.Errorf("%w: %q returned %d bytes that are not a torrent", ErrUnreachable, title, len(body)) + } + return body, nil +} + +// TorrentBody is torrentBody for the other search clients' fetches. +func TorrentBody(title string, body []byte) ([]byte, error) { return torrentBody(title, body) } + +func (c *Torznab) endpoint(params url.Values) string { + if c.Config.APIKey != "" { + params.Set("apikey", c.Config.APIKey) + } + feed := strings.TrimSpace(c.Config.FeedURL) + sep := "?" + if strings.Contains(feed, "?") { + sep = "&" + } + return feed + sep + params.Encode() +} + +func (c *Torznab) get(ctx context.Context, endpoint string) ([]byte, error) { + if strings.TrimSpace(c.Config.FeedURL) == "" { + return nil, ErrUnconfigured + } + timeout := c.Config.Timeout + if timeout <= 0 { + timeout = DefaultTimeout + } + ctx, cancel := context.WithTimeout(ctx, timeout) + defer cancel() + req, err := http.NewRequestWithContext(ctx, http.MethodGet, endpoint, nil) + if err != nil { + return nil, fmt.Errorf("%w: build request: %v", ErrUnreachable, err) + } + req.Header.Set("User-Agent", UserAgent) + resp, body, err := Send(c.HTTP, req) + if err != nil { + return nil, err + } + if err := StatusError(resp, "the indexer"); err != nil { + return nil, err + } + // Newznab reports errors as HTTP 200 with an document, so a + // client checking only the status reads a refused key as an empty result + // set: search quietly stops finding things. + if err := errorDocument(body); err != nil { + return nil, err + } + return body, nil +} + +// errorDocument reads an answer. Newznab's 1xx +// codes (100 wrong key, 101 suspended, 102 insufficient privileges) are all +// about who is asking, so they are ErrAuth. +func errorDocument(body []byte) error { + head := body + if len(head) > 512 { + head = head[:512] + } + if !strings.Contains(string(head), " + // and repeats it in , either populated by version. + for _, candidate := range []string{it.Link, it.Enclosure.URL} { + if strings.HasPrefix(candidate, "magnet:") && r.Magnet == "" { + r.Magnet = candidate + } else if candidate != "" && r.DownloadURL == "" { + r.DownloadURL = candidate + } + } + // The infohash from the explicit attr, then the magnet, then the GUID + // only when it is shaped like a hash: Prowlarr puts a URL there as + // often as a hash. + r.InfoHash = NormalizeHash(it.attr("infohash")) + if r.InfoHash == "" { + r.InfoHash = MagnetHash(r.Magnet) + } + if r.InfoHash == "" && IsInfoHash(it.GUID) { + r.InfoHash = NormalizeHash(it.GUID) + } + if t, err := time.Parse(time.RFC1123Z, strings.TrimSpace(it.PubDate)); err == nil { + r.Published = t + } else if t, err := time.Parse(time.RFC1123, strings.TrimSpace(it.PubDate)); err == nil { + r.Published = t + } + out = append(out, r) + } + return out, nil +} + +func atoi(s string) int { + n, err := strconv.Atoi(strings.TrimSpace(s)) + if err != nil { + return 0 + } + return n +} diff --git a/internal/indexer/torznab_test.go b/internal/indexer/torznab_test.go new file mode 100644 index 00000000..2855f0af --- /dev/null +++ b/internal/indexer/torznab_test.go @@ -0,0 +1,188 @@ +package indexer + +import ( + "context" + "errors" + "net/http" + "net/http/httptest" + "testing" + "time" +) + +const sampleFeed = ` + + + + Radiohead - OK Computer (1997) [FLAC] + https://tracker.example/details/1 + https://prowlarr.example/1/download?link=abc + 512000000 + Tue, 07 Oct 2026 10:00:00 +0000 + + + + + + + + Radiohead - Kid A (2000) [MP3 320] + 0123456789abcdef0123456789abcdef01234567 + magnet:?xt=urn:btih:FFEEDDCCBBAA99887766554433221100FFEEDDCC&dn=kid+a + + + + + +` + +func TestTorznabSearch_ParsesTheFeedAndStampsTheSource(t *testing.T) { + reqs := make(chan *http.Request, 1) + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + reqs <- r + _, _ = w.Write([]byte(sampleFeed)) + })) + defer srv.Close() + + c := &Torznab{Config: TorznabConfig{Code: "torznab-2", FeedURL: srv.URL + "/12/api", APIKey: "k"}} + results, err := c.Search(context.Background(), Query{Text: "radiohead", Categories: MusicCategories}) + if err != nil { + t.Fatalf("search: %v", err) + } + got := <-reqs + if got.URL.Path != "/12/api" || got.URL.Query().Get("t") != "search" || got.URL.Query().Get("apikey") != "k" { + t.Errorf("request = %s, want the feed URL with t=search and the key", got.URL) + } + if cats := got.URL.Query()["cat"]; len(cats) != 3 || cats[0] != "3000" { + t.Errorf("categories = %v, want the music categories", cats) + } + if len(results) != 2 { + t.Fatalf("results = %d, want 2 (the untitled item dropped)", len(results)) + } + first := results[0] + if first.InfoHash != "aabbccddeeff00112233445566778899aabbccdd" || first.Seeders != 12 || + first.Category != 3040 || first.Indexer != "RED" || first.DownloadURL == "" || first.Source != "torznab-2" { + t.Errorf("first = %+v", first) + } + if first.Published.IsZero() { + t.Error("pubDate was not parsed") + } + // The magnet's hash wins over a GUID shaped like another hash. + if results[1].Magnet == "" || results[1].InfoHash != "ffeeddccbbaa99887766554433221100ffeeddcc" { + t.Errorf("second = %+v, want the magnet's hash", results[1]) + } +} + +func TestTorznabSearch_KeepsAQueryAlreadyInTheFeedURL(t *testing.T) { + reqs := make(chan *http.Request, 1) + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + reqs <- r + _, _ = w.Write([]byte(``)) + })) + defer srv.Close() + + c := &Torznab{Config: TorznabConfig{FeedURL: srv.URL + "/api?passkey=p"}} + if _, err := c.Search(context.Background(), Query{Text: "x"}); err != nil { + t.Fatalf("search: %v", err) + } + got := <-reqs + if got.URL.Query().Get("passkey") != "p" || got.URL.Query().Get("q") != "x" { + t.Errorf("request = %s, want the pasted query kept and ours added", got.URL) + } +} + +// Newznab reports a refused key as HTTP 200 with an document. Read +// as a feed, that is an empty result set: search would quietly find nothing. +func TestTorznabSearch_ReadsAnErrorDocument(t *testing.T) { + for name, tc := range map[string]struct { + doc string + want error + }{ + "a wrong key": {``, ErrAuth}, + "anything else": {``, ErrUnreachable}, + } { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + _, _ = w.Write([]byte(`` + tc.doc)) + })) + c := &Torznab{Config: TorznabConfig{FeedURL: srv.URL + "/api"}} + _, err := c.Search(context.Background(), Query{Text: "x"}) + srv.Close() + if !errors.Is(err, tc.want) { + t.Errorf("%s: err = %v, want %v", name, err, tc.want) + } + } +} + +func TestTorznabSearch_A429IsRateLimitedWithItsRetryAfter(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Retry-After", "90") + w.WriteHeader(http.StatusTooManyRequests) + })) + defer srv.Close() + + c := &Torznab{Config: TorznabConfig{FeedURL: srv.URL + "/api"}} + _, err := c.Search(context.Background(), Query{Text: "x"}) + rl, ok := AsRateLimited(err) + if !ok || rl.RetryAfter != 90*time.Second { + t.Fatalf("err = %v, want RateLimited for 90s", err) + } + if !errors.Is(err, ErrUnreachable) { + t.Error("a 429 should still read as unreachable to everything else") + } +} + +func TestTorznabSearch_NoAddressIsUnconfigured(t *testing.T) { + _, err := (&Torznab{}).Search(context.Background(), Query{Text: "x"}) + if !errors.Is(err, ErrUnconfigured) { + t.Fatalf("err = %v, want ErrUnconfigured", err) + } +} + +func TestTorznabDownload_RefusesWhatIsNotATorrent(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/login": + _, _ = w.Write([]byte(`please log in`)) + case "/magnet": + _, _ = w.Write([]byte("magnet:?xt=urn:btih:abc")) + case "/redirect": + http.Redirect(w, r, "magnet:?xt=urn:btih:def", http.StatusFound) + default: + _, _ = w.Write([]byte("d8:announce0:e")) + } + })) + defer srv.Close() + + c := &Torznab{Config: TorznabConfig{FeedURL: srv.URL + "/api"}} + ctx := context.Background() + if _, err := c.Download(ctx, Result{Title: "t", DownloadURL: srv.URL + "/login"}); !errors.Is(err, ErrUnreachable) { + t.Errorf("an HTML page: err = %v, want ErrUnreachable", err) + } + for _, path := range []string{"/magnet", "/redirect"} { + _, err := c.Download(ctx, Result{Title: "t", DownloadURL: srv.URL + path}) + if _, ok := AsMagnet(err); !ok { + t.Errorf("%s: err = %v, want a MagnetLink", path, err) + } + } + body, err := c.Download(ctx, Result{Title: "t", DownloadURL: srv.URL + "/file"}) + if err != nil || string(body) != "d8:announce0:e" { + t.Errorf("a torrent: body %q err %v", body, err) + } +} + +func TestRetryAfter(t *testing.T) { + now := time.Date(2026, 10, 10, 0, 0, 0, 0, time.UTC) + for v, want := range map[string]time.Duration{ + "": 0, + "30": 30 * time.Second, + "soon": 0, + "Sat, 10 Oct 2026 00:02:00 GMT": 2 * time.Minute, + } { + h := http.Header{} + if v != "" { + h.Set("Retry-After", v) + } + if got := RetryAfter(h, now); got != want { + t.Errorf("RetryAfter(%q) = %v, want %v", v, got, want) + } + } +} diff --git a/internal/prowlarr/prowlarr.go b/internal/prowlarr/prowlarr.go new file mode 100644 index 00000000..17b3e639 --- /dev/null +++ b/internal/prowlarr/prowlarr.go @@ -0,0 +1,177 @@ +// Package prowlarr searches through Prowlarr's native API (M502 #5518). +// Ported from Fabled Librarian's internal/prowlarr. +// +// Native rather than Torznab because Prowlarr has no aggregate Torznab feed: +// `/api/v1/indexer/all/newznab` 404s exactly like an indexer id that does not +// exist (measured by Fabled Librarian against a real Prowlarr, its #4304). +// `/api/v1/search` is the one place Prowlarr asks every indexer at once. +package prowlarr + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "net/url" + "strings" + "time" + + "git.fabledsword.com/bvandeusen/minstrel/internal/indexer" +) + +// SearchPath is Prowlarr's aggregate search. +const SearchPath = "/api/v1/search" + +// DefaultTimeout is longer than a Torznab feed's: Prowlarr waits on every +// indexer it has before answering. +const DefaultTimeout = 60 * time.Second + +// Config is one Prowlarr. +type Config struct { + // Code is the acquisition_services row, stamped onto every Result. + Code string + BaseURL string + // APIKey is required: Prowlarr's API refuses every call without one. + APIKey string + Timeout time.Duration +} + +// Client searches one Prowlarr and fetches .torrent files through it. +type Client struct { + Config Config + // HTTP is injectable for tests. Nil takes http.DefaultClient. + HTTP *http.Client +} + +type searchResult struct { + Title string `json:"title"` + InfoHash string `json:"infoHash"` + GUID string `json:"guid"` + MagnetURL string `json:"magnetUrl"` + DownloadURL string `json:"downloadUrl"` + Size int64 `json:"size"` + Seeders int `json:"seeders"` + Leechers int `json:"leechers"` + Indexer string `json:"indexer"` + PublishDate string `json:"publishDate"` + Categories []struct { + ID int `json:"id"` + } `json:"categories"` +} + +// Search asks every indexer in this Prowlarr. `type=search` is free text; +// `type=music` takes artist and album separately, and a release name does not +// reliably say which part is which. +func (c *Client) Search(ctx context.Context, q indexer.Query) ([]indexer.Result, error) { + params := url.Values{"query": {q.Text}, "type": {"search"}} + for _, cat := range q.Categories { + params.Add("categories", fmt.Sprint(cat)) + } + endpoint := strings.TrimRight(c.Config.BaseURL, "/") + SearchPath + "?" + params.Encode() + body, err := c.get(ctx, endpoint) + if err != nil { + return nil, err + } + out, err := parseResults(body) + if err != nil { + return nil, err + } + for i := range out { + out[i].Source = c.Config.Code + } + return out, nil +} + +func parseResults(body []byte) ([]indexer.Result, error) { + var rows []searchResult + if err := json.Unmarshal(body, &rows); err != nil { + return nil, fmt.Errorf("%w: parse search results: %v", indexer.ErrUnreachable, err) + } + out := make([]indexer.Result, 0, len(rows)) + for _, row := range rows { + title := strings.TrimSpace(row.Title) + if title == "" { + continue + } + r := indexer.Result{ + Title: title, + SizeBytes: row.Size, + Seeders: row.Seeders, + Peers: row.Leechers, + Indexer: strings.TrimSpace(row.Indexer), + Magnet: firstMagnet(row.MagnetURL, row.GUID), + DownloadURL: strings.TrimSpace(row.DownloadURL), + } + if len(row.Categories) > 0 { + r.Category = row.Categories[0].ID + } + r.InfoHash = indexer.NormalizeHash(row.InfoHash) + if r.InfoHash == "" { + r.InfoHash = indexer.MagnetHash(r.Magnet) + } + if r.InfoHash == "" && indexer.IsInfoHash(row.GUID) { + r.InfoHash = indexer.NormalizeHash(row.GUID) + } + if t, err := time.Parse(time.RFC3339, strings.TrimSpace(row.PublishDate)); err == nil { + r.Published = t + } + out = append(out, r) + } + return out, nil +} + +func firstMagnet(candidates ...string) string { + for _, c := range candidates { + c = strings.TrimSpace(c) + if strings.HasPrefix(c, "magnet:") { + return c + } + } + return "" +} + +// Download fetches a result's .torrent through Prowlarr, which holds the +// tracker credentials. A public tracker's link often redirects to a magnet, +// which comes back as an *indexer.MagnetLink. +func (c *Client) Download(ctx context.Context, r indexer.Result) ([]byte, error) { + if r.DownloadURL == "" { + return nil, fmt.Errorf("%w: %q has no download URL", indexer.ErrUnreachable, r.Title) + } + body, err := c.get(ctx, r.DownloadURL) + if err != nil { + return nil, err + } + return indexer.TorrentBody(r.Title, body) +} + +func (c *Client) get(ctx context.Context, endpoint string) ([]byte, error) { + if c.Config.BaseURL == "" || c.Config.APIKey == "" { + return nil, indexer.ErrUnconfigured + } + timeout := c.Config.Timeout + if timeout <= 0 { + timeout = DefaultTimeout + } + ctx, cancel := context.WithTimeout(ctx, timeout) + defer cancel() + req, err := http.NewRequestWithContext(ctx, http.MethodGet, endpoint, nil) + if err != nil { + return nil, fmt.Errorf("%w: build request: %v", indexer.ErrUnreachable, err) + } + req.Header.Set("User-Agent", indexer.UserAgent) + // In a header, not ?apikey=, so the key stays out of URLs, logs and + // error strings. + req.Header.Set("X-Api-Key", c.Config.APIKey) + req.Header.Set("Accept", "application/json") + resp, body, err := indexer.Send(c.HTTP, req) + if err != nil { + return nil, err + } + if resp.StatusCode == http.StatusUnauthorized || resp.StatusCode == http.StatusForbidden { + return nil, fmt.Errorf("%w: HTTP %d", indexer.ErrAuth, resp.StatusCode) + } + if err := indexer.StatusError(resp, "Prowlarr"); err != nil { + return nil, err + } + return body, nil +} diff --git a/internal/prowlarr/prowlarr_test.go b/internal/prowlarr/prowlarr_test.go new file mode 100644 index 00000000..2d4f3011 --- /dev/null +++ b/internal/prowlarr/prowlarr_test.go @@ -0,0 +1,78 @@ +package prowlarr + +import ( + "context" + "errors" + "net/http" + "net/http/httptest" + "testing" + + "git.fabledsword.com/bvandeusen/minstrel/internal/indexer" +) + +const sampleResults = `[ + {"title": "Radiohead - In Rainbows (2007) [FLAC]", "infoHash": "AABBCCDDEEFF00112233445566778899AABBCCDD", + "downloadUrl": "http://prowlarr/1/download?link=x", "size": 400000000, "seeders": 30, "leechers": 4, + "indexer": "RED", "publishDate": "2026-10-01T12:00:00Z", "categories": [{"id": 3040}]}, + {"title": "Radiohead - Amnesiac", "guid": "magnet:?xt=urn:btih:ffeeddccbbaa99887766554433221100ffeeddcc"}, + {"title": ""} +]` + +func TestSearch_AsksTheNativeAPIWithTheKeyInAHeader(t *testing.T) { + reqs := make(chan *http.Request, 1) + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + reqs <- r + _, _ = w.Write([]byte(sampleResults)) + })) + defer srv.Close() + + c := &Client{Config: Config{Code: "prowlarr", BaseURL: srv.URL + "/", APIKey: "secret"}} + results, err := c.Search(context.Background(), indexer.Query{Text: "radiohead", Categories: []int{3000}}) + if err != nil { + t.Fatalf("search: %v", err) + } + got := <-reqs + if got.URL.Path != SearchPath || got.URL.Query().Get("type") != "search" || got.URL.Query().Get("categories") != "3000" { + t.Errorf("request = %s, want %s with type=search and the categories", got.URL, SearchPath) + } + if got.Header.Get("X-Api-Key") != "secret" || got.URL.Query().Get("apikey") != "" { + t.Error("the key belongs in X-Api-Key, never in the URL") + } + if len(results) != 2 { + t.Fatalf("results = %d, want 2", len(results)) + } + if r := results[0]; r.InfoHash != "aabbccddeeff00112233445566778899aabbccdd" || r.Peers != 4 || + r.Category != 3040 || r.Source != "prowlarr" || r.Published.IsZero() { + t.Errorf("first = %+v", r) + } + if r := results[1]; r.Magnet == "" || r.InfoHash != "ffeeddccbbaa99887766554433221100ffeeddcc" { + t.Errorf("second = %+v, want the GUID's magnet and its hash", r) + } +} + +func TestSearch_Refusals(t *testing.T) { + for name, tc := range map[string]struct { + status int + want error + }{ + "a refused key": {http.StatusUnauthorized, indexer.ErrAuth}, + "an outage": {http.StatusBadGateway, indexer.ErrUnreachable}, + } { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(tc.status) + })) + c := &Client{Config: Config{BaseURL: srv.URL, APIKey: "k"}} + _, err := c.Search(context.Background(), indexer.Query{Text: "x"}) + srv.Close() + if !errors.Is(err, tc.want) { + t.Errorf("%s: err = %v, want %v", name, err, tc.want) + } + } +} + +func TestSearch_NoKeyIsUnconfigured(t *testing.T) { + c := &Client{Config: Config{BaseURL: "http://prowlarr:9696"}} + if _, err := c.Search(context.Background(), indexer.Query{Text: "x"}); !errors.Is(err, indexer.ErrUnconfigured) { + t.Fatalf("err = %v, want ErrUnconfigured", err) + } +} diff --git a/web/src/lib/api/admin.ts b/web/src/lib/api/admin.ts index 77e1a3b7..b56a8ad8 100644 --- a/web/src/lib/api/admin.ts +++ b/web/src/lib/api/admin.ts @@ -1009,3 +1009,82 @@ export async function updateNotificationEmailSettings( ): Promise { return api.put('/api/admin/notification-email', s); } + +// Indexers and download clients (M502 #5518) ------------------------------- + +export type AcquisitionProtocol = 'prowlarr' | 'torznab' | 'qbittorrent' | 'exchange'; + +/** One indexer or download client. Secrets never come back, only whether set. */ +export type AcquisitionService = { + code: string; + kind: 'search' | 'download'; + protocol: AcquisitionProtocol; + label: string; + enabled: boolean; + base_url: string; + has_api_key: boolean; + has_password: boolean; + username: string; + category: string; + timeout_ms: number; + /** The three switches mean something on an exchange row only. */ + serves_search: boolean; + serves_downloads: boolean; + serves_enrichment: boolean; + local_path: string; + remote_path: string; + searches_per_hour: number; + /** Switched on and addressed. */ + usable: boolean; +}; + +export type AcquisitionUpdate = Omit< + AcquisitionService, + 'code' | 'kind' | 'protocol' | 'has_api_key' | 'has_password' | 'usable' +>; + +export type AcquisitionTestResult = { ok: boolean; detail: string }; + +const acquisitionBase = '/api/admin/acquisition/services'; + +export async function listAcquisitionServices(): Promise { + const res = await api.get<{ services: AcquisitionService[] }>(acquisitionBase); + return res.services; +} + +export async function addAcquisitionService( + protocol: AcquisitionProtocol +): Promise { + return api.post(acquisitionBase, { protocol }); +} + +export async function updateAcquisitionService( + code: string, + u: AcquisitionUpdate +): Promise { + return api.put(`${acquisitionBase}/${encodeURIComponent(code)}`, u); +} + +export async function removeAcquisitionService(code: string): Promise { + await api.del(`${acquisitionBase}/${encodeURIComponent(code)}`); +} + +export async function testAcquisitionService(code: string): Promise { + return api.post(`${acquisitionBase}/${encodeURIComponent(code)}/test`, {}); +} + +/** Stores a key or password. Write-only: nothing comes back. */ +export async function setAcquisitionSecret( + code: string, + which: 'api-key' | 'password', + value: string +): Promise { + await api.put(`${acquisitionBase}/${encodeURIComponent(code)}/${which}`, { value }); +} + +export async function clearAcquisitionSecret( + code: string, + which: 'api-key' | 'password' +): Promise { + await api.del(`${acquisitionBase}/${encodeURIComponent(code)}/${which}`); +} diff --git a/web/src/lib/components/AcquisitionCard.svelte b/web/src/lib/components/AcquisitionCard.svelte new file mode 100644 index 00000000..5e711b97 --- /dev/null +++ b/web/src/lib/components/AcquisitionCard.svelte @@ -0,0 +1,196 @@ + + +{#snippet addMenu(kind: 'search' | 'download', label: string)} + {#if adding === kind} +
    + {#each choices[kind] as choice (choice.protocol)} + + {/each} + +
    + {:else} + + {/if} +{/snippet} + +
    +
    +

    + Indexers and download clients +

    +

    + Two jobs, kept apart: something to search with, and something to download with. Minstrel + speaks Prowlarr, Torznab and the qBittorrent WebUI, so a stock setup is a complete one. +

    +
    + + {#if loadError} +

    + Couldn't load indexers and download clients. + +

    + {:else if services === null} +

    Loading indexers and download clients…

    + {:else} +

    + {#if !canFetch} + + No download client is switched on, so nothing can be downloaded. + + {:else if !canSearch} + + No indexer is switched on, so nothing new can be found. + + {:else} + Music can be searched for and downloaded. + {/if} +

    + +
    +
    +

    Indexers: where music is looked for

    +

    + Every indexer switched on here is searched, and the results are merged. If one is + unreachable the search still runs, narrower. +

    +
    + {#each indexers as service (service.code)} + + {:else} +

    No indexer yet. Prowlarr is the usual choice.

    + {/each} + {#if searchingExchanges.length > 0} +

    + Also searched: {searchingExchanges.map((s) => s.label).join(', ')}, set up under download + clients below. +

    + {/if} + {@render addMenu('search', 'Add indexer')} +
    + +
    +
    +

    + Download clients: what actually downloads +

    +

    + What a grab is handed to. A FabledExchange here can do more than download, and its entry + says which of its jobs are switched on. +

    +
    + {#each clients as service (service.code)} + + {:else} +

    No download client yet.

    + {/each} + {@render addMenu('download', 'Add download client')} +
    + {/if} +
    diff --git a/web/src/lib/components/AcquisitionCard.test.ts b/web/src/lib/components/AcquisitionCard.test.ts new file mode 100644 index 00000000..3db6448f --- /dev/null +++ b/web/src/lib/components/AcquisitionCard.test.ts @@ -0,0 +1,168 @@ +import { afterEach, describe, expect, test, vi } from 'vitest'; +import { render, screen, fireEvent, waitFor, within } from '@testing-library/svelte'; +import type { AcquisitionService } from '#lib/api/admin.js'; + +vi.mock('#lib/api/admin.js', () => ({ + listAcquisitionServices: vi.fn(), + addAcquisitionService: vi.fn(), + updateAcquisitionService: vi.fn(), + removeAcquisitionService: vi.fn(), + testAcquisitionService: vi.fn(), + setAcquisitionSecret: vi.fn(), + clearAcquisitionSecret: vi.fn() +})); + +vi.mock('#lib/stores/toast.svelte.js', () => ({ pushToast: vi.fn() })); + +import AcquisitionCard from './AcquisitionCard.svelte'; +import { + addAcquisitionService, + listAcquisitionServices, + setAcquisitionSecret, + testAcquisitionService, + updateAcquisitionService +} from '#lib/api/admin.js'; + +function service(over: Partial = {}): AcquisitionService { + return { + code: 'prowlarr', + kind: 'search', + protocol: 'prowlarr', + label: 'Prowlarr', + enabled: false, + base_url: '', + has_api_key: false, + has_password: false, + username: '', + category: '', + timeout_ms: 60000, + serves_search: true, + serves_downloads: true, + serves_enrichment: true, + local_path: '/downloads', + remote_path: '', + searches_per_hour: 120, + usable: false, + ...over + }; +} + +const exchange = (over: Partial = {}) => + service({ + code: 'exchange', + kind: 'download', + protocol: 'exchange', + label: 'FabledExchange', + category: 'minstrel', + searches_per_hour: 0, + ...over + }); + +afterEach(() => vi.clearAllMocks()); + +async function renderCard(services: AcquisitionService[]) { + vi.mocked(listAcquisitionServices).mockResolvedValue(services); + render(AcquisitionCard); + await screen.findByTestId('acquisition-health'); +} + +describe('AcquisitionCard', () => { + test('starts empty: nothing is seeded', async () => { + await renderCard([]); + expect(screen.getByText(/no indexer yet/i)).toBeInTheDocument(); + expect(screen.getByText(/no download client yet/i)).toBeInTheDocument(); + expect(screen.getByTestId('acquisition-health')).toHaveTextContent(/nothing can be downloaded/i); + }); + + test('adding offers the types each list takes', async () => { + await renderCard([]); + await fireEvent.click(screen.getByRole('button', { name: 'Add indexer' })); + const indexerChoices = within(screen.getByTestId('add-search')); + expect(indexerChoices.getByRole('button', { name: /prowlarr/i })).toBeInTheDocument(); + expect(indexerChoices.getByRole('button', { name: /torznab feed/i })).toBeInTheDocument(); + expect(indexerChoices.queryByRole('button', { name: /qbittorrent/i })).toBeNull(); + + vi.mocked(addAcquisitionService).mockResolvedValue(service()); + await fireEvent.click(indexerChoices.getByRole('button', { name: /prowlarr/i })); + expect(addAcquisitionService).toHaveBeenCalledWith('prowlarr'); + + await fireEvent.click(screen.getByRole('button', { name: 'Add download client' })); + const clientChoices = within(screen.getByTestId('add-download')); + expect(clientChoices.getByRole('button', { name: /qbittorrent/i })).toBeInTheDocument(); + expect(clientChoices.getByRole('button', { name: /fabledexchange/i })).toBeInTheDocument(); + }); + + test('an exchange sits under download clients with a switch per job; others have none', async () => { + await renderCard([ + service({ code: 'qbittorrent', kind: 'download', protocol: 'qbittorrent', label: 'qBittorrent' }), + exchange() + ]); + const clients = within(screen.getByTestId('download-clients')); + expect(clients.getAllByTestId('acquisition-service')).toHaveLength(2); + // Only the exchange carries the job switches. + expect(clients.getAllByTestId('exchange-jobs')).toHaveLength(1); + expect(clients.getByRole('checkbox', { name: /downloads what is grabbed/i })).toBeChecked(); + expect(clients.getByRole('checkbox', { name: /searches its dht index/i })).toBeChecked(); + expect(clients.getByRole('checkbox', { name: /checks a release/i })).toBeChecked(); + // qBittorrent asks for a username and a password; the exchange for an optional key. + expect(clients.getByText('Username')).toBeInTheDocument(); + expect(clients.getByText(/save password/i)).toBeInTheDocument(); + expect(clients.getByText(/save key/i)).toBeInTheDocument(); + }); + + test('an exchange searching is named under indexers too', async () => { + await renderCard([exchange({ usable: true, enabled: true, base_url: 'http://ex' })]); + expect(within(screen.getByTestId('indexers')).getByText(/also searched: fabledexchange/i)).toBeInTheDocument(); + expect(screen.getByTestId('acquisition-health')).toHaveTextContent(/searched for and downloaded/i); + }); + + test('switching a job off and saving sends the switches', async () => { + const ex = exchange({ base_url: 'http://ex' }); + await renderCard([ex]); + await fireEvent.click(screen.getByRole('checkbox', { name: /checks a release/i })); + vi.mocked(updateAcquisitionService).mockResolvedValue({ ...ex, serves_enrichment: false }); + await fireEvent.click(screen.getByRole('button', { name: /^save$/i })); + await waitFor(() => expect(updateAcquisitionService).toHaveBeenCalled()); + const [code, sent] = vi.mocked(updateAcquisitionService).mock.calls[0]; + expect(code).toBe('exchange'); + expect(sent).toMatchObject({ serves_enrichment: false, serves_search: true, serves_downloads: true }); + }); + + test('a refused setting is shown beside the form in the server’s words', async () => { + await renderCard([service({ base_url: 'http://p' })]); + await fireEvent.input(screen.getByRole('textbox', { name: /^name$/i }), { + target: { value: 'Mine' } + }); + vi.mocked(updateAcquisitionService).mockRejectedValue({ + code: 'invalid_setting', + message: 'Address: start with http:// or https://.', + status: 400 + }); + await fireEvent.click(screen.getByRole('button', { name: /^save$/i })); + expect(await screen.findByTestId('acquisition-problem')).toHaveTextContent( + 'Address: start with http:// or https://.' + ); + }); + + test('Test shows what the server found, and needs an address', async () => { + await renderCard([service({ code: 'a' }), service({ code: 'b', label: 'Second', base_url: 'http://p' })]); + const [noAddress, addressed] = screen.getAllByRole('button', { name: 'Test' }); + expect(noAddress).toBeDisabled(); + vi.mocked(testAcquisitionService).mockResolvedValue({ + ok: false, + detail: 'the API key was refused: HTTP 401. Check it on the service’s own settings page.' + }); + await fireEvent.click(addressed); + expect(testAcquisitionService).toHaveBeenCalledWith('b'); + expect(await screen.findByTestId('acquisition-test-result')).toHaveTextContent(/key was refused/i); + }); + + test('a key is sent write-only and never shown back', async () => { + await renderCard([service({ base_url: 'http://p', has_api_key: true })]); + expect(screen.getByLabelText(/replace the api key/i)).toHaveValue(''); + await fireEvent.input(screen.getByLabelText(/replace the api key/i), { target: { value: 'k2' } }); + vi.mocked(setAcquisitionSecret).mockResolvedValue(undefined); + await fireEvent.click(screen.getByRole('button', { name: /save key/i })); + await waitFor(() => expect(setAcquisitionSecret).toHaveBeenCalledWith('prowlarr', 'api-key', 'k2')); + }); +}); diff --git a/web/src/lib/components/AcquisitionServiceRow.svelte b/web/src/lib/components/AcquisitionServiceRow.svelte new file mode 100644 index 00000000..dfe2ad99 --- /dev/null +++ b/web/src/lib/components/AcquisitionServiceRow.svelte @@ -0,0 +1,394 @@ + + +
    +
    + +
    + {#each jobs as job (job)} + {job} + {/each} + {#if service.usable} + On + {:else if service.enabled} + On, but no address + {:else} + Off + {/if} +
    +
    + +

    {blurb[service.protocol]}

    + +
    + Connection +
    + + + + + {#if isExchange} +
    + What it does here + + + +
    + {/if} + + {#if isQbit} + + {/if} + + {#if fetches} + + + + {/if} + + {#if searches} + + {/if} + + +
    + +
    + +
    + + + {#if secretSet} + + {/if} +
    + + {#if service.protocol === 'prowlarr'} + Prowlarr shows it under Settings → General. + {:else if isExchange} + Optional: only if your FabledExchange asks for one. Give Minstrel a key of its own there, + so it can be revoked on its own. + {:else if service.protocol === 'torznab'} + Optional: only if the feed asks for one. + {:else} + Empty if your WebUI does not ask for one. + {/if} + {secretSet ? 'One is saved.' : ''} Never shown again once saved. + +
    +
    + + {#if problem} +

    {problem}

    + {/if} + {#if result} +

    + {result.detail} +

    + {/if} + +
    + {#if confirmRemove} + + Remove {service.label}? Minstrel forgets its address and credentials. + Nothing on the service itself changes. + + + + {:else} + + + + {/if} +
    +
    diff --git a/web/src/lib/styles/error-copy.json b/web/src/lib/styles/error-copy.json index b3b56229..caf14ca5 100644 --- a/web/src/lib/styles/error-copy.json +++ b/web/src/lib/styles/error-copy.json @@ -53,5 +53,7 @@ "album_not_found": "That album no longer exists.", "artist_not_found": "That artist no longer exists.", "playlist_not_found": "That playlist no longer exists.", - "user_not_found": "That user no longer exists." + "user_not_found": "That user no longer exists.", + "service_not_found": "That connection no longer exists. Refresh the page.", + "too_many_services": "There are already twenty of these. Remove one you no longer use first." } diff --git a/web/src/routes/admin/integrations/+page.svelte b/web/src/routes/admin/integrations/+page.svelte index 81d06060..ae0a137c 100644 --- a/web/src/routes/admin/integrations/+page.svelte +++ b/web/src/routes/admin/integrations/+page.svelte @@ -28,6 +28,7 @@ import { pushToast } from '#lib/stores/toast.svelte.js'; import Modal from '#lib/components/Modal.svelte'; import AcoustIDCard from '#lib/components/AcoustIDCard.svelte'; + import AcquisitionCard from '#lib/components/AcquisitionCard.svelte'; import NetworkSettingsCard from '#lib/components/NetworkSettingsCard.svelte'; import PublicAddressCard from '#lib/components/PublicAddressCard.svelte'; import NotificationEmailCard from '#lib/components/NotificationEmailCard.svelte'; @@ -420,6 +421,8 @@ {/if} + +
    diff --git a/web/src/routes/admin/integrations/integrations.test.ts b/web/src/routes/admin/integrations/integrations.test.ts index cfb148e9..577ee94c 100644 --- a/web/src/routes/admin/integrations/integrations.test.ts +++ b/web/src/routes/admin/integrations/integrations.test.ts @@ -37,7 +37,10 @@ vi.mock('#lib/api/admin.js', () => ({ listUnsettledAcoustID: vi.fn(), // NotificationEmailCard loads on mount; left pending like AcoustIDCard. getNotificationEmailSettings: vi.fn(() => new Promise(() => {})), - updateNotificationEmailSettings: vi.fn() + updateNotificationEmailSettings: vi.fn(), + // AcquisitionCard loads on mount; left pending like AcoustIDCard. + listAcquisitionServices: vi.fn(() => new Promise(() => {})), + addAcquisitionService: vi.fn() })); import IntegrationsPage from './+page.svelte'; -- 2.54.0 From 77389f88bc379e69d98480aef6b16cfd410e6488 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Fri, 9 Oct 2026 21:08:22 -0400 Subject: [PATCH 006/117] feat: a release catalogue from MusicBrainz; request search asks it directly (M502 #5519) Minstrel keeps its own record of what artists have released, read from MusicBrainz, so the wanted list, search and import no longer need Lidarr. - internal/musicbrainz: searches (artists, release groups, recordings) and browses (an artist's release groups; a group's releases with track lists). One process-wide pacer at 1.1s, shared with tag enrichment, because MusicBrainz's limit is per address. A 503 throttle is retried. - Migration 0080: mb_artists, mb_release_groups, mb_artist_release_groups (a collaboration belongs to each credited artist), mb_releases, mb_release_tracks. - internal/catalogue: Discography and Releases serve the held copy and re-read it when older than 7 or 30 days. An unreachable MusicBrainz serves the stale copy, marked. An hourly pass re-reads due artists. - Art for what isn't held comes through /api/catalogue/art from the enabled cover-art sources (Cover Art Archive by release group, then the name-based sources) and is kept on disk, so the browser never fetches a third-party host. A miss is a 404 and the card shows its icon. - Request search moves from /api/lidarr/search to /api/catalogue/search, with the same response shape. Albums are release groups, and "in library" matches albums.release_group_mbid. A 10-minute cache absorbs search-as-you-type. Discover suggestions point at the art endpoint instead of looking up Lidarr. - Android calls the new path and loads Discover art through ServerImage. Installed builds older than this one lose Discover search until they update (no compatibility path, rule 22). Co-Authored-By: Claude Opus 5.5 --- .../minstrel/api/endpoints/DiscoverApi.kt | 9 +- .../minstrel/discover/ui/DiscoverTiles.kt | 17 +- .../minstrel/models/wire/DiscoverWire.kt | 7 +- cmd/minstrel/main.go | 6 + internal/api/api.go | 27 +- internal/api/catalogue.go | 284 +++++++++++++ internal/api/catalogue_test.go | 253 +++++++++++ internal/api/lidarr.go | 138 ------ internal/api/lidarr_test.go | 280 +----------- internal/api/suggestions.go | 62 +-- internal/catalogue/art.go | 214 ++++++++++ internal/catalogue/art_test.go | 165 +++++++ internal/catalogue/catalogue.go | 311 ++++++++++++++ internal/catalogue/catalogue_test.go | 247 +++++++++++ internal/coverart/provider.go | 8 + internal/coverart/provider_mbcaa.go | 18 +- internal/coverart/provider_mbcaa_test.go | 21 + internal/db/dbq/catalogue.sql.go | 402 ++++++++++++++++++ internal/db/dbq/models.go | 53 +++ .../0080_release_catalogue.down.sql | 5 + .../migrations/0080_release_catalogue.up.sql | 86 ++++ internal/db/queries/catalogue.sql | 101 +++++ internal/dbtest/reset.go | 5 + internal/musicbrainz/client.go | 208 +++++++++ internal/musicbrainz/client_test.go | 202 +++++++++ internal/musicbrainz/entities.go | 383 +++++++++++++++++ internal/musicbrainz/pace.go | 59 +++ internal/tags/httpclient.go | 7 + internal/tags/provider_musicbrainz.go | 18 +- .../api/{lidarr.test.ts => catalogue.test.ts} | 32 +- web/src/lib/api/catalogue.ts | 26 ++ web/src/lib/api/lidarr.ts | 28 -- web/src/lib/api/queries.ts | 4 +- web/src/lib/api/types.ts | 5 +- .../lib/components/DiscoverResultCard.svelte | 14 +- .../lib/components/DiscoverResultCard.test.ts | 16 + web/src/lib/styles/error-copy.json | 1 + .../routes/admin/integrations/+page.svelte | 4 +- web/src/routes/discover/+page.svelte | 24 +- web/src/routes/discover/discover.test.ts | 50 +-- 40 files changed, 3223 insertions(+), 577 deletions(-) create mode 100644 internal/api/catalogue.go create mode 100644 internal/api/catalogue_test.go delete mode 100644 internal/api/lidarr.go create mode 100644 internal/catalogue/art.go create mode 100644 internal/catalogue/art_test.go create mode 100644 internal/catalogue/catalogue.go create mode 100644 internal/catalogue/catalogue_test.go create mode 100644 internal/db/dbq/catalogue.sql.go create mode 100644 internal/db/migrations/0080_release_catalogue.down.sql create mode 100644 internal/db/migrations/0080_release_catalogue.up.sql create mode 100644 internal/db/queries/catalogue.sql create mode 100644 internal/musicbrainz/client.go create mode 100644 internal/musicbrainz/client_test.go create mode 100644 internal/musicbrainz/entities.go create mode 100644 internal/musicbrainz/pace.go rename web/src/lib/api/{lidarr.test.ts => catalogue.test.ts} (60%) create mode 100644 web/src/lib/api/catalogue.ts delete mode 100644 web/src/lib/api/lidarr.ts diff --git a/android/app/src/main/java/com/fabledsword/minstrel/api/endpoints/DiscoverApi.kt b/android/app/src/main/java/com/fabledsword/minstrel/api/endpoints/DiscoverApi.kt index 2b8e1945..9d46b41e 100644 --- a/android/app/src/main/java/com/fabledsword/minstrel/api/endpoints/DiscoverApi.kt +++ b/android/app/src/main/java/com/fabledsword/minstrel/api/endpoints/DiscoverApi.kt @@ -13,10 +13,11 @@ import retrofit2.http.Path import retrofit2.http.Query /** - * Retrofit interface for Discover / Lidarr search / request creation. + * Retrofit interface for Discover / catalogue search / request creation. * - * `/api/lidarr/search` has a 60s LRU on the server so quick re-types - * of the same query are cheap. + * `/api/catalogue/search` asks MusicBrainz, which allows one request a + * second, so the Discover screen searches when the query is submitted, + * not on every keystroke. * * Cancel + my-requests-list endpoints live on RequestsApi (Phase 10 * Commit B), not here. @@ -25,7 +26,7 @@ interface DiscoverApi { @GET("api/discover/suggestions") suspend fun listSuggestions(): List - @GET("api/lidarr/search") + @GET("api/catalogue/search") suspend fun search( @Query("q") query: String, @Query("kind") kind: String, diff --git a/android/app/src/main/java/com/fabledsword/minstrel/discover/ui/DiscoverTiles.kt b/android/app/src/main/java/com/fabledsword/minstrel/discover/ui/DiscoverTiles.kt index cbad59cb..46b74506 100644 --- a/android/app/src/main/java/com/fabledsword/minstrel/discover/ui/DiscoverTiles.kt +++ b/android/app/src/main/java/com/fabledsword/minstrel/discover/ui/DiscoverTiles.kt @@ -25,7 +25,6 @@ import androidx.compose.ui.draw.clip import androidx.compose.ui.graphics.vector.ImageVector import androidx.compose.ui.text.style.TextOverflow import androidx.compose.ui.unit.dp -import coil3.compose.AsyncImage import com.composables.icons.lucide.Clock import com.composables.icons.lucide.Disc3 import com.composables.icons.lucide.Lucide @@ -33,6 +32,7 @@ import com.composables.icons.lucide.User import com.fabledsword.minstrel.models.ArtistSuggestionRef import com.fabledsword.minstrel.models.LidarrSearchResultRef import com.fabledsword.minstrel.models.SuggestionSnoozeRef +import com.fabledsword.minstrel.shared.widgets.ServerImage @Composable internal fun SuggestionTile( @@ -173,18 +173,19 @@ private fun Avatar(imageUrl: String, fallback: ImageVector, circular: Boolean) { .background(MaterialTheme.colorScheme.surfaceVariant), contentAlignment = Alignment.Center, ) { - if (imageUrl.isEmpty()) { + // The server hands back its own art endpoint (relative, M502 #5519), + // which answers 404 when no source has an image: ServerImage resolves + // the path and shows the icon while loading and on a miss. + ServerImage( + url = imageUrl, + contentDescription = null, + modifier = Modifier.fillMaxSize(), + ) { Icon( imageVector = fallback, contentDescription = null, tint = MaterialTheme.colorScheme.onSurfaceVariant, ) - } else { - AsyncImage( - model = imageUrl, - contentDescription = null, - modifier = Modifier.fillMaxSize(), - ) } } } diff --git a/android/app/src/main/java/com/fabledsword/minstrel/models/wire/DiscoverWire.kt b/android/app/src/main/java/com/fabledsword/minstrel/models/wire/DiscoverWire.kt index d7db19fa..b37043e9 100644 --- a/android/app/src/main/java/com/fabledsword/minstrel/models/wire/DiscoverWire.kt +++ b/android/app/src/main/java/com/fabledsword/minstrel/models/wire/DiscoverWire.kt @@ -4,9 +4,10 @@ import kotlinx.serialization.SerialName import kotlinx.serialization.Serializable /** - * One row of `GET /api/lidarr/search`. Mirrors - * `web/src/lib/api/types.ts LidarrSearchResult` / - * the Flutter client's `LidarrSearchResult`. + * One row of `GET /api/catalogue/search` (the MusicBrainz search that replaced + * Lidarr's, M502 #5519). Mirrors `web/src/lib/api/types.ts + * CatalogueSearchResult`. `imageUrl` is the server's own art endpoint, + * relative, so it goes through ServerImage. * * `inLibrary` and `requested` let the UI greyout rows the user can't * act on (already imported / already awaiting review). All defaults diff --git a/cmd/minstrel/main.go b/cmd/minstrel/main.go index 5036d8d1..9a218e51 100644 --- a/cmd/minstrel/main.go +++ b/cmd/minstrel/main.go @@ -15,6 +15,7 @@ import ( "github.com/jackc/pgx/v5/pgxpool" "git.fabledsword.com/bvandeusen/minstrel/internal/acoustid" + "git.fabledsword.com/bvandeusen/minstrel/internal/catalogue" "git.fabledsword.com/bvandeusen/minstrel/internal/config" "git.fabledsword.com/bvandeusen/minstrel/internal/coplay" "git.fabledsword.com/bvandeusen/minstrel/internal/coverart" @@ -325,6 +326,11 @@ func run() error { library.SetNotifier(notifier) go lidarrReconciler.Run(ctx) + // Release catalogue (M502 #5519): re-reads discographies older than a + // week from MusicBrainz, a batch an hour, through the process-wide + // MusicBrainz pacer it shares with tag enrichment. + go catalogue.New(pool, nil, logger.With("component", "catalogue")).Run(ctx) + // Duplicate resolver (M498, M502): classifies the sweep's groups and, with // the operator's auto-resolve on, merges the copies that are safe to merge. // Started after SetNotifier so its summaries reach admins. See diff --git a/internal/api/api.go b/internal/api/api.go index ec2419bf..94518874 100644 --- a/internal/api/api.go +++ b/internal/api/api.go @@ -15,6 +15,7 @@ import ( "git.fabledsword.com/bvandeusen/minstrel/internal/acquire" "git.fabledsword.com/bvandeusen/minstrel/internal/auth" + "git.fabledsword.com/bvandeusen/minstrel/internal/catalogue" "git.fabledsword.com/bvandeusen/minstrel/internal/config" "git.fabledsword.com/bvandeusen/minstrel/internal/coverart" "git.fabledsword.com/bvandeusen/minstrel/internal/eventbus" @@ -72,6 +73,9 @@ func Mount(r chi.Router, pool *pgxpool.Pool, logger *slog.Logger, events *playev loudnessSettings: loudSettings, acoustIDLookup: acoustIDLookup, acquisition: acquire.NewStore(pool), + catalogue: catalogue.New(pool, nil, logger), + catalogueSearches: newSearchCache(), + catalogueArt: catalogue.NewArt(dataDir, artSources(coverSettings)), leveled: newLeveledRenderer(dataDir, loudSettings, logger), librarySize: recommendation.NewLibrarySize(nil), loginGuard: auth.NewLoginGuard(), @@ -184,7 +188,11 @@ func Mount(r chi.Router, pool *pgxpool.Pool, logger *slog.Logger, events *playev authed.Get("/likes/artists", h.handleListLikedArtists) authed.Get("/likes/ids", h.handleGetLikedIDs) - authed.Get("/lidarr/search", h.handleLidarrSearch) + // The release catalogue (M502 #5519): request search asks + // MusicBrainz, and art for what isn't held comes through + // Minstrel from the enabled cover-art sources. + authed.Get("/catalogue/search", h.handleCatalogueSearch) + authed.Get("/catalogue/art/{kind}/{mbid}", h.handleCatalogueArt) authed.Post("/requests", h.handleCreateRequest) authed.Get("/requests", h.handleListRequests) @@ -385,6 +393,13 @@ type handlers struct { acoustIDLookup *library.AcoustIDLookupWorker // acquisition is the operator's indexers and download clients (M502). acquisition *acquire.Store + // catalogue is the release catalogue read from MusicBrainz, and + // catalogueArt the images for what isn't held (M502 #5519). + catalogue *catalogue.Service + catalogueArt *catalogue.Art + // catalogueSearches reuses recent MusicBrainz answers; nil asks every + // time, which is what handler tests get. + catalogueSearches *searchCache // leveled renders the leveled streams handed to Sonos/UPnP speakers // (M464 #5001). Nil when its cache directory cannot be made: a level // request then gets the plain stream. @@ -427,3 +442,13 @@ func newLeveledRenderer(dataDir string, settings *library.LoudnessSettingsServic } return r } + +// artSources hands the art cache the cover-art settings, or nothing when +// there are none: a nil *SettingsService inside the interface would not +// compare equal to nil and would be called. +func artSources(s *coverart.SettingsService) catalogue.ArtSources { + if s == nil { + return nil + } + return s +} diff --git a/internal/api/catalogue.go b/internal/api/catalogue.go new file mode 100644 index 00000000..2a712613 --- /dev/null +++ b/internal/api/catalogue.go @@ -0,0 +1,284 @@ +package api + +import ( + "context" + "errors" + "net/http" + "net/url" + "slices" + "strings" + "sync" + "time" + + "github.com/go-chi/chi/v5" + + "git.fabledsword.com/bvandeusen/minstrel/internal/apierror" + "git.fabledsword.com/bvandeusen/minstrel/internal/catalogue" + "git.fabledsword.com/bvandeusen/minstrel/internal/db/dbq" + "git.fabledsword.com/bvandeusen/minstrel/internal/musicbrainz" +) + +// catalogueSearchLimit is how many results one search shows. +const catalogueSearchLimit = 25 + +// catalogueSearchResult is one row of GET /api/catalogue/search. The shape is +// the one the Lidarr search answered with, so the Discover page and request +// rows read it unchanged. artist_mbid and album_mbid are the parents a +// request needs: empty on an artist, the artist on an album, both on a track. +// An album is a MusicBrainz release group, as Lidarr's albums were. +type catalogueSearchResult struct { + MBID string `json:"mbid"` + Name string `json:"name"` + SecondaryText string `json:"secondary_text"` + ImageURL string `json:"image_url"` + ArtistMBID string `json:"artist_mbid"` + AlbumMBID string `json:"album_mbid"` + InLibrary bool `json:"in_library"` + Requested bool `json:"requested"` +} + +// handleCatalogueSearch implements GET /api/catalogue/search?q=&kind=artist|album|track +// (M502 #5519), asking MusicBrainz directly. An unreachable MusicBrainz is a +// 503 the page shows as such (rule 164). +func (h *handlers) handleCatalogueSearch(w http.ResponseWriter, r *http.Request) { + kind := strings.TrimSpace(r.URL.Query().Get("kind")) + switch kind { + case "artist", "album", "track": + default: + writeErr(w, apierror.BadRequest("bad_kind", "kind must be artist, album, or track")) + return + } + q := strings.TrimSpace(r.URL.Query().Get("q")) + if q == "" { + writeErr(w, apierror.BadRequest("missing_query", "q is required")) + return + } + + out, err := h.catalogueSearches.get(r.Context(), kind, q, h.searchCatalogue) + if err != nil { + switch { + case errors.Is(err, musicbrainz.ErrUnreachable), errors.Is(err, musicbrainz.ErrBadResponse): + writeErr(w, &apierror.Error{Status: 503, Code: "musicbrainz_unreachable", Message: "MusicBrainz can't be reached", Cause: err}) + case r.Context().Err() != nil: + return + default: + h.logger.Error("api: catalogue search", "kind", kind, "err", err) + writeErr(w, apierror.InternalMsg("search failed", err)) + } + return + } + + dbQ := dbq.New(h.pool) + for i := range out { + inLib, err := h.catalogueInLibrary(r.Context(), kind, out[i].MBID) + if err != nil { + writeErr(w, apierror.InternalMsg("library check failed", err)) + return + } + requested, err := dbQ.HasNonTerminalRequestForMBID(r.Context(), out[i].MBID) + if err != nil { + writeErr(w, apierror.InternalMsg("request check failed", err)) + return + } + out[i].InLibrary, out[i].Requested = inLib, requested + } + writeJSON(w, http.StatusOK, out) +} + +func (h *handlers) searchCatalogue(ctx context.Context, kind, q string) ([]catalogueSearchResult, error) { + mb := h.catalogue.MusicBrainz() + out := []catalogueSearchResult{} + switch kind { + case "artist": + artists, err := mb.SearchArtists(ctx, q, catalogueSearchLimit) + if err != nil { + return nil, err + } + for _, a := range artists { + out = append(out, catalogueSearchResult{ + MBID: a.MBID, Name: a.Name, SecondaryText: artistLine(a), + ImageURL: artistArtURL(a.MBID, a.Name), + }) + } + case "album": + groups, err := mb.SearchReleaseGroups(ctx, q, catalogueSearchLimit) + if err != nil { + return nil, err + } + for _, g := range groups { + out = append(out, catalogueSearchResult{ + MBID: g.MBID, Name: g.Title, SecondaryText: joinNonEmpty(g.ArtistCredit, g.Year(), groupType(g)), + ImageURL: groupArtURL(g), ArtistMBID: first(g.ArtistMBIDs), + }) + } + case "track": + recs, err := mb.SearchRecordings(ctx, q, catalogueSearchLimit) + if err != nil { + return nil, err + } + for _, rec := range recs { + res := catalogueSearchResult{ + MBID: rec.MBID, Name: rec.Title, + SecondaryText: joinNonEmpty(rec.ReleaseGroup.Title, rec.ArtistCredit), + ArtistMBID: first(rec.ArtistMBIDs), AlbumMBID: rec.ReleaseGroup.MBID, + } + if rec.ReleaseGroup.MBID != "" { + res.ImageURL = groupArtURL(rec.ReleaseGroup) + } + out = append(out, res) + } + } + return out, nil +} + +// catalogueSearchTTL is how long a search's MusicBrainz answer is reused. +// MusicBrainz allows a request a second, and Discover searches as the user +// types, so the same words are often asked again within moments. +const catalogueSearchTTL = 10 * time.Minute + +// catalogueSearchCacheMax bounds the cache; past it, the cache starts over. +const catalogueSearchCacheMax = 500 + +// searchCache keeps recent MusicBrainz answers by kind and query. A nil +// cache asks every time. Whether a result is held or requested is not +// cached: the handler works that out on every request. +type searchCache struct { + mu sync.Mutex + entries map[string]searchEntry + now func() time.Time +} + +type searchEntry struct { + at time.Time + results []catalogueSearchResult +} + +func newSearchCache() *searchCache { + return &searchCache{entries: map[string]searchEntry{}, now: time.Now} +} + +func (c *searchCache) get(ctx context.Context, kind, q string, + ask func(context.Context, string, string) ([]catalogueSearchResult, error), +) ([]catalogueSearchResult, error) { + if c == nil { + return ask(ctx, kind, q) + } + key := kind + "\x00" + strings.ToLower(q) + c.mu.Lock() + e, ok := c.entries[key] + c.mu.Unlock() + if ok && c.now().Sub(e.at) < catalogueSearchTTL { + return slices.Clone(e.results), nil + } + results, err := ask(ctx, kind, q) + if err != nil { + return nil, err + } + c.mu.Lock() + if len(c.entries) >= catalogueSearchCacheMax { + c.entries = map[string]searchEntry{} + } + c.entries[key] = searchEntry{at: c.now(), results: slices.Clone(results)} + c.mu.Unlock() + return results, nil +} + +// artistLine is what tells two artists of one name apart: MusicBrainz's own +// disambiguation when it has one, else the kind of artist and the country. +func artistLine(a musicbrainz.Artist) string { + if a.Disambiguation != "" { + return a.Disambiguation + } + return joinNonEmpty(a.Type, a.Country) +} + +// groupType names an album's kind the way people do: "Album", "EP", +// "Album · Live", "Single · Remix". +func groupType(g musicbrainz.ReleaseGroup) string { + return joinNonEmpty(append([]string{g.PrimaryType}, g.SecondaryTypes...)...) +} + +func joinNonEmpty(parts ...string) string { + kept := parts[:0:0] + for _, p := range parts { + if p != "" { + kept = append(kept, p) + } + } + return strings.Join(kept, " · ") +} + +func first(s []string) string { + if len(s) == 0 { + return "" + } + return s[0] +} + +// artistArtURL and groupArtURL point at Minstrel's own art endpoints. The +// names ride along for the sources that look art up by name. +func artistArtURL(mbid, name string) string { + if mbid == "" { + return "" + } + return "/api/catalogue/art/artist/" + mbid + "?" + url.Values{"name": {name}}.Encode() +} + +func groupArtURL(g musicbrainz.ReleaseGroup) string { + return "/api/catalogue/art/release-group/" + g.MBID + "?" + + url.Values{"artist": {g.ArtistCredit}, "title": {g.Title}}.Encode() +} + +// catalogueInLibrary reports whether a result is already held: an artist by +// its MBID, an album by its release group (or a release id that happens to be +// the same), a track by its recording. +func (h *handlers) catalogueInLibrary(ctx context.Context, kind, mbid string) (bool, error) { + var query string + switch kind { + case "artist": + query = `SELECT EXISTS(SELECT 1 FROM artists WHERE mbid = $1)` + case "album": + query = `SELECT EXISTS(SELECT 1 FROM albums WHERE release_group_mbid = $1 OR mbid = $1)` + case "track": + query = `SELECT EXISTS(SELECT 1 FROM tracks WHERE mbid = $1)` + default: + return false, nil + } + var exists bool + err := h.pool.QueryRow(ctx, query, mbid).Scan(&exists) + return exists, err +} + +// handleCatalogueArt implements GET /api/catalogue/art/{kind}/{mbid} for an +// artist or a release group Minstrel doesn't hold. The image comes from the +// operator's enabled cover-art sources and is kept on disk, so the browser +// never fetches a third-party host. A 404 shows the placeholder. +func (h *handlers) handleCatalogueArt(w http.ResponseWriter, r *http.Request) { + kind, mbid := chi.URLParam(r, "kind"), chi.URLParam(r, "mbid") + if !catalogue.ValidMBID(mbid) { + writeErr(w, apierror.NotFound("art")) + return + } + var ( + body []byte + err error + ) + switch kind { + case "artist": + body, err = h.catalogueArt.Artist(r.Context(), mbid, r.URL.Query().Get("name")) + case "release-group": + body, err = h.catalogueArt.ReleaseGroup(r.Context(), mbid, r.URL.Query().Get("artist"), r.URL.Query().Get("title")) + default: + writeErr(w, apierror.NotFound("art")) + return + } + if err != nil { + if r.Context().Err() == nil { + writeErr(w, apierror.NotFound("art")) + } + return + } + w.Header().Set("Content-Type", http.DetectContentType(body)) + w.Header().Set("Cache-Control", "private, max-age=86400") + _, _ = w.Write(body) +} diff --git a/internal/api/catalogue_test.go b/internal/api/catalogue_test.go new file mode 100644 index 00000000..de879d2e --- /dev/null +++ b/internal/api/catalogue_test.go @@ -0,0 +1,253 @@ +package api + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "net/url" + "testing" + "time" + + "github.com/go-chi/chi/v5" + + "git.fabledsword.com/bvandeusen/minstrel/internal/catalogue" + "git.fabledsword.com/bvandeusen/minstrel/internal/coverart" + "git.fabledsword.com/bvandeusen/minstrel/internal/db/dbq" + "git.fabledsword.com/bvandeusen/minstrel/internal/dbtest" + "git.fabledsword.com/bvandeusen/minstrel/internal/musicbrainz" +) + +const ( + searchArtistMBID = "a74b1b7f-71a5-4011-9441-d0b5e4122711" + searchGroupMBID = "b1392450-e666-3926-a536-22c65f834433" + searchRecMBID = "c1a0e1b2-0000-4000-8000-000000000001" +) + +// fakeMusicBrainz answers the three searches with one Radiohead result each, +// or 503 when down is set. +func fakeMusicBrainz(t *testing.T, down bool) *httptest.Server { + t.Helper() + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if down { + w.WriteHeader(http.StatusServiceUnavailable) + return + } + credit := `[{"name": "Radiohead", "artist": {"id": "` + searchArtistMBID + `"}}]` + group := `{"id": "` + searchGroupMBID + `", "title": "OK Computer", "primary-type": "Album", + "first-release-date": "1997-05-21", "artist-credit": ` + credit + `}` + switch r.URL.Path { + case "/artist": + _, _ = w.Write([]byte(`{"artists": [{"id": "` + searchArtistMBID + `", "name": "Radiohead", "type": "Group", "country": "GB"}]}`)) + case "/release-group": + _, _ = w.Write([]byte(`{"release-groups": [` + group + `]}`)) + case "/recording": + _, _ = w.Write([]byte(`{"recordings": [{"id": "` + searchRecMBID + `", "title": "Karma Police", + "artist-credit": ` + credit + `, + "releases": [{"id": "r1", "status": "Official", "release-group": ` + group + `}]}]}`)) + default: + w.WriteHeader(http.StatusNotFound) + } + })) + t.Cleanup(srv.Close) + return srv +} + +func catalogueHandlers(t *testing.T, down bool) *handlers { + t.Helper() + h, pool := testHandlers(t) + resetLidarrState(t, h) + mb := &musicbrainz.Client{BaseURL: fakeMusicBrainz(t, down).URL, Pace: &musicbrainz.Pacer{}, Retries: -1} + h.catalogue = catalogue.New(pool, mb, h.logger) + return h +} + +func doCatalogueSearch(t *testing.T, h *handlers, q, kind string) *httptest.ResponseRecorder { + t.Helper() + req := httptest.NewRequest(http.MethodGet, "/api/catalogue/search?"+url.Values{"q": {q}, "kind": {kind}}.Encode(), nil) + w := httptest.NewRecorder() + h.handleCatalogueSearch(w, req) + return w +} + +func decodeResults(t *testing.T, w *httptest.ResponseRecorder) []catalogueSearchResult { + t.Helper() + if w.Code != http.StatusOK { + t.Fatalf("status = %d, body = %s", w.Code, w.Body.String()) + } + var out []catalogueSearchResult + if err := json.Unmarshal(w.Body.Bytes(), &out); err != nil { + t.Fatalf("decode: %v; body = %s", err, w.Body.String()) + } + return out +} + +func errorCode(t *testing.T, w *httptest.ResponseRecorder) string { + t.Helper() + var resp struct { + Error struct { + Code string `json:"code"` + } `json:"error"` + } + if err := json.Unmarshal(w.Body.Bytes(), &resp); err != nil { + t.Fatalf("decode: %v; body = %s", err, w.Body.String()) + } + return resp.Error.Code +} + +func TestCatalogueSearch_RefusesABadKindOrNoQuery(t *testing.T) { + h := catalogueHandlers(t, false) + if w := doCatalogueSearch(t, h, "x", "playlist"); w.Code != http.StatusBadRequest || errorCode(t, w) != "bad_kind" { + t.Errorf("bad kind: %d %s", w.Code, w.Body.String()) + } + if w := doCatalogueSearch(t, h, " ", "artist"); w.Code != http.StatusBadRequest || errorCode(t, w) != "missing_query" { + t.Errorf("no query: %d %s", w.Code, w.Body.String()) + } +} + +func TestCatalogueSearch_EachKindAsksMusicBrainz(t *testing.T) { + h := catalogueHandlers(t, false) + + artists := decodeResults(t, doCatalogueSearch(t, h, "radiohead", "artist")) + if len(artists) != 1 || artists[0].MBID != searchArtistMBID || artists[0].SecondaryText != "Group · GB" { + t.Fatalf("artists = %+v", artists) + } + if artists[0].ImageURL != "/api/catalogue/art/artist/"+searchArtistMBID+"?name=Radiohead" { + t.Errorf("artist art = %q: it must be Minstrel's own endpoint", artists[0].ImageURL) + } + + albums := decodeResults(t, doCatalogueSearch(t, h, "ok computer", "album")) + if len(albums) != 1 || albums[0].MBID != searchGroupMBID || albums[0].ArtistMBID != searchArtistMBID || + albums[0].SecondaryText != "Radiohead · 1997 · Album" { + t.Fatalf("albums = %+v", albums) + } + + tracks := decodeResults(t, doCatalogueSearch(t, h, "karma police", "track")) + if len(tracks) != 1 || tracks[0].AlbumMBID != searchGroupMBID || tracks[0].ArtistMBID != searchArtistMBID || + tracks[0].SecondaryText != "OK Computer · Radiohead" { + t.Fatalf("tracks = %+v", tracks) + } +} + +func TestCatalogueSearch_MarksWhatIsHeldAndWhatIsRequested(t *testing.T) { + h := catalogueHandlers(t, false) + ctx := context.Background() + q := dbq.New(h.pool) + + artistMBID := searchArtistMBID + artist, err := q.UpsertArtist(ctx, dbq.UpsertArtistParams{Name: dbtest.TestUserPrefix + "Radiohead", SortName: "Radiohead", Mbid: &artistMBID}) + if err != nil { + t.Fatal(err) + } + // The album is held under a release id; its release group is what matches. + if _, err := h.pool.Exec(ctx, + `INSERT INTO albums (title, sort_title, artist_id, mbid, release_group_mbid) VALUES ('OK Computer', 'OK Computer', $1, 'some-release-id', $2)`, + artist.ID, searchGroupMBID); err != nil { + t.Fatal(err) + } + if got := decodeResults(t, doCatalogueSearch(t, h, "radiohead", "artist")); !got[0].InLibrary { + t.Error("the artist is held") + } + if got := decodeResults(t, doCatalogueSearch(t, h, "ok computer", "album")); !got[0].InLibrary { + t.Error("the album is held, matched by release group") + } + + user := seedUser(t, h.pool, "requester", "pw", false) + track := searchRecMBID + if _, err := q.CreateLidarrRequest(ctx, dbq.CreateLidarrRequestParams{ + UserID: user.ID, Kind: dbq.LidarrRequestKindTrack, LidarrArtistMbid: searchArtistMBID, + LidarrTrackMbid: &track, ArtistName: "Radiohead", + }); err != nil { + t.Fatal(err) + } + got := decodeResults(t, doCatalogueSearch(t, h, "karma police", "track")) + if got[0].InLibrary || !got[0].Requested { + t.Errorf("track = %+v, want requested and not held", got[0]) + } +} + +func TestCatalogueSearch_UnreachableMusicBrainzIs503(t *testing.T) { + h := catalogueHandlers(t, true) + w := doCatalogueSearch(t, h, "radiohead", "artist") + if w.Code != http.StatusServiceUnavailable || errorCode(t, w) != "musicbrainz_unreachable" { + t.Fatalf("status %d, body %s", w.Code, w.Body.String()) + } +} + +// artSource is an artist-art source with one image. +type artSource struct{ apiTestAlbumProvider } + +func (artSource) FetchArtistArt(_ context.Context, _ coverart.ArtistRef) ([]byte, []byte, error) { + return []byte("\x89PNG\r\n\x1a\nrest"), nil, nil +} + +type fakeArtSources struct{ artists []coverart.ArtistArtProvider } + +func (s fakeArtSources) EnabledArtistProviders() []coverart.ArtistArtProvider { return s.artists } +func (fakeArtSources) EnabledAlbumProviders() []coverart.AlbumCoverProvider { return nil } + +func TestCatalogueArt_ServesTheImageOrAPlaceholder404(t *testing.T) { + h := &handlers{catalogueArt: catalogue.NewArt(t.TempDir(), fakeArtSources{artists: []coverart.ArtistArtProvider{&artSource{}}})} + r := chi.NewRouter() + r.Get("/api/catalogue/art/{kind}/{mbid}", h.handleCatalogueArt) + + get := func(path string) *httptest.ResponseRecorder { + w := httptest.NewRecorder() + r.ServeHTTP(w, httptest.NewRequest(http.MethodGet, path, nil)) + return w + } + if w := get("/api/catalogue/art/artist/" + searchArtistMBID + "?name=Radiohead"); w.Code != http.StatusOK || w.Header().Get("Content-Type") != "image/png" { + t.Errorf("artist: %d %q", w.Code, w.Header().Get("Content-Type")) + } + // No album source is enabled: a 404, which the page shows as its placeholder. + if w := get("/api/catalogue/art/release-group/" + searchGroupMBID); w.Code != http.StatusNotFound { + t.Errorf("release group with no source: %d", w.Code) + } + for _, bad := range []string{"/api/catalogue/art/artist/not-an-id", "/api/catalogue/art/label/" + searchArtistMBID} { + if w := get(bad); w.Code != http.StatusNotFound { + t.Errorf("%s: %d", bad, w.Code) + } + } +} + +func TestSuggestionArt_PointsAtMinstrelsOwnEndpoint(t *testing.T) { + views := []suggestionView{{MBID: searchArtistMBID, Name: "Sigur Rós"}, {Name: "No MBID"}} + resolveSuggestionArt(views) + if views[0].ImageURL != "/api/catalogue/art/artist/"+searchArtistMBID+"?name=Sigur+R%C3%B3s" { + t.Errorf("image_url = %q", views[0].ImageURL) + } + if views[1].ImageURL != "" { + t.Errorf("no MBID, no image: %q", views[1].ImageURL) + } +} + +func TestSearchCache_ReusesAnAnswerButNotTheFlags(t *testing.T) { + c := newSearchCache() + now := time.Now() + c.now = func() time.Time { return now } + asks := 0 + ask := func(context.Context, string, string) ([]catalogueSearchResult, error) { + asks++ + return []catalogueSearchResult{{MBID: "m"}}, nil + } + first, _ := c.get(context.Background(), "artist", "Radiohead", ask) + first[0].InLibrary = true // the handler sets flags on what it is given + again, _ := c.get(context.Background(), "artist", "radiohead", ask) + if asks != 1 { + t.Errorf("asks = %d: the same words in another case are the same search", asks) + } + if again[0].InLibrary { + t.Error("a flag set on one answer leaked into the cache") + } + if _, _ = c.get(context.Background(), "album", "radiohead", ask); asks != 2 { + t.Error("another kind is another search") + } + now = now.Add(catalogueSearchTTL + time.Second) + if _, _ = c.get(context.Background(), "artist", "radiohead", ask); asks != 3 { + t.Error("an old answer is asked again") + } + var none *searchCache + if _, _ = none.get(context.Background(), "artist", "x", ask); asks != 4 { + t.Error("a nil cache asks every time") + } +} diff --git a/internal/api/lidarr.go b/internal/api/lidarr.go deleted file mode 100644 index 59e0dda6..00000000 --- a/internal/api/lidarr.go +++ /dev/null @@ -1,138 +0,0 @@ -package api - -import ( - "context" - "errors" - "net/http" - "strings" - - "git.fabledsword.com/bvandeusen/minstrel/internal/apierror" - "git.fabledsword.com/bvandeusen/minstrel/internal/db/dbq" - "git.fabledsword.com/bvandeusen/minstrel/internal/lidarr" -) - -// lidarrSearchResult is the JSON shape returned by GET /api/lidarr/search. -// Field names follow the lowercase snake_case convention used by all /api/* -// responses. artist_mbid and album_mbid are the parent identifiers surfaced -// by LookupAlbum / LookupTrack; they are empty strings on artist results. -type lidarrSearchResult struct { - MBID string `json:"mbid"` - Name string `json:"name"` - SecondaryText string `json:"secondary_text"` - ImageURL string `json:"image_url"` - ArtistMBID string `json:"artist_mbid"` - AlbumMBID string `json:"album_mbid"` - InLibrary bool `json:"in_library"` - Requested bool `json:"requested"` -} - -// handleLidarrSearch implements GET /api/lidarr/search?q=&kind=artist|album|track. -// -// Steps: -// 1. Validate kind and q params. -// 2. Load lidarr config; 503 when disabled. -// 3. Call the matching Lidarr lookup. -// 4. For each result, compute in_library (EXISTS by MBID) and requested -// (HasNonTerminalRequestForMBID). -// 5. Return normalized JSON array. -func (h *handlers) handleLidarrSearch(w http.ResponseWriter, r *http.Request) { - kind := strings.TrimSpace(r.URL.Query().Get("kind")) - switch kind { - case "artist", "album", "track": - // valid - default: - writeErr(w, apierror.BadRequest("bad_kind", "kind must be artist, album, or track")) - return - } - - q := strings.TrimSpace(r.URL.Query().Get("q")) - if q == "" { - writeErr(w, apierror.BadRequest("missing_query", "q is required")) - return - } - - cfg, err := h.lidarrCfg.Get(r.Context()) - if err != nil { - h.logger.Error("api: lidarr search: load config", "err", err) - writeErr(w, &apierror.Error{Status: 500, Code: "lidarr_lookup_failed", Message: "failed to load lidarr config", Cause: err}) - return - } - if !cfg.Enabled { - writeErr(w, &apierror.Error{Status: 503, Code: "lidarr_disabled", Message: "Lidarr integration is not enabled"}) - return - } - - client := lidarr.NewClient(cfg.BaseURL, cfg.APIKey) - - var results []lidarr.LookupResult - switch kind { - case "artist": - results, err = client.LookupArtist(r.Context(), q) - case "album": - results, err = client.LookupAlbum(r.Context(), q) - case "track": - results, err = client.LookupTrack(r.Context(), q) - } - if err != nil { - switch { - case errors.Is(err, lidarr.ErrUnreachable): - writeErr(w, &apierror.Error{Status: 503, Code: "lidarr_unreachable", Message: "Lidarr is unreachable", Cause: err}) - case errors.Is(err, lidarr.ErrAuthFailed): - writeErr(w, &apierror.Error{Status: 503, Code: "lidarr_auth_failed", Message: "Lidarr authentication failed", Cause: err}) - default: - h.logger.Error("api: lidarr search: lookup failed", "kind", kind, "err", err) - writeErr(w, &apierror.Error{Status: 500, Code: "lidarr_lookup_failed", Message: "Lidarr lookup failed", Cause: err}) - } - return - } - - dbQ := dbq.New(h.pool) - out := make([]lidarrSearchResult, 0, len(results)) - for _, res := range results { - inLib, libErr := h.lidarrInLibrary(r.Context(), kind, res.MBID) - if libErr != nil { - h.logger.Error("api: lidarr search: in_library check failed", "err", libErr) - writeErr(w, &apierror.Error{Status: 500, Code: "lidarr_lookup_failed", Message: "library check failed", Cause: libErr}) - return - } - requested, reqErr := dbQ.HasNonTerminalRequestForMBID(r.Context(), res.MBID) - if reqErr != nil { - h.logger.Error("api: lidarr search: requested check failed", "err", reqErr) - writeErr(w, &apierror.Error{Status: 500, Code: "lidarr_lookup_failed", Message: "request check failed", Cause: reqErr}) - return - } - out = append(out, lidarrSearchResult{ - MBID: res.MBID, - Name: res.Name, - SecondaryText: res.Secondary, - ImageURL: res.ImageURL, - ArtistMBID: res.ArtistMBID, - AlbumMBID: res.AlbumMBID, - InLibrary: inLib, - Requested: requested, - }) - } - writeJSON(w, http.StatusOK, out) -} - -// lidarrInLibrary checks whether the given MBID exists in the corresponding -// local library table based on kind. Runs a direct EXISTS query since -// these per-kind MBID lookups are not in the sqlc-generated query set. -func (h *handlers) lidarrInLibrary(ctx context.Context, kind, mbid string) (bool, error) { - var query string - switch kind { - case "artist": - query = `SELECT EXISTS(SELECT 1 FROM artists WHERE mbid = $1)` - case "album": - query = `SELECT EXISTS(SELECT 1 FROM albums WHERE mbid = $1)` - case "track": - query = `SELECT EXISTS(SELECT 1 FROM tracks WHERE mbid = $1)` - default: - return false, nil - } - var exists bool - if err := h.pool.QueryRow(ctx, query, mbid).Scan(&exists); err != nil { - return false, err - } - return exists, nil -} diff --git a/internal/api/lidarr_test.go b/internal/api/lidarr_test.go index d800292e..3d18771f 100644 --- a/internal/api/lidarr_test.go +++ b/internal/api/lidarr_test.go @@ -2,20 +2,17 @@ package api import ( "context" - "encoding/json" - "fmt" "net/http" "net/http/httptest" "testing" - "time" - "github.com/jackc/pgx/v5/pgtype" - - "git.fabledsword.com/bvandeusen/minstrel/internal/db/dbq" - "git.fabledsword.com/bvandeusen/minstrel/internal/dbtest" "git.fabledsword.com/bvandeusen/minstrel/internal/lidarrconfig" ) +// Helpers shared by the request, quarantine and notification tests. The +// Lidarr search they once sat beside is now the catalogue search +// (catalogue_test.go, M502 #5519). + // resetLidarrState clears lidarr_requests rows and resets lidarr_config // to the factory-default state (enabled=false, nulls) between tests. func resetLidarrState(t *testing.T, h *handlers) { @@ -44,11 +41,6 @@ func saveLidarrConfig(t *testing.T, h *handlers, baseURL string, enabled bool) { } } -// lidarrArtistStubBody returns a minimal Lidarr artist lookup JSON payload. -func lidarrArtistStubBody(mbid, name string) string { - return fmt.Sprintf(`[{"foreignArtistId":%q,"artistName":%q,"genres":["Rock"],"albumCount":5,"images":[]}]`, mbid, name) -} - // newLidarrStub creates an httptest.Server that always returns the given body // with 200 OK for any /api/v1/* path. func newLidarrStub(t *testing.T, body string) *httptest.Server { @@ -61,267 +53,3 @@ func newLidarrStub(t *testing.T, body string) *httptest.Server { t.Cleanup(srv.Close) return srv } - -// seedArtistWithMBID inserts an artist row with a specific MBID for in_library testing. -func seedArtistWithMBID(t *testing.T, h *handlers, name, mbid string) dbq.Artist { - t.Helper() - a, err := dbq.New(h.pool).UpsertArtist(context.Background(), dbq.UpsertArtistParams{ - Name: name, - SortName: name, - Mbid: &mbid, - }) - if err != nil { - t.Fatalf("seedArtistWithMBID: %v", err) - } - return a -} - -// seedRequestForMBID inserts a pending lidarr_requests row for the given MBID. -func seedRequestForMBID(t *testing.T, h *handlers, userID pgtype.UUID, kind dbq.LidarrRequestKind, artistMBID, mbid string) { - t.Helper() - var albumMBID, trackMBID *string - switch kind { - case dbq.LidarrRequestKindAlbum: - albumMBID = &mbid - case dbq.LidarrRequestKindTrack: - trackMBID = &mbid - } - _, err := dbq.New(h.pool).CreateLidarrRequest(context.Background(), dbq.CreateLidarrRequestParams{ - UserID: userID, - Kind: kind, - LidarrArtistMbid: artistMBID, - LidarrAlbumMbid: albumMBID, - LidarrTrackMbid: trackMBID, - ArtistName: "Test Artist", - AlbumTitle: nil, - TrackTitle: nil, - }) - if err != nil { - t.Fatalf("seedRequestForMBID: %v", err) - } -} - -// doLidarrSearch fires a GET /api/lidarr/search request directly at the handler. -// It simulates the RequireUser middleware by injecting a dummy user context. -func doLidarrSearch(t *testing.T, h *handlers, q, kind string) *httptest.ResponseRecorder { - t.Helper() - url := fmt.Sprintf("/api/lidarr/search?q=%s&kind=%s", q, kind) - req := httptest.NewRequest(http.MethodGet, url, nil) - // Inject a minimal user context the same way other handler tests do it. - // The lidarr search handler doesn't use the user from context, so any - // non-nil user value keeps the handler from panicking. - w := httptest.NewRecorder() - h.handleLidarrSearch(w, req) - return w -} - -// TestHandleLidarrSearch_ValidationErrors covers bad kind and missing query -// in a table-driven test. -func TestHandleLidarrSearch_ValidationErrors(t *testing.T) { - h, _ := testHandlers(t) - - cases := []struct { - name string - q string - kind string - wantStatus int - wantCode string - }{ - { - name: "bad kind playlist", - q: "Beatles", - kind: "playlist", - wantStatus: http.StatusBadRequest, - wantCode: "bad_kind", - }, - { - name: "missing query", - q: "", - kind: "artist", - wantStatus: http.StatusBadRequest, - wantCode: "missing_query", - }, - } - - for _, tc := range cases { - t.Run(tc.name, func(t *testing.T) { - url := fmt.Sprintf("/api/lidarr/search?q=%s&kind=%s", tc.q, tc.kind) - req := httptest.NewRequest(http.MethodGet, url, nil) - w := httptest.NewRecorder() - h.handleLidarrSearch(w, req) - - if w.Code != tc.wantStatus { - t.Fatalf("status = %d, want %d; body = %s", w.Code, tc.wantStatus, w.Body.String()) - } - var resp struct { - Error struct { - Code string `json:"code"` - } `json:"error"` - } - if err := json.Unmarshal(w.Body.Bytes(), &resp); err != nil { - t.Fatalf("decode body: %v; body=%s", err, w.Body.String()) - } - if resp.Error.Code != tc.wantCode { - t.Errorf("error.code = %q, want %q", resp.Error.Code, tc.wantCode) - } - }) - } -} - -// TestHandleLidarrSearch_DisabledReturns503 verifies that a disabled Lidarr -// config causes the handler to return 503 lidarr_disabled. -func TestHandleLidarrSearch_DisabledReturns503(t *testing.T) { - h, _ := testHandlers(t) - resetLidarrState(t, h) - - // Config is already disabled after resetLidarrState; ensure it's explicit. - saveLidarrConfig(t, h, "http://127.0.0.1:1", false) - - req := httptest.NewRequest(http.MethodGet, "/api/lidarr/search?q=Beatles&kind=artist", nil) - w := httptest.NewRecorder() - h.handleLidarrSearch(w, req) - - if w.Code != http.StatusServiceUnavailable { - t.Fatalf("status = %d, want 503; body = %s", w.Code, w.Body.String()) - } - var resp struct { - Error struct { - Code string `json:"code"` - } `json:"error"` - } - if err := json.Unmarshal(w.Body.Bytes(), &resp); err != nil { - t.Fatalf("decode body: %v; body=%s", err, w.Body.String()) - } - if resp.Error.Code != "lidarr_disabled" { - t.Errorf("error.code = %q, want lidarr_disabled", resp.Error.Code) - } -} - -// TestHandleLidarrSearch_HappyPath_RequestableResult checks a result that is -// neither in the library nor already requested. -func TestHandleLidarrSearch_HappyPath_RequestableResult(t *testing.T) { - h, _ := testHandlers(t) - resetLidarrState(t, h) - - const mbid = "artist-mbid-abc123" - stub := newLidarrStub(t, lidarrArtistStubBody(mbid, "The Beatles")) - saveLidarrConfig(t, h, stub.URL, true) - - w := doLidarrSearch(t, h, "Beatles", "artist") - - if w.Code != http.StatusOK { - t.Fatalf("status = %d, want 200; body = %s", w.Code, w.Body.String()) - } - var results []lidarrSearchResult - if err := json.Unmarshal(w.Body.Bytes(), &results); err != nil { - t.Fatalf("decode: %v; body=%s", err, w.Body.String()) - } - if len(results) != 1 { - t.Fatalf("len(results) = %d, want 1", len(results)) - } - r := results[0] - if r.MBID != mbid { - t.Errorf("mbid = %q, want %q", r.MBID, mbid) - } - if r.InLibrary { - t.Errorf("in_library = true, want false") - } - if r.Requested { - t.Errorf("requested = true, want false") - } -} - -// TestHandleLidarrSearch_HappyPath_InLibraryResult checks that a result whose -// MBID matches an artist in the local DB has in_library=true. -func TestHandleLidarrSearch_HappyPath_InLibraryResult(t *testing.T) { - h, _ := testHandlers(t) - resetLidarrState(t, h) - - const mbid = "artist-mbid-inlib" - stub := newLidarrStub(t, lidarrArtistStubBody(mbid, "Library Artist")) - saveLidarrConfig(t, h, stub.URL, true) - - // Seed the artist with the matching MBID so in_library becomes true. - seedArtistWithMBID(t, h, dbtest.TestUserPrefix+"LibraryArtist", mbid) - - w := doLidarrSearch(t, h, "Library+Artist", "artist") - - if w.Code != http.StatusOK { - t.Fatalf("status = %d, want 200; body = %s", w.Code, w.Body.String()) - } - var results []lidarrSearchResult - if err := json.Unmarshal(w.Body.Bytes(), &results); err != nil { - t.Fatalf("decode: %v; body=%s", err, w.Body.String()) - } - if len(results) != 1 { - t.Fatalf("len(results) = %d, want 1", len(results)) - } - if !results[0].InLibrary { - t.Errorf("in_library = false, want true") - } -} - -// TestHandleLidarrSearch_HappyPath_AlreadyRequested verifies that a result -// whose MBID matches a pending lidarr_requests row has requested=true. -func TestHandleLidarrSearch_HappyPath_AlreadyRequested(t *testing.T) { - h, _ := testHandlers(t) - resetLidarrState(t, h) - - const mbid = "artist-mbid-requested" - stub := newLidarrStub(t, lidarrArtistStubBody(mbid, "Requested Artist")) - saveLidarrConfig(t, h, stub.URL, true) - - // Seed a user and a pending request for the artist MBID. - user := seedUser(t, h.pool, "requester", "pw", false) - seedRequestForMBID(t, h, user.ID, dbq.LidarrRequestKindArtist, mbid, mbid) - - w := doLidarrSearch(t, h, "Requested+Artist", "artist") - - if w.Code != http.StatusOK { - t.Fatalf("status = %d, want 200; body = %s", w.Code, w.Body.String()) - } - var results []lidarrSearchResult - if err := json.Unmarshal(w.Body.Bytes(), &results); err != nil { - t.Fatalf("decode: %v; body=%s", err, w.Body.String()) - } - if len(results) != 1 { - t.Fatalf("len(results) = %d, want 1", len(results)) - } - if !results[0].Requested { - t.Errorf("requested = false, want true") - } -} - -// TestHandleLidarrSearch_LidarrUnreachableReturns503 verifies that a -// connection-refused Lidarr base URL returns 503 lidarr_unreachable. -func TestHandleLidarrSearch_LidarrUnreachableReturns503(t *testing.T) { - h, _ := testHandlers(t) - resetLidarrState(t, h) - - // Port 1 is reserved and will refuse connections immediately. - saveLidarrConfig(t, h, "http://127.0.0.1:1", true) - - req := httptest.NewRequest(http.MethodGet, "/api/lidarr/search?q=Beatles&kind=artist", nil) - // Use a short-deadline context so the test doesn't hang waiting for TCP. - ctx, cancel := context.WithTimeout(req.Context(), 3*time.Second) - defer cancel() - req = req.WithContext(ctx) - - w := httptest.NewRecorder() - h.handleLidarrSearch(w, req) - - if w.Code != http.StatusServiceUnavailable { - t.Fatalf("status = %d, want 503; body = %s", w.Code, w.Body.String()) - } - var resp struct { - Error struct { - Code string `json:"code"` - } `json:"error"` - } - if err := json.Unmarshal(w.Body.Bytes(), &resp); err != nil { - t.Fatalf("decode: %v; body=%s", err, w.Body.String()) - } - if resp.Error.Code != "lidarr_unreachable" { - t.Errorf("error.code = %q, want lidarr_unreachable", resp.Error.Code) - } -} diff --git a/internal/api/suggestions.go b/internal/api/suggestions.go index 977d571f..cb1a814c 100644 --- a/internal/api/suggestions.go +++ b/internal/api/suggestions.go @@ -1,21 +1,18 @@ package api import ( - "context" "encoding/json" "errors" "io" "net/http" "strconv" "strings" - "sync" "github.com/go-chi/chi/v5" "github.com/jackc/pgx/v5/pgtype" "git.fabledsword.com/bvandeusen/minstrel/internal/apierror" "git.fabledsword.com/bvandeusen/minstrel/internal/db/dbq" - "git.fabledsword.com/bvandeusen/minstrel/internal/lidarr" "git.fabledsword.com/bvandeusen/minstrel/internal/recommendation" ) @@ -31,10 +28,10 @@ type suggestionView struct { // out-of-library artists is permanently partial (#2376), and the card // falls back to the seed attribution it has always shown. MatchedTags []string `json:"matched_tags,omitempty"` - // ImageURL is resolved on-demand from Lidarr (out-of-library - // artists have no local art row). Omitted when Lidarr is disabled - // or has no match — the client falls back to a placeholder. Not - // cached: a remote URL Lidarr surfaced, fetched by the browser. + // ImageURL is Minstrel's catalogue art endpoint for the candidate + // (out-of-library artists have no local art row, M502 #5519). It + // answers 404 when no enabled source has an image, and the client + // falls back to a placeholder. ImageURL string `json:"image_url,omitempty"` } @@ -105,7 +102,7 @@ func (h *handlers) handleListSuggestions(w http.ResponseWriter, r *http.Request) MatchedTags: s.MatchedTags, }) } - h.resolveSuggestionArt(r.Context(), out) + resolveSuggestionArt(out) writeJSON(w, http.StatusOK, out) } @@ -250,48 +247,15 @@ func (h *handlers) handleListSuggestionSnoozes(w http.ResponseWriter, r *http.Re writeJSON(w, http.StatusOK, out) } -// resolveSuggestionArt fills ImageURL on-demand from Lidarr's artist -// lookup, matched by MBID (foreignArtistId). Best-effort and cache-free: -// Lidarr is the only source — when it's disabled, unreachable, or has -// no match for a candidate, that entry keeps an empty ImageURL and the -// client renders its placeholder. Lookups run with bounded concurrency -// so a full Discover page doesn't serialize ~12 round-trips. Never -// fails the request; the suggestions list is the contract, art is a -// nicety. -func (h *handlers) resolveSuggestionArt(ctx context.Context, views []suggestionView) { - if len(views) == 0 { - return - } - cfg, err := h.lidarrCfg.Get(ctx) - if err != nil || !cfg.Enabled || cfg.BaseURL == "" || cfg.APIKey == "" { - return // Lidarr off / unconfigured → placeholders only - } - client := lidarr.NewClient(cfg.BaseURL, cfg.APIKey) - - const maxConcurrent = 6 - sem := make(chan struct{}, maxConcurrent) - var wg sync.WaitGroup +// resolveSuggestionArt points each suggestion's ImageURL at Minstrel's +// catalogue art endpoint (M502 #5519). Nothing is fetched here: the image is +// looked up from the enabled cover-art sources when the browser asks for it, +// and kept, so listing suggestions makes no outbound call. A candidate with +// no MBID keeps an empty ImageURL and the client renders its placeholder. +func resolveSuggestionArt(views []suggestionView) { for i := range views { - if views[i].MBID == "" || views[i].Name == "" { - continue + if views[i].MBID != "" { + views[i].ImageURL = artistArtURL(views[i].MBID, views[i].Name) } - wg.Add(1) - sem <- struct{}{} - go func(idx int) { - defer wg.Done() - defer func() { <-sem }() - results, lerr := client.LookupArtist(ctx, views[idx].Name) - if lerr != nil { - return // best-effort: no art on lookup failure - } - for _, res := range results { - if res.MBID == views[idx].MBID && res.ImageURL != "" { - // Distinct slice index per goroutine → race-free. - views[idx].ImageURL = res.ImageURL - return - } - } - }(i) } - wg.Wait() } diff --git a/internal/catalogue/art.go b/internal/catalogue/art.go new file mode 100644 index 00000000..ca60066d --- /dev/null +++ b/internal/catalogue/art.go @@ -0,0 +1,214 @@ +package catalogue + +import ( + "context" + "errors" + "fmt" + "os" + "path/filepath" + "regexp" + "sync" + "time" + + "git.fabledsword.com/bvandeusen/minstrel/internal/coverart" +) + +// NoArtRetry is how long "no source has art for this" is believed before +// the sources are asked again. +const NoArtRetry = 7 * 24 * time.Hour + +// fetchTimeout bounds one look-up across every source (rule 156). +const fetchTimeout = 30 * time.Second + +// ErrNoArt: no enabled source has an image, or the sources could not be +// reached. The caller shows its placeholder. +var ErrNoArt = errors.New("catalogue: no art") + +var mbidPattern = regexp.MustCompile(`^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$`) + +// ValidMBID reports whether s is shaped like a MusicBrainz id. It is the +// guard that keeps a request path out of the cache directory's parents. +func ValidMBID(s string) bool { return mbidPattern.MatchString(s) } + +// Art serves images for artists and release groups that are not in the +// library (search results, Discover suggestions, wanted albums), from the +// operator's enabled cover-art sources. Each image is fetched once and kept +// under the data directory, so the browser only ever talks to Minstrel. +type Art struct { + dir string + artistSources func() []coverart.ArtistArtProvider + albumSources func() []coverart.AlbumCoverProvider + now func() time.Time + + mu sync.Mutex + inflight map[string]*artCall +} + +type artCall struct { + done chan struct{} + body []byte + err error +} + +// ArtSources lists the image sources to ask, in order. The cover-art +// settings service is one: it answers with what the operator has enabled. +type ArtSources interface { + EnabledArtistProviders() []coverart.ArtistArtProvider + EnabledAlbumProviders() []coverart.AlbumCoverProvider +} + +// NewArt keeps images under dataDir/catalogue-art and asks the given +// sources. Nil sources ask nothing, so every image is the placeholder. +func NewArt(dataDir string, sources ArtSources) *Art { + a := &Art{dir: filepath.Join(dataDir, "catalogue-art"), now: time.Now, inflight: map[string]*artCall{}} + if sources != nil { + a.artistSources = sources.EnabledArtistProviders + a.albumSources = sources.EnabledAlbumProviders + } + return a +} + +// Artist returns an image of the artist: the first enabled artist source +// with one wins, its thumbnail preferred over its backdrop. +func (a *Art) Artist(ctx context.Context, mbid, name string) ([]byte, error) { + return a.cached(ctx, "artist", mbid, func(ctx context.Context) ([]byte, error) { + if a.artistSources == nil { + return nil, coverart.ErrNotFound + } + ref := coverart.ArtistRef{MBID: mbid, Name: name} + return firstOf(a.artistSources(), func(p coverart.ArtistArtProvider) ([]byte, error) { + thumb, fanart, err := p.FetchArtistArt(ctx, ref) + if err == nil && len(thumb) == 0 { + thumb = fanart + } + return thumb, err + }) + }) +} + +// ReleaseGroup returns a cover for the release group: a source that can +// answer by group (the Cover Art Archive) is asked by its id, the rest by +// artist and title. +func (a *Art) ReleaseGroup(ctx context.Context, mbid, artist, title string) ([]byte, error) { + return a.cached(ctx, "release-group", mbid, func(ctx context.Context) ([]byte, error) { + if a.albumSources == nil { + return nil, coverart.ErrNotFound + } + byName := coverart.AlbumRef{ArtistName: artist, AlbumTitle: title} + return firstOf(a.albumSources(), func(p coverart.AlbumCoverProvider) ([]byte, error) { + if g, ok := p.(coverart.ReleaseGroupCoverProvider); ok { + return g.FetchReleaseGroupCover(ctx, mbid) + } + if artist == "" || title == "" { + return nil, coverart.ErrNotFound + } + return p.FetchAlbumCover(ctx, byName) + }) + }) +} + +// firstOf asks each source in turn. It reports ErrNotFound only when every +// source said so; if any could not be reached, the miss is transient. +func firstOf[P any](sources []P, ask func(P) ([]byte, error)) ([]byte, error) { + transient := false + for _, p := range sources { + body, err := ask(p) + if err == nil && len(body) > 0 { + return body, nil + } + if err != nil && !errors.Is(err, coverart.ErrNotFound) { + transient = true + } + } + if transient { + return nil, coverart.ErrTransient + } + return nil, coverart.ErrNotFound +} + +// cached serves kind/mbid from disk, or fetches it once however many +// requests ask at the same moment. A definite "nobody has one" is written +// down and believed for NoArtRetry; an unreachable source is not, so the +// next request asks again. +func (a *Art) cached(ctx context.Context, kind, mbid string, fetch func(context.Context) ([]byte, error)) ([]byte, error) { + if !ValidMBID(mbid) { + return nil, ErrNoArt + } + dir := filepath.Join(a.dir, kind) + imgPath := filepath.Join(dir, mbid) + nonePath := imgPath + ".none" + + if body, err := os.ReadFile(imgPath); err == nil && len(body) > 0 { + return body, nil + } + if fi, err := os.Stat(nonePath); err == nil && a.now().Sub(fi.ModTime()) < NoArtRetry { + return nil, ErrNoArt + } + + key := kind + "/" + mbid + a.mu.Lock() + if call, ok := a.inflight[key]; ok { + a.mu.Unlock() + select { + case <-call.done: + return call.body, call.err + case <-ctx.Done(): + return nil, ctx.Err() + } + } + call := &artCall{done: make(chan struct{})} + a.inflight[key] = call + a.mu.Unlock() + + defer func() { + a.mu.Lock() + delete(a.inflight, key) + a.mu.Unlock() + close(call.done) + }() + + // The fetch outlives a browser that navigates away: a second request is + // likely, and the image is worth keeping. + fctx, cancel := context.WithTimeout(context.WithoutCancel(ctx), fetchTimeout) + defer cancel() + body, err := fetch(fctx) + switch { + case err == nil && len(body) > 0: + if werr := writeFile(dir, imgPath, body); werr != nil { + // Serving it uncached is still better than a placeholder. + call.body = body + return body, nil + } + _ = os.Remove(nonePath) + call.body = body + return body, nil + case err == nil || errors.Is(err, coverart.ErrNotFound): + _ = writeFile(dir, nonePath, nil) + } + call.err = ErrNoArt + return nil, ErrNoArt +} + +func writeFile(dir, path string, body []byte) error { + if err := os.MkdirAll(dir, 0o755); err != nil { + return fmt.Errorf("catalogue art: %w", err) + } + tmp, err := os.CreateTemp(dir, ".part-*") + if err != nil { + return fmt.Errorf("catalogue art: %w", err) + } + if _, err := tmp.Write(body); err != nil { + _ = tmp.Close() + _ = os.Remove(tmp.Name()) + return fmt.Errorf("catalogue art: %w", err) + } + if err := tmp.Close(); err != nil { + _ = os.Remove(tmp.Name()) + return fmt.Errorf("catalogue art: %w", err) + } + if err := os.Rename(tmp.Name(), path); err != nil { + _ = os.Remove(tmp.Name()) + return fmt.Errorf("catalogue art: %w", err) + } + return nil +} diff --git a/internal/catalogue/art_test.go b/internal/catalogue/art_test.go new file mode 100644 index 00000000..70d099e9 --- /dev/null +++ b/internal/catalogue/art_test.go @@ -0,0 +1,165 @@ +package catalogue + +import ( + "context" + "errors" + "os" + "path/filepath" + "sync" + "sync/atomic" + "testing" + "time" + + "git.fabledsword.com/bvandeusen/minstrel/internal/coverart" +) + +const someMBID = "e21857d5-3256-4547-afb3-4b6ded592596" + +// fakeSource is an artist and album source answering with fixed results. +type fakeSource struct { + thumb, fanart, cover []byte + err error + calls atomic.Int32 + gate chan struct{} // when set, each fetch waits for it + lastRef coverart.AlbumRef + mu sync.Mutex +} + +func (f *fakeSource) ID() string { return "fake" } +func (f *fakeSource) DisplayName() string { return "Fake" } +func (f *fakeSource) RequiresAPIKey() bool { return false } +func (f *fakeSource) DefaultEnabled() bool { return true } +func (f *fakeSource) Configure(_ coverart.ProviderSettings) error { return nil } + +func (f *fakeSource) wait() { + f.calls.Add(1) + if f.gate != nil { + <-f.gate + } +} + +func (f *fakeSource) FetchArtistArt(_ context.Context, _ coverart.ArtistRef) ([]byte, []byte, error) { + f.wait() + return f.thumb, f.fanart, f.err +} + +func (f *fakeSource) FetchAlbumCover(_ context.Context, ref coverart.AlbumRef) ([]byte, error) { + f.wait() + f.mu.Lock() + f.lastRef = ref + f.mu.Unlock() + return f.cover, f.err +} + +// groupSource can also answer by release group, as the Cover Art Archive does. +type groupSource struct{ *fakeSource } + +func (g groupSource) FetchReleaseGroupCover(_ context.Context, _ string) ([]byte, error) { + g.wait() + return []byte("by-group"), nil +} + +type sources struct { + artists []coverart.ArtistArtProvider + albums []coverart.AlbumCoverProvider +} + +func (s sources) EnabledArtistProviders() []coverart.ArtistArtProvider { return s.artists } +func (s sources) EnabledAlbumProviders() []coverart.AlbumCoverProvider { return s.albums } + +func TestArt_FetchesOnceAndKeepsIt(t *testing.T) { + src := &fakeSource{fanart: []byte("backdrop")} + a := NewArt(t.TempDir(), sources{artists: []coverart.ArtistArtProvider{src}}) + + body, err := a.Artist(context.Background(), someMBID, "Gorillaz") + if err != nil || string(body) != "backdrop" { + t.Fatalf("with only a backdrop, use it: %q, %v", body, err) + } + if body, _ := a.Artist(context.Background(), someMBID, "Gorillaz"); string(body) != "backdrop" || src.calls.Load() != 1 { + t.Errorf("the second ask should come from disk: %q after %d calls", body, src.calls.Load()) + } +} + +func TestArt_NobodyHasOneIsBelievedForAWeek(t *testing.T) { + src := &fakeSource{err: coverart.ErrNotFound} + a := NewArt(t.TempDir(), sources{artists: []coverart.ArtistArtProvider{src}}) + + if _, err := a.Artist(context.Background(), someMBID, "x"); !errors.Is(err, ErrNoArt) { + t.Fatalf("err = %v, want ErrNoArt", err) + } + if _, err := a.Artist(context.Background(), someMBID, "x"); !errors.Is(err, ErrNoArt) || src.calls.Load() != 1 { + t.Errorf("a definite miss is remembered: %d calls", src.calls.Load()) + } + a.now = func() time.Time { return time.Now().Add(NoArtRetry + time.Hour) } + src.err, src.thumb = nil, []byte("thumb") + if body, err := a.Artist(context.Background(), someMBID, "x"); err != nil || string(body) != "thumb" { + t.Errorf("after a week the sources are asked again: %q, %v", body, err) + } +} + +func TestArt_AnUnreachableSourceIsAskedAgain(t *testing.T) { + src := &fakeSource{err: coverart.ErrTransient} + a := NewArt(t.TempDir(), sources{artists: []coverart.ArtistArtProvider{src}}) + for range 2 { + if _, err := a.Artist(context.Background(), someMBID, "x"); !errors.Is(err, ErrNoArt) { + t.Fatalf("err = %v", err) + } + } + if src.calls.Load() != 2 { + t.Errorf("calls = %d: an outage must not be written down as 'no art'", src.calls.Load()) + } +} + +func TestArt_ReleaseGroupAsksByGroupThenByName(t *testing.T) { + byName := &fakeSource{cover: []byte("by-name")} + caa := groupSource{&fakeSource{}} + a := NewArt(t.TempDir(), sources{albums: []coverart.AlbumCoverProvider{caa, byName}}) + if body, _ := a.ReleaseGroup(context.Background(), someMBID, "Gorillaz", "Demon Days"); string(body) != "by-group" { + t.Errorf("a source that answers by group is asked by group: %q", body) + } + + other := "0c4d5b5f-2f3c-3d31-9c4e-4b4f6b4c1a01" + a = NewArt(t.TempDir(), sources{albums: []coverart.AlbumCoverProvider{byName}}) + if body, _ := a.ReleaseGroup(context.Background(), other, "Gorillaz", "Demon Days"); string(body) != "by-name" { + t.Errorf("by name: %q", body) + } + if byName.lastRef.MBID != "" || byName.lastRef.AlbumTitle != "Demon Days" { + t.Errorf("a release-group id is not a release id, so it is not passed as one: %+v", byName.lastRef) + } +} + +func TestArt_ConcurrentAsksShareOneFetch(t *testing.T) { + src := &fakeSource{thumb: []byte("t"), gate: make(chan struct{})} + a := NewArt(t.TempDir(), sources{artists: []coverart.ArtistArtProvider{src}}) + var wg sync.WaitGroup + for range 5 { + wg.Add(1) + go func() { + defer wg.Done() + if body, err := a.Artist(context.Background(), someMBID, "x"); err != nil || string(body) != "t" { + t.Errorf("%q, %v", body, err) + } + }() + } + // Let the fetch through once every caller has had time to queue on it. + time.Sleep(50 * time.Millisecond) + close(src.gate) + wg.Wait() + if src.calls.Load() != 1 { + t.Errorf("calls = %d, want one fetch for five asks", src.calls.Load()) + } +} + +func TestArt_RefusesWhatIsNotAnMBID(t *testing.T) { + dir := t.TempDir() + a := NewArt(dir, sources{artists: []coverart.ArtistArtProvider{&fakeSource{thumb: []byte("t")}}}) + if _, err := a.Artist(context.Background(), "../../etc/passwd", "x"); !errors.Is(err, ErrNoArt) { + t.Fatalf("err = %v", err) + } + if _, err := os.Stat(filepath.Join(dir, "catalogue-art")); !os.IsNotExist(err) { + t.Error("nothing should be written for a refused id") + } + if _, err := NewArt(dir, nil).Artist(context.Background(), someMBID, "x"); !errors.Is(err, ErrNoArt) { + t.Errorf("no sources: err = %v", err) + } +} diff --git a/internal/catalogue/catalogue.go b/internal/catalogue/catalogue.go new file mode 100644 index 00000000..9084f9dd --- /dev/null +++ b/internal/catalogue/catalogue.go @@ -0,0 +1,311 @@ +// Package catalogue is Minstrel's own record of what artists have released, +// read from MusicBrainz and kept in the database (M502 #5519). The wanted +// list, search and import read it; MusicBrainz is asked only when what is +// held is missing or old. +package catalogue + +import ( + "context" + "errors" + "fmt" + "log/slog" + "time" + + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgtype" + "github.com/jackc/pgx/v5/pgxpool" + + "git.fabledsword.com/bvandeusen/minstrel/internal/db/dbq" + "git.fabledsword.com/bvandeusen/minstrel/internal/musicbrainz" +) + +const ( + // DiscographyMaxAge is how long an artist's list of release groups is + // trusted before it is read again. New releases appear on this cadence. + DiscographyMaxAge = 7 * 24 * time.Hour + // ReleasesMaxAge is how long a release group's editions and track lists + // are trusted. They change rarely once a group is out. + ReleasesMaxAge = 30 * 24 * time.Hour +) + +// ErrNotFound: MusicBrainz has no artist or release group under that id. +var ErrNotFound = errors.New("catalogue: not in MusicBrainz") + +// Service reads and refreshes the catalogue. +type Service struct { + pool *pgxpool.Pool + mb *musicbrainz.Client + logger *slog.Logger + now func() time.Time +} + +// New makes a Service. A nil client talks to the public MusicBrainz. +func New(pool *pgxpool.Pool, mb *musicbrainz.Client, logger *slog.Logger) *Service { + if mb == nil { + mb = &musicbrainz.Client{} + } + if logger == nil { + logger = slog.Default() + } + return &Service{pool: pool, mb: mb, logger: logger, now: time.Now} +} + +// MusicBrainz is the client the service asks, for callers that search. +func (s *Service) MusicBrainz() *musicbrainz.Client { return s.mb } + +// Discography is an artist and the release groups credited to them. +type Discography struct { + Artist dbq.MbArtist + Groups []dbq.MbReleaseGroup + // Stale: MusicBrainz could not be asked, so this is the last copy read. + Stale bool +} + +// Discography returns the artist's release groups, reading them from +// MusicBrainz first when they have never been read or are older than +// DiscographyMaxAge. When MusicBrainz is unreachable, a copy already held is +// returned marked Stale; with nothing held, the error is returned. +func (s *Service) Discography(ctx context.Context, artistMBID string) (Discography, error) { + q := dbq.New(s.pool) + artist, err := q.GetMBArtist(ctx, artistMBID) + held := err == nil && artist.DiscographyRefreshedAt.Valid + if err != nil && !errors.Is(err, pgx.ErrNoRows) { + return Discography{}, fmt.Errorf("catalogue: read artist: %w", err) + } + stale := false + if !held || s.now().Sub(artist.DiscographyRefreshedAt.Time) > DiscographyMaxAge { + if rerr := s.RefreshArtist(ctx, artistMBID); rerr != nil { + if !held || !errors.Is(rerr, musicbrainz.ErrUnreachable) { + return Discography{}, rerr + } + stale = true + } + if artist, err = q.GetMBArtist(ctx, artistMBID); err != nil { + return Discography{}, fmt.Errorf("catalogue: read artist: %w", err) + } + } + groups, err := q.ListMBArtistReleaseGroups(ctx, artistMBID) + if err != nil { + return Discography{}, fmt.Errorf("catalogue: read discography: %w", err) + } + return Discography{Artist: artist, Groups: groups, Stale: stale}, nil +} + +// RefreshArtist reads the artist and every release group credited to them, +// and replaces the held discography in one transaction. +func (s *Service) RefreshArtist(ctx context.Context, artistMBID string) error { + artist, err := s.mb.Artist(ctx, artistMBID) + if err != nil { + return mbErr(err) + } + groups, err := s.mb.ArtistReleaseGroups(ctx, artistMBID) + if err != nil { + return mbErr(err) + } + return pgx.BeginFunc(ctx, s.pool, func(tx pgx.Tx) error { + q := dbq.New(tx) + if err := q.UpsertMBArtist(ctx, dbq.UpsertMBArtistParams{ + Mbid: artist.MBID, Name: artist.Name, SortName: artist.SortName, + Disambiguation: artist.Disambiguation, ArtistType: artist.Type, Country: artist.Country, + }); err != nil { + return fmt.Errorf("catalogue: save artist: %w", err) + } + if err := q.DeleteMBArtistReleaseGroups(ctx, artist.MBID); err != nil { + return fmt.Errorf("catalogue: clear discography: %w", err) + } + for _, g := range groups { + if err := upsertGroup(ctx, q, g); err != nil { + return err + } + if err := q.AddMBArtistReleaseGroup(ctx, dbq.AddMBArtistReleaseGroupParams{ + ArtistMbid: artist.MBID, ReleaseGroupMbid: g.MBID, + }); err != nil { + return fmt.Errorf("catalogue: link release group: %w", err) + } + } + return q.MarkMBArtistDiscographyRefreshed(ctx, artist.MBID) + }) +} + +func upsertGroup(ctx context.Context, q *dbq.Queries, g musicbrainz.ReleaseGroup) error { + secondary := g.SecondaryTypes + if secondary == nil { + secondary = []string{} + } + if err := q.UpsertMBReleaseGroup(ctx, dbq.UpsertMBReleaseGroupParams{ + Mbid: g.MBID, Title: g.Title, PrimaryType: g.PrimaryType, SecondaryTypes: secondary, + FirstReleaseDate: g.FirstReleaseDate, ArtistCredit: g.ArtistCredit, + }); err != nil { + return fmt.Errorf("catalogue: save release group: %w", err) + } + return nil +} + +// Edition is one release of a group with its track list. +type Edition struct { + Release dbq.MbRelease + Tracks []dbq.MbReleaseTrack +} + +// Editions is a release group and its releases. +type Editions struct { + Group dbq.MbReleaseGroup + Editions []Edition + Stale bool +} + +// Releases returns a release group's releases with their track lists, read +// from MusicBrainz first when never read or older than ReleasesMaxAge. Like +// Discography, an unreachable MusicBrainz serves the held copy as Stale. +func (s *Service) Releases(ctx context.Context, groupMBID string) (Editions, error) { + q := dbq.New(s.pool) + group, err := q.GetMBReleaseGroup(ctx, groupMBID) + if err != nil && !errors.Is(err, pgx.ErrNoRows) { + return Editions{}, fmt.Errorf("catalogue: read release group: %w", err) + } + held := err == nil && group.ReleasesRefreshedAt.Valid + stale := false + if !held || s.now().Sub(group.ReleasesRefreshedAt.Time) > ReleasesMaxAge { + if rerr := s.RefreshReleaseGroup(ctx, groupMBID); rerr != nil { + if !held || !errors.Is(rerr, musicbrainz.ErrUnreachable) { + return Editions{}, rerr + } + stale = true + } + if group, err = q.GetMBReleaseGroup(ctx, groupMBID); err != nil { + return Editions{}, fmt.Errorf("catalogue: read release group: %w", err) + } + } + releases, err := q.ListMBReleasesForGroup(ctx, groupMBID) + if err != nil { + return Editions{}, fmt.Errorf("catalogue: read releases: %w", err) + } + tracks, err := q.ListMBReleaseTracksForGroup(ctx, groupMBID) + if err != nil { + return Editions{}, fmt.Errorf("catalogue: read track lists: %w", err) + } + byRelease := make(map[string][]dbq.MbReleaseTrack, len(releases)) + for _, t := range tracks { + byRelease[t.ReleaseMbid] = append(byRelease[t.ReleaseMbid], t) + } + out := Editions{Group: group, Stale: stale} + for _, r := range releases { + out.Editions = append(out.Editions, Edition{Release: r, Tracks: byRelease[r.Mbid]}) + } + return out, nil +} + +// RefreshReleaseGroup reads a group's releases and track lists and replaces +// the held copy in one transaction. A group not yet held is looked up first. +func (s *Service) RefreshReleaseGroup(ctx context.Context, groupMBID string) error { + q := dbq.New(s.pool) + var group *musicbrainz.ReleaseGroup + if _, err := q.GetMBReleaseGroup(ctx, groupMBID); errors.Is(err, pgx.ErrNoRows) { + g, gerr := s.mb.ReleaseGroup(ctx, groupMBID) + if gerr != nil { + return mbErr(gerr) + } + group = &g + } else if err != nil { + return fmt.Errorf("catalogue: read release group: %w", err) + } + releases, err := s.mb.ReleaseGroupReleases(ctx, groupMBID) + if err != nil { + return mbErr(err) + } + return pgx.BeginFunc(ctx, s.pool, func(tx pgx.Tx) error { + tq := dbq.New(tx) + if group != nil { + if err := upsertGroup(ctx, tq, *group); err != nil { + return err + } + } + if err := tq.DeleteMBReleasesForGroup(ctx, groupMBID); err != nil { + return fmt.Errorf("catalogue: clear releases: %w", err) + } + for _, r := range releases { + if err := tq.InsertMBRelease(ctx, dbq.InsertMBReleaseParams{ + Mbid: r.MBID, ReleaseGroupMbid: groupMBID, Title: r.Title, Status: r.Status, + ReleaseDate: r.Date, Country: r.Country, Disambiguation: r.Disambiguation, + Formats: r.Formats(), TrackCount: int32(r.TrackCount()), + }); err != nil { + return fmt.Errorf("catalogue: save release: %w", err) + } + for _, m := range r.Media { + for _, t := range m.Tracks { + if err := tq.InsertMBReleaseTrack(ctx, dbq.InsertMBReleaseTrackParams{ + ReleaseMbid: r.MBID, MediumPosition: int32(m.Position), Position: int32(t.Position), + MediumFormat: m.Format, Number: t.Number, Title: t.Title, + LengthMs: int32(t.LengthMs), RecordingMbid: t.RecordingMBID, + }); err != nil { + return fmt.Errorf("catalogue: save track: %w", err) + } + } + } + } + return tq.MarkMBReleaseGroupReleasesRefreshed(ctx, groupMBID) + }) +} + +// RefreshDue re-reads up to limit discographies that are older than +// DiscographyMaxAge, never-read first. It stops at the first unreachable +// MusicBrainz, since the rest would fail the same way. +func (s *Service) RefreshDue(ctx context.Context, limit int) (refreshed int, err error) { + due, err := dbq.New(s.pool).ListMBArtistsDueForRefresh(ctx, dbq.ListMBArtistsDueForRefreshParams{ + Before: pgtype.Timestamptz{Time: s.now().Add(-DiscographyMaxAge), Valid: true}, + MaxRows: int32(limit), + }) + if err != nil { + return 0, fmt.Errorf("catalogue: list due artists: %w", err) + } + for _, mbid := range due { + if err := s.RefreshArtist(ctx, mbid); err != nil { + if errors.Is(err, musicbrainz.ErrUnreachable) || ctx.Err() != nil { + return refreshed, err + } + // One artist MusicBrainz cannot describe (merged away, say) + // must not stop the rest. + s.logger.Warn("catalogue: refresh artist", "mbid", mbid, "err", err) + continue + } + refreshed++ + } + return refreshed, nil +} + +// RefreshInterval is how often Run looks for discographies due a re-read, +// and RefreshBatch how many it reads per look: 20 artists is about a minute +// of MusicBrainz's budget. +const ( + RefreshInterval = time.Hour + RefreshBatch = 20 +) + +// Run refreshes due discographies every RefreshInterval until ctx ends. +func (s *Service) Run(ctx context.Context) { + ticker := time.NewTicker(RefreshInterval) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + } + n, err := s.RefreshDue(ctx, RefreshBatch) + switch { + case err != nil && ctx.Err() == nil: + s.logger.Warn("catalogue: refresh pass stopped", "refreshed", n, "err", err) + case n > 0: + s.logger.Info("catalogue: refresh pass complete", "refreshed", n) + } + } +} + +// mbErr maps MusicBrainz's not-found to the catalogue's; anything else passes +// through so callers can tell an unreachable MusicBrainz apart. +func mbErr(err error) error { + if errors.Is(err, musicbrainz.ErrNotFound) { + return fmt.Errorf("%w: %v", ErrNotFound, err) + } + return err +} diff --git a/internal/catalogue/catalogue_test.go b/internal/catalogue/catalogue_test.go new file mode 100644 index 00000000..5b495122 --- /dev/null +++ b/internal/catalogue/catalogue_test.go @@ -0,0 +1,247 @@ +package catalogue + +import ( + "context" + "errors" + "fmt" + "io" + "log/slog" + "net/http" + "net/http/httptest" + "os" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/jackc/pgx/v5/pgxpool" + + "git.fabledsword.com/bvandeusen/minstrel/internal/db" + "git.fabledsword.com/bvandeusen/minstrel/internal/dbtest" + "git.fabledsword.com/bvandeusen/minstrel/internal/musicbrainz" +) + +func newPool(t *testing.T) *pgxpool.Pool { + t.Helper() + if testing.Short() { + t.Skip("skipping integration test in -short mode") + } + dsn := os.Getenv("MINSTREL_TEST_DATABASE_URL") + if dsn == "" { + t.Skip("MINSTREL_TEST_DATABASE_URL not set") + } + if err := db.Migrate(dsn, slog.New(slog.NewTextHandler(io.Discard, nil))); err != nil { + t.Fatalf("migrate: %v", err) + } + pool, err := pgxpool.New(context.Background(), dsn) + if err != nil { + t.Fatalf("pool: %v", err) + } + t.Cleanup(pool.Close) + dbtest.ResetDB(t, pool) + return pool +} + +const ( + gorillaz = "e21857d5-3256-4547-afb3-4b6ded592596" + deLaSoul = "4f2a6b9e-9ae6-4f34-9a41-3e9f2f7c4c1d" + demonDays = "0c4d5b5f-2f3c-3d31-9c4e-4b4f6b4c1a01" + feelGood = "0c4d5b5f-2f3c-3d31-9c4e-4b4f6b4c1a02" // credited to both + demonDaysR = "1a2b3c4d-0000-4000-8000-000000000001" + demonDaysJ = "1a2b3c4d-0000-4000-8000-000000000002" +) + +// fakeMB is a MusicBrainz with two artists, a collaboration between them, +// and one album with two editions. down makes every request a 503. +type fakeMB struct { + srv *httptest.Server + calls atomic.Int32 + down atomic.Bool +} + +func newFakeMB(t *testing.T) *fakeMB { + t.Helper() + f := &fakeMB{} + f.srv = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + f.calls.Add(1) + if f.down.Load() { + w.WriteHeader(http.StatusServiceUnavailable) + return + } + q := r.URL.Query() + switch { + case r.URL.Path == "/artist/"+gorillaz: + _, _ = fmt.Fprintf(w, `{"id": %q, "name": "Gorillaz", "sort-name": "Gorillaz", "type": "Group", "country": "GB"}`, gorillaz) + case r.URL.Path == "/artist/"+deLaSoul: + _, _ = fmt.Fprintf(w, `{"id": %q, "name": "De La Soul", "type": "Group", "country": "US"}`, deLaSoul) + case r.URL.Path == "/release-group" && q.Get("artist") == gorillaz: + _, _ = fmt.Fprintf(w, `{"release-group-count": 2, "release-groups": [ + {"id": %q, "title": "Demon Days", "primary-type": "Album", "first-release-date": "2005-05-11", + "artist-credit": [{"name": "Gorillaz", "artist": {"id": %q}}]}, + {"id": %q, "title": "Feel Good Inc.", "primary-type": "Single", "secondary-types": [], "first-release-date": "2005-05-09", + "artist-credit": [{"name": "Gorillaz", "joinphrase": " feat. ", "artist": {"id": %q}}, {"name": "De La Soul", "artist": {"id": %q}}]} + ]}`, demonDays, gorillaz, feelGood, gorillaz, deLaSoul) + case r.URL.Path == "/release-group" && q.Get("artist") == deLaSoul: + _, _ = fmt.Fprintf(w, `{"release-group-count": 1, "release-groups": [ + {"id": %q, "title": "Feel Good Inc.", "primary-type": "Single", "first-release-date": "2005-05-09"} + ]}`, feelGood) + case r.URL.Path == "/release-group/"+demonDays: + _, _ = fmt.Fprintf(w, `{"id": %q, "title": "Demon Days", "primary-type": "Album", "first-release-date": "2005-05-11", + "artist-credit": [{"name": "Gorillaz", "artist": {"id": %q}}]}`, demonDays, gorillaz) + case r.URL.Path == "/release" && q.Get("release-group") == demonDays: + _, _ = fmt.Fprintf(w, `{"release-count": 2, "releases": [ + {"id": %q, "title": "Demon Days", "status": "Promotion", "date": "2005-04", "country": "JP", + "media": [{"position": 1, "format": "CD", "tracks": [ + {"position": 1, "number": "1", "title": "Intro", "length": 63000, "recording": {"id": "rec-intro"}}]}]}, + {"id": %q, "title": "Demon Days", "status": "Official", "date": "2005-05-11", "country": "GB", + "media": [{"position": 1, "format": "CD", "tracks": [ + {"position": 1, "number": "1", "title": "Intro", "length": 63000, "recording": {"id": "rec-intro"}}, + {"position": 2, "number": "2", "title": "Last Living Souls", "length": 190000, "recording": {"id": "rec-lls"}}]}]} + ]}`, demonDaysJ, demonDaysR) + default: + w.WriteHeader(http.StatusNotFound) + } + })) + t.Cleanup(f.srv.Close) + return f +} + +func (f *fakeMB) client() *musicbrainz.Client { + return &musicbrainz.Client{BaseURL: f.srv.URL, Pace: &musicbrainz.Pacer{}, Retries: -1} +} + +func TestDiscography_ReadsOnceThenServesTheHeldCopy_Integration(t *testing.T) { + ctx := context.Background() + mb := newFakeMB(t) + s := New(newPool(t), mb.client(), nil) + + d, err := s.Discography(ctx, gorillaz) + if err != nil { + t.Fatal(err) + } + if d.Artist.Name != "Gorillaz" || d.Artist.Country != "GB" || len(d.Groups) != 2 || d.Stale { + t.Fatalf("discography = %+v", d) + } + // Oldest first: the single came out two days before the album. + if d.Groups[0].Mbid != feelGood || d.Groups[1].PrimaryType != "Album" { + t.Errorf("order = %s, %s", d.Groups[0].Title, d.Groups[1].Title) + } + if d.Groups[0].ArtistCredit != "Gorillaz feat. De La Soul" { + t.Errorf("credit = %q", d.Groups[0].ArtistCredit) + } + + before := mb.calls.Load() + if _, err := s.Discography(ctx, gorillaz); err != nil { + t.Fatal(err) + } + if mb.calls.Load() != before { + t.Error("a fresh discography should be served without asking MusicBrainz") + } + + // The collaboration belongs to both artists. + dls, err := s.Discography(ctx, deLaSoul) + if err != nil { + t.Fatal(err) + } + if len(dls.Groups) != 1 || dls.Groups[0].Mbid != feelGood { + t.Errorf("De La Soul's discography = %+v", dls.Groups) + } + if again, _ := s.Discography(ctx, gorillaz); len(again.Groups) != 2 { + t.Error("refreshing one artist must not take a shared group from the other") + } +} + +func TestDiscography_UnreachableServesStaleOrFails_Integration(t *testing.T) { + ctx := context.Background() + mb := newFakeMB(t) + pool := newPool(t) + s := New(pool, mb.client(), nil) + + mb.down.Store(true) + if _, err := s.Discography(ctx, gorillaz); !errors.Is(err, musicbrainz.ErrUnreachable) { + t.Fatalf("nothing held and MusicBrainz down: err = %v, want ErrUnreachable", err) + } + + mb.down.Store(false) + if _, err := s.Discography(ctx, gorillaz); err != nil { + t.Fatal(err) + } + // A week and a day later, MusicBrainz is down: the held copy is served. + s.now = func() time.Time { return time.Now().Add(DiscographyMaxAge + 24*time.Hour) } + mb.down.Store(true) + d, err := s.Discography(ctx, gorillaz) + if err != nil || !d.Stale || len(d.Groups) != 2 { + t.Fatalf("stale copy: %+v, %v", d, err) + } +} + +func TestDiscography_UnknownArtistIsNotFound_Integration(t *testing.T) { + s := New(newPool(t), newFakeMB(t).client(), nil) + if _, err := s.Discography(context.Background(), "00000000-0000-4000-8000-000000000000"); !errors.Is(err, ErrNotFound) { + t.Fatalf("err = %v, want ErrNotFound", err) + } +} + +func TestReleases_ReadsEditionsAndTrackLists_Integration(t *testing.T) { + ctx := context.Background() + mb := newFakeMB(t) + s := New(newPool(t), mb.client(), nil) + + // The group isn't held yet: it is looked up first. + e, err := s.Releases(ctx, demonDays) + if err != nil { + t.Fatal(err) + } + if e.Group.Title != "Demon Days" || len(e.Editions) != 2 { + t.Fatalf("editions = %+v", e) + } + official := e.Editions[0] + if official.Release.Mbid != demonDaysR || official.Release.TrackCount != 2 || official.Release.Formats != "CD" { + t.Errorf("the official edition should come first: %+v", official.Release) + } + if len(official.Tracks) != 2 || official.Tracks[1].RecordingMbid != "rec-lls" || official.Tracks[1].LengthMs != 190000 { + t.Errorf("tracks = %+v", official.Tracks) + } + + before := mb.calls.Load() + if _, err := s.Releases(ctx, demonDays); err != nil { + t.Fatal(err) + } + if mb.calls.Load() != before { + t.Error("fresh editions should be served without asking MusicBrainz") + } + // Refreshing replaces rather than adds. + if err := s.RefreshReleaseGroup(ctx, demonDays); err != nil { + t.Fatal(err) + } + if again, _ := s.Releases(ctx, demonDays); len(again.Editions) != 2 || len(again.Editions[0].Tracks) != 2 { + t.Errorf("after a refresh: %+v", again) + } +} + +func TestRefreshDue_ReadsWhatIsOldAndStopsWhenMusicBrainzIsDown_Integration(t *testing.T) { + ctx := context.Background() + mb := newFakeMB(t) + s := New(newPool(t), mb.client(), nil) + for _, a := range []string{gorillaz, deLaSoul} { + if err := s.RefreshArtist(ctx, a); err != nil { + t.Fatal(err) + } + } + if n, err := s.RefreshDue(ctx, 10); err != nil || n != 0 { + t.Fatalf("nothing is due yet: %d, %v", n, err) + } + + s.now = func() time.Time { return time.Now().Add(DiscographyMaxAge + time.Hour) } + if n, err := s.RefreshDue(ctx, 1); err != nil || n != 1 { + t.Errorf("one due, limit one: %d, %v", n, err) + } + mb.down.Store(true) + n, err := s.RefreshDue(ctx, 10) + if !errors.Is(err, musicbrainz.ErrUnreachable) || n != 0 { + t.Errorf("MusicBrainz down: %d, %v", n, err) + } + if !strings.Contains(err.Error(), "503") { + t.Errorf("the error should say what MusicBrainz answered: %v", err) + } +} diff --git a/internal/coverart/provider.go b/internal/coverart/provider.go index 92303ece..122f02dd 100644 --- a/internal/coverart/provider.go +++ b/internal/coverart/provider.go @@ -82,6 +82,14 @@ type AlbumCoverProvider interface { FetchAlbumCover(ctx context.Context, ref AlbumRef) ([]byte, error) } +// ReleaseGroupCoverProvider is an opt-in capability for sources that can +// answer by MusicBrainz release group, which is how the release catalogue +// names an album before any edition of it is held (M502 #5519). +type ReleaseGroupCoverProvider interface { + AlbumCoverProvider + FetchReleaseGroupCover(ctx context.Context, groupMBID string) ([]byte, error) +} + // ArtistArtProvider is an opt-in capability: providers implementing // it can fetch artist images. Returns thumb + fanart atomically at // the JSON metadata step (one round-trip yields both URLs); the two diff --git a/internal/coverart/provider_mbcaa.go b/internal/coverart/provider_mbcaa.go index a448555d..35975f5d 100644 --- a/internal/coverart/provider_mbcaa.go +++ b/internal/coverart/provider_mbcaa.go @@ -92,10 +92,22 @@ func (p *mbcaaProvider) Configure(s ProviderSettings) error { // release MBID. MBCAA is MBID-only; returns ErrNotFound when // ref.MBID is empty. Returns ErrNotFound on 404 or ErrTransient otherwise. func (p *mbcaaProvider) FetchAlbumCover(ctx context.Context, ref AlbumRef) ([]byte, error) { + return p.fetchFront(ctx, "release", ref.MBID) +} + +// FetchReleaseGroupCover retrieves the 500px front cover the Cover Art +// Archive chose for a release group: what a search result or a wanted album +// shows before any edition of it is held (M502 #5519). +func (p *mbcaaProvider) FetchReleaseGroupCover(ctx context.Context, groupMBID string) ([]byte, error) { + return p.fetchFront(ctx, "release-group", groupMBID) +} + +// fetchFront fetches /{entity}/{mbid}/front-500, honouring the rate limit. +func (p *mbcaaProvider) fetchFront(ctx context.Context, entity, mbid string) ([]byte, error) { if !p.enabled.Load() { return nil, ErrNotFound // defensive; enricher already filters on enabled } - if ref.MBID == "" { + if mbid == "" { return nil, ErrNotFound // MBCAA is MBID-only } @@ -119,7 +131,7 @@ func (p *mbcaaProvider) FetchAlbumCover(ctx context.Context, ref AlbumRef) ([]by p.lastCall = time.Now() p.mu.Unlock() - url := fmt.Sprintf("%s/release/%s/front-500", cfg.BaseURL, ref.MBID) + url := fmt.Sprintf("%s/%s/%s/front-500", cfg.BaseURL, entity, mbid) req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) if err != nil { return nil, fmt.Errorf("coverart: build request: %w", err) @@ -157,3 +169,5 @@ func (p *mbcaaProvider) TestConnection(ctx context.Context) error { } return err } + +var _ ReleaseGroupCoverProvider = (*mbcaaProvider)(nil) diff --git a/internal/coverart/provider_mbcaa_test.go b/internal/coverart/provider_mbcaa_test.go index b837617e..18090712 100644 --- a/internal/coverart/provider_mbcaa_test.go +++ b/internal/coverart/provider_mbcaa_test.go @@ -166,3 +166,24 @@ func TestNewMBCAAProviderFromConfig_ReplacesRegisteredConfig(t *testing.T) { t.Errorf("cfg.UserAgent = %q, want %q", gotUA, "Minstrel/test-replaced (x)") } } + +func TestMBCAAProvider_FetchReleaseGroupCover_AsksByGroup(t *testing.T) { + var path string + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + path = r.URL.Path + _, _ = w.Write([]byte("jpeg")) + })) + defer srv.Close() + p := newMBCAAProviderForTest(t, srv.URL) + + body, err := p.FetchReleaseGroupCover(context.Background(), "rg-1") + if err != nil || string(body) != "jpeg" { + t.Fatalf("body %q, err %v", body, err) + } + if path != "/release-group/rg-1/front-500" { + t.Errorf("path = %q", path) + } + if _, err := p.FetchReleaseGroupCover(context.Background(), ""); !errors.Is(err, ErrNotFound) { + t.Errorf("no id: err = %v, want ErrNotFound", err) + } +} diff --git a/internal/db/dbq/catalogue.sql.go b/internal/db/dbq/catalogue.sql.go new file mode 100644 index 00000000..d18a4261 --- /dev/null +++ b/internal/db/dbq/catalogue.sql.go @@ -0,0 +1,402 @@ +// Code generated by sqlc. DO NOT EDIT. +// versions: +// sqlc v1.31.1 +// source: catalogue.sql + +package dbq + +import ( + "context" + + "github.com/jackc/pgx/v5/pgtype" +) + +const addMBArtistReleaseGroup = `-- name: AddMBArtistReleaseGroup :exec +INSERT INTO mb_artist_release_groups (artist_mbid, release_group_mbid) +VALUES ($1, $2) +ON CONFLICT DO NOTHING +` + +type AddMBArtistReleaseGroupParams struct { + ArtistMbid string + ReleaseGroupMbid string +} + +func (q *Queries) AddMBArtistReleaseGroup(ctx context.Context, arg AddMBArtistReleaseGroupParams) error { + _, err := q.db.Exec(ctx, addMBArtistReleaseGroup, arg.ArtistMbid, arg.ReleaseGroupMbid) + return err +} + +const deleteMBArtistReleaseGroups = `-- name: DeleteMBArtistReleaseGroups :exec +DELETE FROM mb_artist_release_groups WHERE artist_mbid = $1 +` + +func (q *Queries) DeleteMBArtistReleaseGroups(ctx context.Context, artistMbid string) error { + _, err := q.db.Exec(ctx, deleteMBArtistReleaseGroups, artistMbid) + return err +} + +const deleteMBReleasesForGroup = `-- name: DeleteMBReleasesForGroup :exec +DELETE FROM mb_releases WHERE release_group_mbid = $1 +` + +// Tracks go with their releases (ON DELETE CASCADE). +func (q *Queries) DeleteMBReleasesForGroup(ctx context.Context, releaseGroupMbid string) error { + _, err := q.db.Exec(ctx, deleteMBReleasesForGroup, releaseGroupMbid) + return err +} + +const getMBArtist = `-- name: GetMBArtist :one +SELECT mbid, name, sort_name, disambiguation, artist_type, country, discography_refreshed_at, created_at, updated_at FROM mb_artists WHERE mbid = $1 +` + +func (q *Queries) GetMBArtist(ctx context.Context, mbid string) (MbArtist, error) { + row := q.db.QueryRow(ctx, getMBArtist, mbid) + var i MbArtist + err := row.Scan( + &i.Mbid, + &i.Name, + &i.SortName, + &i.Disambiguation, + &i.ArtistType, + &i.Country, + &i.DiscographyRefreshedAt, + &i.CreatedAt, + &i.UpdatedAt, + ) + return i, err +} + +const getMBReleaseGroup = `-- name: GetMBReleaseGroup :one +SELECT mbid, title, primary_type, secondary_types, first_release_date, artist_credit, releases_refreshed_at, created_at, updated_at FROM mb_release_groups WHERE mbid = $1 +` + +func (q *Queries) GetMBReleaseGroup(ctx context.Context, mbid string) (MbReleaseGroup, error) { + row := q.db.QueryRow(ctx, getMBReleaseGroup, mbid) + var i MbReleaseGroup + err := row.Scan( + &i.Mbid, + &i.Title, + &i.PrimaryType, + &i.SecondaryTypes, + &i.FirstReleaseDate, + &i.ArtistCredit, + &i.ReleasesRefreshedAt, + &i.CreatedAt, + &i.UpdatedAt, + ) + return i, err +} + +const insertMBRelease = `-- name: InsertMBRelease :exec +INSERT INTO mb_releases (mbid, release_group_mbid, title, status, release_date, country, + disambiguation, formats, track_count) +VALUES ($1, $2, $3, $4, + $5, $6, $7, $8, + $9) +ON CONFLICT (mbid) DO UPDATE SET + release_group_mbid = EXCLUDED.release_group_mbid, + title = EXCLUDED.title, + status = EXCLUDED.status, + release_date = EXCLUDED.release_date, + country = EXCLUDED.country, + disambiguation = EXCLUDED.disambiguation, + formats = EXCLUDED.formats, + track_count = EXCLUDED.track_count +` + +type InsertMBReleaseParams struct { + Mbid string + ReleaseGroupMbid string + Title string + Status string + ReleaseDate string + Country string + Disambiguation string + Formats string + TrackCount int32 +} + +func (q *Queries) InsertMBRelease(ctx context.Context, arg InsertMBReleaseParams) error { + _, err := q.db.Exec(ctx, insertMBRelease, + arg.Mbid, + arg.ReleaseGroupMbid, + arg.Title, + arg.Status, + arg.ReleaseDate, + arg.Country, + arg.Disambiguation, + arg.Formats, + arg.TrackCount, + ) + return err +} + +const insertMBReleaseTrack = `-- name: InsertMBReleaseTrack :exec +INSERT INTO mb_release_tracks (release_mbid, medium_position, position, medium_format, number, + title, length_ms, recording_mbid) +VALUES ($1, $2, $3, + $4, $5, $6, $7, + $8) +` + +type InsertMBReleaseTrackParams struct { + ReleaseMbid string + MediumPosition int32 + Position int32 + MediumFormat string + Number string + Title string + LengthMs int32 + RecordingMbid string +} + +func (q *Queries) InsertMBReleaseTrack(ctx context.Context, arg InsertMBReleaseTrackParams) error { + _, err := q.db.Exec(ctx, insertMBReleaseTrack, + arg.ReleaseMbid, + arg.MediumPosition, + arg.Position, + arg.MediumFormat, + arg.Number, + arg.Title, + arg.LengthMs, + arg.RecordingMbid, + ) + return err +} + +const listMBArtistReleaseGroups = `-- name: ListMBArtistReleaseGroups :many +SELECT g.mbid, g.title, g.primary_type, g.secondary_types, g.first_release_date, g.artist_credit, g.releases_refreshed_at, g.created_at, g.updated_at FROM mb_release_groups g +JOIN mb_artist_release_groups ag ON ag.release_group_mbid = g.mbid +WHERE ag.artist_mbid = $1 +ORDER BY g.first_release_date = '', g.first_release_date, g.title, g.mbid +` + +// An artist's discography, oldest first; undated groups last. +func (q *Queries) ListMBArtistReleaseGroups(ctx context.Context, artistMbid string) ([]MbReleaseGroup, error) { + rows, err := q.db.Query(ctx, listMBArtistReleaseGroups, artistMbid) + if err != nil { + return nil, err + } + defer rows.Close() + var items []MbReleaseGroup + for rows.Next() { + var i MbReleaseGroup + if err := rows.Scan( + &i.Mbid, + &i.Title, + &i.PrimaryType, + &i.SecondaryTypes, + &i.FirstReleaseDate, + &i.ArtistCredit, + &i.ReleasesRefreshedAt, + &i.CreatedAt, + &i.UpdatedAt, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const listMBArtistsDueForRefresh = `-- name: ListMBArtistsDueForRefresh :many +SELECT mbid FROM mb_artists +WHERE discography_refreshed_at IS NULL OR discography_refreshed_at < $1 +ORDER BY discography_refreshed_at NULLS FIRST, mbid +LIMIT $2 +` + +type ListMBArtistsDueForRefreshParams struct { + Before pgtype.Timestamptz + MaxRows int32 +} + +// Artists whose discography has never been read, or was read before the +// cutoff: never-read first, then oldest first. +func (q *Queries) ListMBArtistsDueForRefresh(ctx context.Context, arg ListMBArtistsDueForRefreshParams) ([]string, error) { + rows, err := q.db.Query(ctx, listMBArtistsDueForRefresh, arg.Before, arg.MaxRows) + if err != nil { + return nil, err + } + defer rows.Close() + var items []string + for rows.Next() { + var mbid string + if err := rows.Scan(&mbid); err != nil { + return nil, err + } + items = append(items, mbid) + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const listMBReleaseTracksForGroup = `-- name: ListMBReleaseTracksForGroup :many +SELECT t.release_mbid, t.medium_position, t.position, t.medium_format, t.number, t.title, t.length_ms, t.recording_mbid FROM mb_release_tracks t +JOIN mb_releases r ON r.mbid = t.release_mbid +WHERE r.release_group_mbid = $1 +ORDER BY t.release_mbid, t.medium_position, t.position +` + +func (q *Queries) ListMBReleaseTracksForGroup(ctx context.Context, releaseGroupMbid string) ([]MbReleaseTrack, error) { + rows, err := q.db.Query(ctx, listMBReleaseTracksForGroup, releaseGroupMbid) + if err != nil { + return nil, err + } + defer rows.Close() + var items []MbReleaseTrack + for rows.Next() { + var i MbReleaseTrack + if err := rows.Scan( + &i.ReleaseMbid, + &i.MediumPosition, + &i.Position, + &i.MediumFormat, + &i.Number, + &i.Title, + &i.LengthMs, + &i.RecordingMbid, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const listMBReleasesForGroup = `-- name: ListMBReleasesForGroup :many +SELECT mbid, release_group_mbid, title, status, release_date, country, disambiguation, formats, track_count, created_at FROM mb_releases +WHERE release_group_mbid = $1 +ORDER BY status <> 'Official', release_date = '', release_date, title, mbid +` + +// Official editions first, then by date; undated last. +func (q *Queries) ListMBReleasesForGroup(ctx context.Context, releaseGroupMbid string) ([]MbRelease, error) { + rows, err := q.db.Query(ctx, listMBReleasesForGroup, releaseGroupMbid) + if err != nil { + return nil, err + } + defer rows.Close() + var items []MbRelease + for rows.Next() { + var i MbRelease + if err := rows.Scan( + &i.Mbid, + &i.ReleaseGroupMbid, + &i.Title, + &i.Status, + &i.ReleaseDate, + &i.Country, + &i.Disambiguation, + &i.Formats, + &i.TrackCount, + &i.CreatedAt, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const markMBArtistDiscographyRefreshed = `-- name: MarkMBArtistDiscographyRefreshed :exec +UPDATE mb_artists SET discography_refreshed_at = now() WHERE mbid = $1 +` + +func (q *Queries) MarkMBArtistDiscographyRefreshed(ctx context.Context, mbid string) error { + _, err := q.db.Exec(ctx, markMBArtistDiscographyRefreshed, mbid) + return err +} + +const markMBReleaseGroupReleasesRefreshed = `-- name: MarkMBReleaseGroupReleasesRefreshed :exec +UPDATE mb_release_groups SET releases_refreshed_at = now() WHERE mbid = $1 +` + +func (q *Queries) MarkMBReleaseGroupReleasesRefreshed(ctx context.Context, mbid string) error { + _, err := q.db.Exec(ctx, markMBReleaseGroupReleasesRefreshed, mbid) + return err +} + +const upsertMBArtist = `-- name: UpsertMBArtist :exec + +INSERT INTO mb_artists (mbid, name, sort_name, disambiguation, artist_type, country) +VALUES ($1, $2, $3, $4, + $5, $6) +ON CONFLICT (mbid) DO UPDATE SET + name = EXCLUDED.name, + sort_name = EXCLUDED.sort_name, + disambiguation = EXCLUDED.disambiguation, + artist_type = EXCLUDED.artist_type, + country = EXCLUDED.country, + updated_at = now() +` + +type UpsertMBArtistParams struct { + Mbid string + Name string + SortName string + Disambiguation string + ArtistType string + Country string +} + +// The release catalogue read from MusicBrainz (M502 #5519). +func (q *Queries) UpsertMBArtist(ctx context.Context, arg UpsertMBArtistParams) error { + _, err := q.db.Exec(ctx, upsertMBArtist, + arg.Mbid, + arg.Name, + arg.SortName, + arg.Disambiguation, + arg.ArtistType, + arg.Country, + ) + return err +} + +const upsertMBReleaseGroup = `-- name: UpsertMBReleaseGroup :exec +INSERT INTO mb_release_groups (mbid, title, primary_type, secondary_types, first_release_date, artist_credit) +VALUES ($1, $2, $3, $4::text[], + $5, $6) +ON CONFLICT (mbid) DO UPDATE SET + title = EXCLUDED.title, + primary_type = EXCLUDED.primary_type, + secondary_types = EXCLUDED.secondary_types, + first_release_date = EXCLUDED.first_release_date, + artist_credit = EXCLUDED.artist_credit, + updated_at = now() +` + +type UpsertMBReleaseGroupParams struct { + Mbid string + Title string + PrimaryType string + SecondaryTypes []string + FirstReleaseDate string + ArtistCredit string +} + +// Leaves releases_refreshed_at alone: re-reading a discography says nothing +// about whether the group's track lists are current. +func (q *Queries) UpsertMBReleaseGroup(ctx context.Context, arg UpsertMBReleaseGroupParams) error { + _, err := q.db.Exec(ctx, upsertMBReleaseGroup, + arg.Mbid, + arg.Title, + arg.PrimaryType, + arg.SecondaryTypes, + arg.FirstReleaseDate, + arg.ArtistCredit, + ) + return err +} diff --git a/internal/db/dbq/models.go b/internal/db/dbq/models.go index 64c066c6..e34dc4bb 100644 --- a/internal/db/dbq/models.go +++ b/internal/db/dbq/models.go @@ -482,6 +482,59 @@ type LoudnessSetting struct { LeveledCacheMb int32 } +type MbArtist struct { + Mbid string + Name string + SortName string + Disambiguation string + ArtistType string + Country string + DiscographyRefreshedAt pgtype.Timestamptz + CreatedAt pgtype.Timestamptz + UpdatedAt pgtype.Timestamptz +} + +type MbArtistReleaseGroup struct { + ArtistMbid string + ReleaseGroupMbid string +} + +type MbRelease struct { + Mbid string + ReleaseGroupMbid string + Title string + Status string + ReleaseDate string + Country string + Disambiguation string + Formats string + TrackCount int32 + CreatedAt pgtype.Timestamptz +} + +type MbReleaseGroup struct { + Mbid string + Title string + PrimaryType string + SecondaryTypes []string + FirstReleaseDate string + ArtistCredit string + ReleasesRefreshedAt pgtype.Timestamptz + CreatedAt pgtype.Timestamptz + UpdatedAt pgtype.Timestamptz +} + +type MbReleaseTrack struct { + ReleaseMbid string + MediumPosition int32 + Position int32 + MediumFormat string + Number string + Title string + LengthMs int32 + RecordingMbid string +} + type MissingReacquisition struct { AlbumID pgtype.UUID Attempts int32 diff --git a/internal/db/migrations/0080_release_catalogue.down.sql b/internal/db/migrations/0080_release_catalogue.down.sql new file mode 100644 index 00000000..bb9e6ac3 --- /dev/null +++ b/internal/db/migrations/0080_release_catalogue.down.sql @@ -0,0 +1,5 @@ +DROP TABLE IF EXISTS mb_release_tracks; +DROP TABLE IF EXISTS mb_releases; +DROP TABLE IF EXISTS mb_artist_release_groups; +DROP TABLE IF EXISTS mb_release_groups; +DROP TABLE IF EXISTS mb_artists; diff --git a/internal/db/migrations/0080_release_catalogue.up.sql b/internal/db/migrations/0080_release_catalogue.up.sql new file mode 100644 index 00000000..7f010aea --- /dev/null +++ b/internal/db/migrations/0080_release_catalogue.up.sql @@ -0,0 +1,86 @@ +-- 0080_release_catalogue.up.sql — Minstrel's own copy of what artists have +-- released, read from MusicBrainz (Scribe milestone #502, #5519). +-- +-- With Lidarr gone, the wanted list, search and import need to know an +-- artist's discography and each release's track list. MusicBrainz allows one +-- request a second, so what has been read is kept here and refreshed on a +-- schedule rather than asked again for every page. +-- +-- mb_artists artists the catalogue has been asked about. +-- mb_release_groups what a person calls an album (Lidarr's albums +-- were release groups too), with MusicBrainz's +-- primary and secondary types. +-- mb_artist_release_groups who is credited on each group: a collaboration +-- belongs to every credited artist's discography. +-- mb_releases the editions of a group, read when something +-- needs a track list. +-- mb_release_tracks each release's track list. +-- +-- Dates stay as MusicBrainz's partial-date text ("1997", "1997-05", +-- "1997-05-21"): most early releases have only a year. + +CREATE TABLE mb_artists ( + mbid text PRIMARY KEY, + name text NOT NULL, + sort_name text NOT NULL DEFAULT '', + disambiguation text NOT NULL DEFAULT '', + artist_type text NOT NULL DEFAULT '', + country text NOT NULL DEFAULT '', + -- When the discography was last read in full; NULL means never. + discography_refreshed_at timestamptz, + created_at timestamptz NOT NULL DEFAULT now(), + updated_at timestamptz NOT NULL DEFAULT now() +); + +CREATE INDEX mb_artists_refresh_idx ON mb_artists (discography_refreshed_at NULLS FIRST); + +CREATE TABLE mb_release_groups ( + mbid text PRIMARY KEY, + title text NOT NULL, + primary_type text NOT NULL DEFAULT '', + secondary_types text[] NOT NULL DEFAULT '{}', + first_release_date text NOT NULL DEFAULT '', + artist_credit text NOT NULL DEFAULT '', + -- When the releases and their track lists were last read; NULL means never. + releases_refreshed_at timestamptz, + created_at timestamptz NOT NULL DEFAULT now(), + updated_at timestamptz NOT NULL DEFAULT now() +); + +CREATE TABLE mb_artist_release_groups ( + artist_mbid text NOT NULL REFERENCES mb_artists (mbid) ON DELETE CASCADE, + release_group_mbid text NOT NULL REFERENCES mb_release_groups (mbid) ON DELETE CASCADE, + PRIMARY KEY (artist_mbid, release_group_mbid) +); + +CREATE INDEX mb_artist_release_groups_group_idx ON mb_artist_release_groups (release_group_mbid); + +CREATE TABLE mb_releases ( + mbid text PRIMARY KEY, + release_group_mbid text NOT NULL REFERENCES mb_release_groups (mbid) ON DELETE CASCADE, + title text NOT NULL, + status text NOT NULL DEFAULT '', + release_date text NOT NULL DEFAULT '', + country text NOT NULL DEFAULT '', + disambiguation text NOT NULL DEFAULT '', + formats text NOT NULL DEFAULT '', + track_count integer NOT NULL DEFAULT 0, + created_at timestamptz NOT NULL DEFAULT now() +); + +CREATE INDEX mb_releases_group_idx ON mb_releases (release_group_mbid); + +CREATE TABLE mb_release_tracks ( + release_mbid text NOT NULL REFERENCES mb_releases (mbid) ON DELETE CASCADE, + medium_position integer NOT NULL, + position integer NOT NULL, + medium_format text NOT NULL DEFAULT '', + number text NOT NULL DEFAULT '', + title text NOT NULL, + length_ms integer NOT NULL DEFAULT 0, + recording_mbid text NOT NULL DEFAULT '', + PRIMARY KEY (release_mbid, medium_position, position) +); + +CREATE INDEX mb_release_tracks_recording_idx ON mb_release_tracks (recording_mbid) + WHERE recording_mbid <> ''; diff --git a/internal/db/queries/catalogue.sql b/internal/db/queries/catalogue.sql new file mode 100644 index 00000000..6518e5d3 --- /dev/null +++ b/internal/db/queries/catalogue.sql @@ -0,0 +1,101 @@ +-- The release catalogue read from MusicBrainz (M502 #5519). + +-- name: UpsertMBArtist :exec +INSERT INTO mb_artists (mbid, name, sort_name, disambiguation, artist_type, country) +VALUES (sqlc.arg(mbid), sqlc.arg(name), sqlc.arg(sort_name), sqlc.arg(disambiguation), + sqlc.arg(artist_type), sqlc.arg(country)) +ON CONFLICT (mbid) DO UPDATE SET + name = EXCLUDED.name, + sort_name = EXCLUDED.sort_name, + disambiguation = EXCLUDED.disambiguation, + artist_type = EXCLUDED.artist_type, + country = EXCLUDED.country, + updated_at = now(); + +-- name: GetMBArtist :one +SELECT * FROM mb_artists WHERE mbid = sqlc.arg(mbid); + +-- name: MarkMBArtistDiscographyRefreshed :exec +UPDATE mb_artists SET discography_refreshed_at = now() WHERE mbid = sqlc.arg(mbid); + +-- name: ListMBArtistsDueForRefresh :many +-- Artists whose discography has never been read, or was read before the +-- cutoff: never-read first, then oldest first. +SELECT mbid FROM mb_artists +WHERE discography_refreshed_at IS NULL OR discography_refreshed_at < sqlc.arg(before) +ORDER BY discography_refreshed_at NULLS FIRST, mbid +LIMIT sqlc.arg(max_rows); + +-- name: UpsertMBReleaseGroup :exec +-- Leaves releases_refreshed_at alone: re-reading a discography says nothing +-- about whether the group's track lists are current. +INSERT INTO mb_release_groups (mbid, title, primary_type, secondary_types, first_release_date, artist_credit) +VALUES (sqlc.arg(mbid), sqlc.arg(title), sqlc.arg(primary_type), sqlc.arg(secondary_types)::text[], + sqlc.arg(first_release_date), sqlc.arg(artist_credit)) +ON CONFLICT (mbid) DO UPDATE SET + title = EXCLUDED.title, + primary_type = EXCLUDED.primary_type, + secondary_types = EXCLUDED.secondary_types, + first_release_date = EXCLUDED.first_release_date, + artist_credit = EXCLUDED.artist_credit, + updated_at = now(); + +-- name: GetMBReleaseGroup :one +SELECT * FROM mb_release_groups WHERE mbid = sqlc.arg(mbid); + +-- name: DeleteMBArtistReleaseGroups :exec +DELETE FROM mb_artist_release_groups WHERE artist_mbid = sqlc.arg(artist_mbid); + +-- name: AddMBArtistReleaseGroup :exec +INSERT INTO mb_artist_release_groups (artist_mbid, release_group_mbid) +VALUES (sqlc.arg(artist_mbid), sqlc.arg(release_group_mbid)) +ON CONFLICT DO NOTHING; + +-- name: ListMBArtistReleaseGroups :many +-- An artist's discography, oldest first; undated groups last. +SELECT g.* FROM mb_release_groups g +JOIN mb_artist_release_groups ag ON ag.release_group_mbid = g.mbid +WHERE ag.artist_mbid = sqlc.arg(artist_mbid) +ORDER BY g.first_release_date = '', g.first_release_date, g.title, g.mbid; + +-- name: MarkMBReleaseGroupReleasesRefreshed :exec +UPDATE mb_release_groups SET releases_refreshed_at = now() WHERE mbid = sqlc.arg(mbid); + +-- name: DeleteMBReleasesForGroup :exec +-- Tracks go with their releases (ON DELETE CASCADE). +DELETE FROM mb_releases WHERE release_group_mbid = sqlc.arg(release_group_mbid); + +-- name: InsertMBRelease :exec +INSERT INTO mb_releases (mbid, release_group_mbid, title, status, release_date, country, + disambiguation, formats, track_count) +VALUES (sqlc.arg(mbid), sqlc.arg(release_group_mbid), sqlc.arg(title), sqlc.arg(status), + sqlc.arg(release_date), sqlc.arg(country), sqlc.arg(disambiguation), sqlc.arg(formats), + sqlc.arg(track_count)) +ON CONFLICT (mbid) DO UPDATE SET + release_group_mbid = EXCLUDED.release_group_mbid, + title = EXCLUDED.title, + status = EXCLUDED.status, + release_date = EXCLUDED.release_date, + country = EXCLUDED.country, + disambiguation = EXCLUDED.disambiguation, + formats = EXCLUDED.formats, + track_count = EXCLUDED.track_count; + +-- name: InsertMBReleaseTrack :exec +INSERT INTO mb_release_tracks (release_mbid, medium_position, position, medium_format, number, + title, length_ms, recording_mbid) +VALUES (sqlc.arg(release_mbid), sqlc.arg(medium_position), sqlc.arg(position), + sqlc.arg(medium_format), sqlc.arg(number), sqlc.arg(title), sqlc.arg(length_ms), + sqlc.arg(recording_mbid)); + +-- name: ListMBReleasesForGroup :many +-- Official editions first, then by date; undated last. +SELECT * FROM mb_releases +WHERE release_group_mbid = sqlc.arg(release_group_mbid) +ORDER BY status <> 'Official', release_date = '', release_date, title, mbid; + +-- name: ListMBReleaseTracksForGroup :many +SELECT t.* FROM mb_release_tracks t +JOIN mb_releases r ON r.mbid = t.release_mbid +WHERE r.release_group_mbid = sqlc.arg(release_group_mbid) +ORDER BY t.release_mbid, t.medium_position, t.position; diff --git a/internal/dbtest/reset.go b/internal/dbtest/reset.go index 7824ccf0..b22504a1 100644 --- a/internal/dbtest/reset.go +++ b/internal/dbtest/reset.go @@ -103,6 +103,11 @@ var dataTables = []string{ "user_notification_prefs", "user_notification_email_state", // #5346 "acquisition_services", // M502 #5518 + "mb_release_tracks", // M502 #5519 + "mb_releases", // M502 #5519 + "mb_artist_release_groups", // M502 #5519 + "mb_release_groups", // M502 #5519 + "mb_artists", // M502 #5519 "tracks", "albums", "artists", diff --git a/internal/musicbrainz/client.go b/internal/musicbrainz/client.go new file mode 100644 index 00000000..2d906a76 --- /dev/null +++ b/internal/musicbrainz/client.go @@ -0,0 +1,208 @@ +package musicbrainz + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "net/url" + "strconv" + "strings" + "time" +) + +// DefaultBaseURL is the public MusicBrainz web service. +const DefaultBaseURL = "https://musicbrainz.org/ws/2" + +// UserAgent identifies Minstrel. MusicBrainz refuses requests without a +// descriptive one. +const UserAgent = "Minstrel/1.0 ( https://git.fabledsword.com/bvandeusen/minstrel )" + +// DefaultTimeout bounds one request (rule 156). A browse page with track +// lists is the slowest thing asked for. +const DefaultTimeout = 20 * time.Second + +// pageSize is the most a browse returns per page. +const pageSize = 100 + +// maxPages bounds a browse, so an artist with thousands of release groups +// cannot hold the pacer for an hour. 25 pages is 2,500 entries. +const maxPages = 25 + +var ( + // ErrUnreachable: MusicBrainz could not be asked, or kept answering with + // its throttle or a server error. Callers degrade visibly (rule 164). + ErrUnreachable = errors.New("musicbrainz: unreachable") + // ErrNotFound: MusicBrainz has nothing under that id. + ErrNotFound = errors.New("musicbrainz: not found") + // ErrBadResponse: MusicBrainz answered with something that is not the + // JSON asked for. + ErrBadResponse = errors.New("musicbrainz: unexpected response") +) + +// Client asks MusicBrainz. The zero value is usable: it talks to the public +// service through the shared pacer. +type Client struct { + BaseURL string + HTTP *http.Client + // Pace spaces requests; nil means Shared(). Tests pass a zero Pacer. + Pace *Pacer + // Retries is how many times a throttled or failed request is tried + // again. Zero means 2; negative means none. + Retries int + // Backoff is the first retry's wait, doubled each time. Zero means 1s. + Backoff time.Duration +} + +func (c *Client) base() string { + if c.BaseURL != "" { + return strings.TrimRight(c.BaseURL, "/") + } + return DefaultBaseURL +} + +func (c *Client) httpClient() *http.Client { + if c.HTTP != nil { + return c.HTTP + } + return &http.Client{Timeout: DefaultTimeout} +} + +func (c *Client) pacer() *Pacer { + if c.Pace != nil { + return c.Pace + } + return Shared() +} + +func (c *Client) retries() int { + switch { + case c.Retries < 0: + return 0 + case c.Retries == 0: + return 2 + } + return c.Retries +} + +func (c *Client) backoff(attempt int) time.Duration { + b := c.Backoff + if b <= 0 { + b = time.Second + } + return b << attempt +} + +// get fetches path?query (fmt=json added) into out, waiting on the pacer +// before every attempt. +func (c *Client) get(ctx context.Context, path string, query url.Values, out any) error { + if query == nil { + query = url.Values{} + } + query.Set("fmt", "json") + full := c.base() + path + "?" + query.Encode() + + var last error + for attempt := 0; attempt <= c.retries(); attempt++ { + if attempt > 0 { + if err := sleep(ctx, c.backoff(attempt-1)); err != nil { + return err + } + } + if err := c.pacer().Wait(ctx); err != nil { + return err + } + retry, err := c.once(ctx, full, out) + if !retry { + return err + } + last = err + } + return last +} + +// once makes one request. retry reports whether trying again could help. +func (c *Client) once(ctx context.Context, full string, out any) (retry bool, err error) { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, full, nil) + if err != nil { + return false, fmt.Errorf("musicbrainz: build request: %w", err) + } + req.Header.Set("Accept", "application/json") + req.Header.Set("User-Agent", UserAgent) + + resp, err := c.httpClient().Do(req) + if err != nil { + if ctx.Err() != nil { + return false, ctx.Err() + } + return true, fmt.Errorf("%w: %v", ErrUnreachable, err) + } + defer func() { _ = resp.Body.Close() }() + + switch { + case resp.StatusCode == http.StatusNotFound: + return false, ErrNotFound + case resp.StatusCode == http.StatusServiceUnavailable, + resp.StatusCode == http.StatusTooManyRequests, + resp.StatusCode >= 500: + // 503 is how MusicBrainz says "slow down". + return true, fmt.Errorf("%w: HTTP %d", ErrUnreachable, resp.StatusCode) + case resp.StatusCode != http.StatusOK: + return false, fmt.Errorf("%w: HTTP %d", ErrBadResponse, resp.StatusCode) + } + body, err := io.ReadAll(io.LimitReader(resp.Body, 8<<20)) + if err != nil { + return true, fmt.Errorf("%w: read: %v", ErrUnreachable, err) + } + if err := json.Unmarshal(body, out); err != nil { + return false, fmt.Errorf("%w: %v", ErrBadResponse, err) + } + return false, nil +} + +func sleep(ctx context.Context, d time.Duration) error { + t := time.NewTimer(d) + defer t.Stop() + select { + case <-ctx.Done(): + return ctx.Err() + case <-t.C: + return nil + } +} + +// luceneSpecial are the characters MusicBrainz's search syntax gives meaning. +const luceneSpecial = `+-&|!(){}[]^"~*?:\/` + +// Escape makes text typed by a person safe to put in a search query, so +// "AC/DC" or "Sunn O)))" searches for what was typed rather than failing to +// parse. +func Escape(s string) string { + var b strings.Builder + for _, r := range s { + if strings.ContainsRune(luceneSpecial, r) { + b.WriteByte('\\') + } + b.WriteRune(r) + } + return b.String() +} + +// fieldQuery searches each field for the escaped text. +func fieldQuery(text string, fields ...string) string { + t := Escape(strings.TrimSpace(text)) + parts := make([]string, len(fields)) + for i, f := range fields { + parts[i] = f + ":(" + t + ")" + } + return strings.Join(parts, " OR ") +} + +func searchLimit(limit int) string { + if limit <= 0 || limit > 100 { + limit = 25 + } + return strconv.Itoa(limit) +} diff --git a/internal/musicbrainz/client_test.go b/internal/musicbrainz/client_test.go new file mode 100644 index 00000000..d645984f --- /dev/null +++ b/internal/musicbrainz/client_test.go @@ -0,0 +1,202 @@ +package musicbrainz + +import ( + "context" + "errors" + "fmt" + "net/http" + "net/http/httptest" + "strconv" + "sync/atomic" + "testing" + "time" +) + +// testClient talks to srv with no pacing and fast retries. +func testClient(srv *httptest.Server) *Client { + return &Client{BaseURL: srv.URL, Pace: &Pacer{}, Backoff: time.Millisecond} +} + +func TestEscape_KeepsWhatWasTyped(t *testing.T) { + for in, want := range map[string]string{ + "Radiohead": "Radiohead", + "AC/DC": `AC\/DC`, + "Sunn O)))": `Sunn O\)\)\)`, + `"Weird" Al`: `\"Weird\" Al`, + } { + if got := Escape(in); got != want { + t.Errorf("Escape(%q) = %q, want %q", in, got, want) + } + } +} + +func TestSearchReleaseGroups_AsksTitleAndArtistAndReadsCredits(t *testing.T) { + var query, agent atomic.Value + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + query.Store(r.URL.Query().Get("query")) + agent.Store(r.Header.Get("User-Agent")) + _, _ = w.Write([]byte(`{"release-groups": [{ + "id": "rg1", "title": "OK Computer", "primary-type": "Album", "secondary-types": [], + "first-release-date": "1997-05-21", + "artist-credit": [{"name": "Radiohead", "joinphrase": "", "artist": {"id": "a1", "name": "Radiohead"}}] + }]}`)) + })) + defer srv.Close() + + got, err := testClient(srv).SearchReleaseGroups(context.Background(), "ok computer", 10) + if err != nil { + t.Fatal(err) + } + if q := query.Load(); q != "releasegroup:(ok computer) OR artist:(ok computer)" { + t.Errorf("query = %q", q) + } + if agent.Load() != UserAgent { + t.Errorf("User-Agent = %q", agent.Load()) + } + if len(got) != 1 || got[0].MBID != "rg1" || got[0].Year() != "1997" || got[0].ArtistCredit != "Radiohead" || got[0].ArtistMBIDs[0] != "a1" { + t.Errorf("got %+v", got) + } +} + +func TestSearchRecordings_FilesASongUnderItsAlbum(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + _, _ = w.Write([]byte(`{"recordings": [{ + "id": "rec1", "title": "Karma Police", "length": 264000, + "artist-credit": [{"name": "Radiohead", "artist": {"id": "a1"}}], + "releases": [ + {"id": "r1", "status": "Official", "release-group": {"id": "rg-live", "title": "Live", "primary-type": "Album", "secondary-types": ["Live"]}}, + {"id": "r2", "status": "Official", "release-group": {"id": "rg1", "title": "OK Computer", "primary-type": "Album"}} + ] + }]}`)) + })) + defer srv.Close() + + got, err := testClient(srv).SearchRecordings(context.Background(), "karma police", 5) + if err != nil { + t.Fatal(err) + } + if len(got) != 1 || got[0].ReleaseGroup.MBID != "rg1" || got[0].LengthMs != 264000 { + t.Errorf("got %+v, want it filed under the studio album", got) + } +} + +func TestArtistReleaseGroups_PagesUntilTheCount(t *testing.T) { + var calls atomic.Int32 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + calls.Add(1) + offset, _ := strconv.Atoi(r.URL.Query().Get("offset")) + if r.URL.Query().Get("artist") != "a1" { + t.Errorf("artist = %q", r.URL.Query().Get("artist")) + } + // 3 in all, two per page. + var items string + for i := offset; i < offset+2 && i < 3; i++ { + if items != "" { + items += "," + } + items += fmt.Sprintf(`{"id": "rg%d", "title": "T%d", "primary-type": "Album"}`, i, i) + } + _, _ = fmt.Fprintf(w, `{"release-group-count": 3, "release-groups": [%s]}`, items) + })) + defer srv.Close() + + got, err := testClient(srv).ArtistReleaseGroups(context.Background(), "a1") + if err != nil { + t.Fatal(err) + } + if len(got) != 3 || calls.Load() != 2 { + t.Errorf("got %d groups in %d calls, want 3 in 2", len(got), calls.Load()) + } +} + +func TestReleaseGroupReleases_ReadsMediaAndTracks(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Query().Get("inc") != "media recordings" { + t.Errorf("inc = %q", r.URL.Query().Get("inc")) + } + _, _ = w.Write([]byte(`{"release-count": 1, "releases": [{ + "id": "r1", "title": "OK Computer", "status": "Official", "date": "1997-06-16", "country": "GB", + "media": [ + {"position": 1, "format": "CD", "tracks": [ + {"position": 1, "number": "1", "title": "Airbag", "length": 284000, "recording": {"id": "rec-a"}}, + {"position": 2, "number": "2", "title": "Paranoid Android", "length": null, "recording": {"id": "rec-b", "length": 383000}} + ]}, + {"position": 2, "format": "CD", "tracks": []} + ] + }]}`)) + })) + defer srv.Close() + + got, err := testClient(srv).ReleaseGroupReleases(context.Background(), "rg1") + if err != nil { + t.Fatal(err) + } + if len(got) != 1 { + t.Fatalf("got %+v", got) + } + r := got[0] + if r.TrackCount() != 2 || r.Formats() != "2×CD" { + t.Errorf("tracks %d, formats %q", r.TrackCount(), r.Formats()) + } + if tr := r.Media[0].Tracks[1]; tr.LengthMs != 383000 || tr.RecordingMBID != "rec-b" { + t.Errorf("a track with no length of its own should take the recording's: %+v", tr) + } +} + +func TestGet_ThrottleIsRetriedThenUnreachable(t *testing.T) { + var calls atomic.Int32 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + if calls.Add(1) == 1 { + w.WriteHeader(http.StatusServiceUnavailable) + return + } + _, _ = w.Write([]byte(`{"artists": [{"id": "a1", "name": "Radiohead"}]}`)) + })) + defer srv.Close() + + got, err := testClient(srv).SearchArtists(context.Background(), "radiohead", 5) + if err != nil || len(got) != 1 || calls.Load() != 2 { + t.Fatalf("a throttled first answer should be retried: %v, %+v, %d calls", err, got, calls.Load()) + } + + down := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusBadGateway) + })) + defer down.Close() + if _, err := testClient(down).SearchArtists(context.Background(), "x", 5); !errors.Is(err, ErrUnreachable) { + t.Errorf("err = %v, want ErrUnreachable", err) + } +} + +func TestGet_NotFoundAndNoServer(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusNotFound) + })) + if _, err := testClient(srv).Artist(context.Background(), "nope"); !errors.Is(err, ErrNotFound) { + t.Errorf("err = %v, want ErrNotFound", err) + } + srv.Close() + c := testClient(srv) + c.Retries = -1 + if _, err := c.Artist(context.Background(), "a1"); !errors.Is(err, ErrUnreachable) { + t.Errorf("a closed server: err = %v, want ErrUnreachable", err) + } +} + +func TestPacer_SpacesCallers(t *testing.T) { + p := &Pacer{Interval: 30 * time.Millisecond} + start := time.Now() + for range 3 { + if err := p.Wait(context.Background()); err != nil { + t.Fatal(err) + } + } + if elapsed := time.Since(start); elapsed < 60*time.Millisecond { + t.Errorf("three waits took %v, want at least two intervals", elapsed) + } + ctx, cancel := context.WithCancel(context.Background()) + cancel() + if err := p.Wait(ctx); err == nil { + t.Error("a cancelled wait should return the context's error") + } +} diff --git a/internal/musicbrainz/entities.go b/internal/musicbrainz/entities.go new file mode 100644 index 00000000..21a449a4 --- /dev/null +++ b/internal/musicbrainz/entities.go @@ -0,0 +1,383 @@ +package musicbrainz + +import ( + "context" + "net/url" + "strconv" + "strings" +) + +// Artist is a MusicBrainz artist. +type Artist struct { + MBID string + Name string + SortName string + Disambiguation string + Type string // Person, Group, Orchestra, …; may be empty + Country string +} + +// ReleaseGroup is what a person calls an album: one work, released any +// number of times. Lidarr's albums were release groups too. +type ReleaseGroup struct { + MBID string + Title string + PrimaryType string // Album, Single, EP, Broadcast, Other; may be empty + SecondaryTypes []string // Compilation, Live, Remix, Soundtrack, … + // FirstReleaseDate is MusicBrainz's partial date: "", "1997", + // "1997-05" or "1997-05-21". + FirstReleaseDate string + ArtistCredit string // the credit as printed, joinphrases included + ArtistMBIDs []string +} + +// Year is the first release's year, or "". +func (g ReleaseGroup) Year() string { + if len(g.FirstReleaseDate) >= 4 { + return g.FirstReleaseDate[:4] + } + return "" +} + +// Release is one issue of a release group: a pressing, an edition, a +// digital release, each with its own track list. +type Release struct { + MBID string + Title string + Status string // Official, Promotion, Bootleg, Pseudo-Release; may be empty + Date string // partial, like ReleaseGroup.FirstReleaseDate + Country string + Disambiguation string + Media []Medium +} + +// TrackCount totals the tracks across every medium. +func (r Release) TrackCount() int { + n := 0 + for _, m := range r.Media { + n += len(m.Tracks) + } + return n +} + +// Formats summarises the media, e.g. "CD", "2×CD" or "CD + DVD". +func (r Release) Formats() string { + var parts []string + counts := map[string]int{} + for _, m := range r.Media { + f := m.Format + if f == "" { + f = "Unknown" + } + if counts[f] == 0 { + parts = append(parts, f) + } + counts[f]++ + } + for i, f := range parts { + if counts[f] > 1 { + parts[i] = strconv.Itoa(counts[f]) + "×" + f + } + } + return strings.Join(parts, " + ") +} + +// Medium is one disc (or side, or file set) of a release. +type Medium struct { + Position int + Format string + Tracks []Track +} + +// Track is a position on a medium, pointing at the recording heard there. +type Track struct { + Position int + Number string // as printed: "1", "A1", … + Title string + LengthMs int // 0 when unknown + RecordingMBID string +} + +// Recording is a piece of audio, as a search returns it. +type Recording struct { + MBID string + Title string + LengthMs int + ArtistCredit string + ArtistMBIDs []string + // ReleaseGroup is the group the recording is best known from: the first + // official album it appears on, else the first release listed. + ReleaseGroup ReleaseGroup +} + +// --- wire shapes --- + +type wireCredit struct { + Name string `json:"name"` + JoinPhrase string `json:"joinphrase"` + Artist struct { + ID string `json:"id"` + Name string `json:"name"` + } `json:"artist"` +} + +func credit(cs []wireCredit) (string, []string) { + var b strings.Builder + ids := make([]string, 0, len(cs)) + for _, c := range cs { + name := c.Name + if name == "" { + name = c.Artist.Name + } + b.WriteString(name) + b.WriteString(c.JoinPhrase) + if c.Artist.ID != "" { + ids = append(ids, c.Artist.ID) + } + } + return b.String(), ids +} + +type wireArtist struct { + ID string `json:"id"` + Name string `json:"name"` + SortName string `json:"sort-name"` + Disambiguation string `json:"disambiguation"` + Type string `json:"type"` + Country string `json:"country"` +} + +func (w wireArtist) artist() Artist { + return Artist{MBID: w.ID, Name: w.Name, SortName: w.SortName, Disambiguation: w.Disambiguation, Type: w.Type, Country: w.Country} +} + +type wireReleaseGroup struct { + ID string `json:"id"` + Title string `json:"title"` + PrimaryType string `json:"primary-type"` + SecondaryTypes []string `json:"secondary-types"` + FirstReleaseDate string `json:"first-release-date"` + ArtistCredit []wireCredit `json:"artist-credit"` +} + +func (w wireReleaseGroup) group() ReleaseGroup { + name, ids := credit(w.ArtistCredit) + return ReleaseGroup{ + MBID: w.ID, Title: w.Title, PrimaryType: w.PrimaryType, + SecondaryTypes: append([]string{}, w.SecondaryTypes...), FirstReleaseDate: w.FirstReleaseDate, + ArtistCredit: name, ArtistMBIDs: ids, + } +} + +type wireTrack struct { + Position int `json:"position"` + Number string `json:"number"` + Title string `json:"title"` + Length *int `json:"length"` + Recording struct { + ID string `json:"id"` + Length *int `json:"length"` + } `json:"recording"` +} + +type wireRelease struct { + ID string `json:"id"` + Title string `json:"title"` + Status string `json:"status"` + Date string `json:"date"` + Country string `json:"country"` + Disambiguation string `json:"disambiguation"` + Media []struct { + Position int `json:"position"` + Format string `json:"format"` + Tracks []wireTrack `json:"tracks"` + } `json:"media"` + ReleaseGroup *wireReleaseGroup `json:"release-group"` +} + +func (w wireRelease) release() Release { + r := Release{MBID: w.ID, Title: w.Title, Status: w.Status, Date: w.Date, Country: w.Country, Disambiguation: w.Disambiguation} + for _, m := range w.Media { + med := Medium{Position: m.Position, Format: m.Format} + for _, t := range m.Tracks { + length := 0 + switch { + case t.Length != nil: + length = *t.Length + case t.Recording.Length != nil: + length = *t.Recording.Length + } + med.Tracks = append(med.Tracks, Track{ + Position: t.Position, Number: t.Number, Title: t.Title, + LengthMs: length, RecordingMBID: t.Recording.ID, + }) + } + r.Media = append(r.Media, med) + } + return r +} + +// --- searches --- + +// SearchArtists finds artists by name or alias. +func (c *Client) SearchArtists(ctx context.Context, text string, limit int) ([]Artist, error) { + var resp struct { + Artists []wireArtist `json:"artists"` + } + q := url.Values{"query": {fieldQuery(text, "artist", "alias")}, "limit": {searchLimit(limit)}} + if err := c.get(ctx, "/artist", q, &resp); err != nil { + return nil, err + } + out := make([]Artist, 0, len(resp.Artists)) + for _, a := range resp.Artists { + out = append(out, a.artist()) + } + return out, nil +} + +// SearchReleaseGroups finds albums by title or by artist, so "radiohead ok +// computer" and "ok computer" both find OK Computer. +func (c *Client) SearchReleaseGroups(ctx context.Context, text string, limit int) ([]ReleaseGroup, error) { + var resp struct { + ReleaseGroups []wireReleaseGroup `json:"release-groups"` + } + q := url.Values{"query": {fieldQuery(text, "releasegroup", "artist")}, "limit": {searchLimit(limit)}} + if err := c.get(ctx, "/release-group", q, &resp); err != nil { + return nil, err + } + out := make([]ReleaseGroup, 0, len(resp.ReleaseGroups)) + for _, g := range resp.ReleaseGroups { + out = append(out, g.group()) + } + return out, nil +} + +// SearchRecordings finds songs by title or by artist. +func (c *Client) SearchRecordings(ctx context.Context, text string, limit int) ([]Recording, error) { + var resp struct { + Recordings []struct { + ID string `json:"id"` + Title string `json:"title"` + Length *int `json:"length"` + ArtistCredit []wireCredit `json:"artist-credit"` + Releases []wireRelease `json:"releases"` + } `json:"recordings"` + } + q := url.Values{"query": {fieldQuery(text, "recording", "artist")}, "limit": {searchLimit(limit)}} + if err := c.get(ctx, "/recording", q, &resp); err != nil { + return nil, err + } + out := make([]Recording, 0, len(resp.Recordings)) + for _, r := range resp.Recordings { + name, ids := credit(r.ArtistCredit) + rec := Recording{MBID: r.ID, Title: r.Title, ArtistCredit: name, ArtistMBIDs: ids} + if r.Length != nil { + rec.LengthMs = *r.Length + } + rec.ReleaseGroup = bestKnownFrom(r.Releases) + out = append(out, rec) + } + return out, nil +} + +// bestKnownFrom picks the release group a recording should be filed under +// in a search result: the first official album, else the first release. +func bestKnownFrom(releases []wireRelease) ReleaseGroup { + var first *wireReleaseGroup + for _, r := range releases { + if r.ReleaseGroup == nil || r.ReleaseGroup.ID == "" { + continue + } + if first == nil { + first = r.ReleaseGroup + } + if r.Status == "Official" && r.ReleaseGroup.PrimaryType == "Album" && len(r.ReleaseGroup.SecondaryTypes) == 0 { + return r.ReleaseGroup.group() + } + } + if first == nil { + return ReleaseGroup{} + } + return first.group() +} + +// --- lookups and browses --- + +// Artist looks one artist up by MBID. +func (c *Client) Artist(ctx context.Context, mbid string) (Artist, error) { + var w wireArtist + if err := c.get(ctx, "/artist/"+url.PathEscape(mbid), nil, &w); err != nil { + return Artist{}, err + } + if w.ID == "" { + return Artist{}, ErrBadResponse + } + return w.artist(), nil +} + +// ArtistReleaseGroups lists every release group the artist is credited on, +// paging through MusicBrainz's browse. +func (c *Client) ArtistReleaseGroups(ctx context.Context, artistMBID string) ([]ReleaseGroup, error) { + var out []ReleaseGroup + for page := 0; page < maxPages; page++ { + var resp struct { + Count int `json:"release-group-count"` + ReleaseGroups []wireReleaseGroup `json:"release-groups"` + } + q := url.Values{ + "artist": {artistMBID}, "inc": {"artist-credits"}, + "limit": {strconv.Itoa(pageSize)}, "offset": {strconv.Itoa(len(out))}, + } + if err := c.get(ctx, "/release-group", q, &resp); err != nil { + return nil, err + } + for _, g := range resp.ReleaseGroups { + out = append(out, g.group()) + } + if len(resp.ReleaseGroups) == 0 || len(out) >= resp.Count { + break + } + } + return out, nil +} + +// ReleaseGroupReleases lists every release of a release group with its +// media and track lists. +func (c *Client) ReleaseGroupReleases(ctx context.Context, groupMBID string) ([]Release, error) { + var out []Release + for page := 0; page < maxPages; page++ { + var resp struct { + Count int `json:"release-count"` + Releases []wireRelease `json:"releases"` + } + q := url.Values{ + "release-group": {groupMBID}, "inc": {"media recordings"}, + "limit": {strconv.Itoa(pageSize)}, "offset": {strconv.Itoa(len(out))}, + } + if err := c.get(ctx, "/release", q, &resp); err != nil { + return nil, err + } + for _, r := range resp.Releases { + out = append(out, r.release()) + } + // Advance by what came back: MusicBrainz may return fewer than the + // page size when track lists are large. + if len(resp.Releases) == 0 || len(out) >= resp.Count { + break + } + } + return out, nil +} + +// ReleaseGroup looks one release group up by MBID, with its artist credit. +func (c *Client) ReleaseGroup(ctx context.Context, mbid string) (ReleaseGroup, error) { + var w wireReleaseGroup + q := url.Values{"inc": {"artist-credits"}} + if err := c.get(ctx, "/release-group/"+url.PathEscape(mbid), q, &w); err != nil { + return ReleaseGroup{}, err + } + if w.ID == "" { + return ReleaseGroup{}, ErrBadResponse + } + return w.group(), nil +} diff --git a/internal/musicbrainz/pace.go b/internal/musicbrainz/pace.go new file mode 100644 index 00000000..4a0b3ed1 --- /dev/null +++ b/internal/musicbrainz/pace.go @@ -0,0 +1,59 @@ +// Package musicbrainz is Minstrel's client for the MusicBrainz web service: +// searches and browses for the release catalogue (M502 #5519), and the one +// pacer every MusicBrainz request in the process waits on. +package musicbrainz + +import ( + "context" + "sync" + "time" +) + +// MinInterval honours MusicBrainz's 1 request/second ceiling, with headroom. +// The limit is per client address, not per caller, which is why there is one +// pacer for the whole process. +const MinInterval = 1100 * time.Millisecond + +// Pacer spaces requests at least Interval apart. Safe for concurrent use; +// callers queue in the order they claim slots. +type Pacer struct { + Interval time.Duration + + mu sync.Mutex + next time.Time +} + +// Wait blocks until this caller's slot comes up, or ctx ends. A zero +// Interval never waits. +func (p *Pacer) Wait(ctx context.Context) error { + if p == nil || p.Interval <= 0 { + return ctx.Err() + } + p.mu.Lock() + now := time.Now() + slot := p.next + if slot.Before(now) { + slot = now + } + p.next = slot.Add(p.Interval) + p.mu.Unlock() + + wait := time.Until(slot) + if wait <= 0 { + return ctx.Err() + } + timer := time.NewTimer(wait) + defer timer.Stop() + select { + case <-ctx.Done(): + return ctx.Err() + case <-timer.C: + return nil + } +} + +var shared = &Pacer{Interval: MinInterval} + +// Shared is the pacer every MusicBrainz request in the process goes through: +// the catalogue's and tag enrichment's alike. +func Shared() *Pacer { return shared } diff --git a/internal/tags/httpclient.go b/internal/tags/httpclient.go index a65f1087..44f3b521 100644 --- a/internal/tags/httpclient.go +++ b/internal/tags/httpclient.go @@ -25,6 +25,10 @@ type httpClientOptions struct { UserAgent string // MinInterval throttles requests; 0 disables rate limiting. MinInterval time.Duration + // Pacer, when set, replaces MinInterval: requests wait on it instead. + // MusicBrainz's provider passes musicbrainz.Shared(), because its limit + // is per address and the release catalogue spends the same budget. + Pacer interface{ Wait(context.Context) error } // MaxRetries is the number of retry attempts on 429 / 5xx. MaxRetries int // BaseBackoff is the base unit for exponential backoff (2^attempt × base ± 20%). @@ -141,6 +145,9 @@ func (c *httpClient) handleStatus(ctx context.Context, resp *http.Response, atte // rateLimitWait blocks until MinInterval has elapsed since the last call, // then claims the slot. Disabled when MinInterval is 0. func (c *httpClient) rateLimitWait(ctx context.Context) error { + if c.opts.Pacer != nil { + return c.opts.Pacer.Wait(ctx) + } if c.opts.MinInterval <= 0 { return nil } diff --git a/internal/tags/provider_musicbrainz.go b/internal/tags/provider_musicbrainz.go index b652a917..c60b6e0f 100644 --- a/internal/tags/provider_musicbrainz.go +++ b/internal/tags/provider_musicbrainz.go @@ -7,16 +7,16 @@ import ( "net/url" "sync/atomic" "time" + + "git.fabledsword.com/bvandeusen/minstrel/internal/musicbrainz" ) // mbBaseURL is a var (not const) so tests can point it at an httptest server. var mbBaseURL = "https://musicbrainz.org/ws/2" const ( - // mbMinPeriod honors MusicBrainz's 1 req/s anonymous ceiling (with headroom). - mbMinPeriod = 1100 * time.Millisecond // mbUserAgent identifies the app — MusicBrainz rejects requests without one. - mbUserAgent = "Minstrel/1.0 ( https://git.fabledsword.com/bvandeusen/minstrel )" + mbUserAgent = musicbrainz.UserAgent // artistTagWeightFactor down-weights artist-level fallback tags relative // to recording-specific tags — the artist's overall character is a coarser // signal than a tag on the exact recording (#1519). @@ -37,11 +37,13 @@ type musicbrainzProvider struct { func init() { Register(&musicbrainzProvider{ client: newHTTPClient(httpClientOptions{ - Name: "musicbrainz", - UserAgent: mbUserAgent, - HTTPClient: &http.Client{Timeout: 30 * time.Second}, - MinInterval: mbMinPeriod, - MaxRetries: 3, + Name: "musicbrainz", + UserAgent: mbUserAgent, + HTTPClient: &http.Client{Timeout: 30 * time.Second}, + // MusicBrainz's 1 request/second is per address: share it with + // the release catalogue (M502 #5519). + Pacer: musicbrainz.Shared(), + MaxRetries: 3, }), }) } diff --git a/web/src/lib/api/lidarr.test.ts b/web/src/lib/api/catalogue.test.ts similarity index 60% rename from web/src/lib/api/lidarr.test.ts rename to web/src/lib/api/catalogue.test.ts index 31e26c90..56fd5623 100644 --- a/web/src/lib/api/lidarr.test.ts +++ b/web/src/lib/api/catalogue.test.ts @@ -9,39 +9,39 @@ vi.mock('./client', () => ({ } })); -import { searchLidarr } from './lidarr'; +import { searchCatalogue } from './catalogue'; import { api } from './client'; import { qk } from './queries'; -import type { LidarrSearchResult } from './types'; +import type { CatalogueSearchResult } from './types'; beforeEach(() => { vi.clearAllMocks(); }); -describe('searchLidarr', () => { +describe('searchCatalogue', () => { test('builds URL with q and kind query params', async () => { (api.get as ReturnType).mockResolvedValueOnce([]); - await searchLidarr('aphex twin', 'artist'); - expect(api.get).toHaveBeenCalledWith('/api/lidarr/search?q=aphex+twin&kind=artist'); + await searchCatalogue('aphex twin', 'artist'); + expect(api.get).toHaveBeenCalledWith('/api/catalogue/search?q=aphex+twin&kind=artist'); }); test('encodes special characters in q (quotes, ampersands, plus)', async () => { (api.get as ReturnType).mockResolvedValueOnce([]); - await searchLidarr('boards of canada & friends', 'album'); + await searchCatalogue('boards of canada & friends', 'album'); const calledWith = (api.get as ReturnType).mock.calls[0][0] as string; // URLSearchParams encodes space as '+' and '&' as '%26'. expect(calledWith).toBe( - '/api/lidarr/search?q=boards+of+canada+%26+friends&kind=album' + '/api/catalogue/search?q=boards+of+canada+%26+friends&kind=album' ); }); test('passes through results unchanged', async () => { - const fixture: LidarrSearchResult[] = [ + const fixture: CatalogueSearchResult[] = [ { mbid: 'mbid-1', name: 'Aphex Twin', secondary_text: 'Electronic', - image_url: 'https://example.test/img.jpg', + image_url: '/api/catalogue/art/artist/mbid-1?name=Aphex+Twin', artist_mbid: '', album_mbid: '', in_library: false, @@ -49,24 +49,24 @@ describe('searchLidarr', () => { } ]; (api.get as ReturnType).mockResolvedValueOnce(fixture); - const out = await searchLidarr('aphex', 'artist'); + const out = await searchCatalogue('aphex', 'artist'); expect(out).toBe(fixture); }); test('propagates errors from api.get', async () => { - const err = { code: 'lidarr_unavailable', message: 'down', status: 503 }; + const err = { code: 'musicbrainz_unreachable', message: 'down', status: 503 }; (api.get as ReturnType).mockRejectedValueOnce(err); - await expect(searchLidarr('x', 'track')).rejects.toMatchObject({ - code: 'lidarr_unavailable', + await expect(searchCatalogue('x', 'track')).rejects.toMatchObject({ + code: 'musicbrainz_unreachable', status: 503 }); }); }); -describe('qk.lidarrSearch', () => { +describe('qk.catalogueSearch', () => { test('keys include q + kind', () => { - expect(qk.lidarrSearch('foo', 'artist')).toEqual([ - 'lidarrSearch', + expect(qk.catalogueSearch('foo', 'artist')).toEqual([ + 'catalogueSearch', { q: 'foo', kind: 'artist' } ]); }); diff --git a/web/src/lib/api/catalogue.ts b/web/src/lib/api/catalogue.ts new file mode 100644 index 00000000..093a8183 --- /dev/null +++ b/web/src/lib/api/catalogue.ts @@ -0,0 +1,26 @@ +import { createQuery } from '@tanstack/svelte-query'; +import { api } from './client'; +import { qk } from './queries'; +import type { CatalogueSearchResult, LidarrRequestKind } from './types'; + +// searchCatalogue asks MusicBrainz through Minstrel's release catalogue +// (M502 #5519). URLSearchParams handles encoding (spaces, quotes, +// ampersands) so callers can pass q raw. +export async function searchCatalogue( + q: string, + kind: LidarrRequestKind +): Promise { + const params = new URLSearchParams({ q, kind }); + return api.get(`/api/catalogue/search?${params.toString()}`); +} + +// MusicBrainz allows a request a second, so a repeated search within a minute +// is served from the client's cache rather than asked again. +export function createCatalogueSearchQuery(q: string, kind: LidarrRequestKind) { + return createQuery({ + queryKey: qk.catalogueSearch(q, kind), + queryFn: () => searchCatalogue(q, kind), + enabled: q.length > 0, + staleTime: 60_000 + }); +} diff --git a/web/src/lib/api/lidarr.ts b/web/src/lib/api/lidarr.ts deleted file mode 100644 index e3ddd332..00000000 --- a/web/src/lib/api/lidarr.ts +++ /dev/null @@ -1,28 +0,0 @@ -import { createQuery } from '@tanstack/svelte-query'; -import { api } from './client'; -import { qk } from './queries'; -import type { LidarrSearchResult, LidarrRequestKind } from './types'; - -// Search-only. Admin-side Lidarr config + connection probing lives in admin.ts. - -// searchLidarr proxies the user-facing Lidarr search endpoint. URLSearchParams -// handles encoding (spaces, quotes, ampersands) so callers can pass q raw. -export async function searchLidarr( - q: string, - kind: LidarrRequestKind -): Promise { - const params = new URLSearchParams({ q, kind }); - return api.get(`/api/lidarr/search?${params.toString()}`); -} - -// staleTime mirrors the spec §11 60s server-side LRU cache: re-issuing the -// same query within a minute hits cache on the server, so there's no point -// in marking it stale client-side any sooner. -export function createLidarrSearchQuery(q: string, kind: LidarrRequestKind) { - return createQuery({ - queryKey: qk.lidarrSearch(q, kind), - queryFn: () => searchLidarr(q, kind), - enabled: q.length > 0, - staleTime: 60_000 - }); -} diff --git a/web/src/lib/api/queries.ts b/web/src/lib/api/queries.ts index 8ed04848..058318c7 100644 --- a/web/src/lib/api/queries.ts +++ b/web/src/lib/api/queries.ts @@ -30,8 +30,8 @@ export const qk = { likedArtists: () => ['likedArtists'] as const, history: () => ['history'] as const, // Lidarr / requests / admin. - lidarrSearch: (q: string, kind: string) => - ['lidarrSearch', { q, kind }] as const, + catalogueSearch: (q: string, kind: string) => + ['catalogueSearch', { q, kind }] as const, myRequests: () => ['myRequests'] as const, lidarrConfig: () => ['lidarrConfig'] as const, lidarrQualityProfiles: () => ['lidarrQualityProfiles'] as const, diff --git a/web/src/lib/api/types.ts b/web/src/lib/api/types.ts index 8cfc208a..83279c2f 100644 --- a/web/src/lib/api/types.ts +++ b/web/src/lib/api/types.ts @@ -146,7 +146,10 @@ export type LidarrRequestStatus = export type LidarrRequestKind = 'artist' | 'album' | 'track'; -export type LidarrSearchResult = { +// One row of the request search (GET /api/catalogue/search, M502 #5519). +// An album is a MusicBrainz release group; image_url is Minstrel's own art +// endpoint, which answers 404 when no source has an image. +export type CatalogueSearchResult = { mbid: string; name: string; secondary_text: string; diff --git a/web/src/lib/components/DiscoverResultCard.svelte b/web/src/lib/components/DiscoverResultCard.svelte index 5dd986ed..7730db02 100644 --- a/web/src/lib/components/DiscoverResultCard.svelte +++ b/web/src/lib/components/DiscoverResultCard.svelte @@ -28,12 +28,21 @@ state: DiscoverCardState; attribution?: string; onRequest?: () => void; - // Omit both to get a card with no snooze affordance — the Lidarr search + // Omit both to get a card with no snooze affordance — the catalogue search // results reuse this component and have nothing to park. onSnooze?: () => void; onUnsnooze?: () => void; } = $props(); + // Art comes from Minstrel's catalogue endpoint, which answers 404 when no + // source has an image (M502 #5519): show the icon then, not a broken image. + // Reset when the card is reused for another result. + let imageFailed = $state(false); + $effect(() => { + void imageUrl; + imageFailed = false; + }); + const FallbackIcon = $derived( kind === 'artist' ? Disc3 : kind === 'album' ? Album : Music2 ); @@ -50,12 +59,13 @@
    - {#if imageUrl} + {#if imageUrl && !imageFailed} (imageFailed = true)} /> {:else} diff --git a/web/src/lib/components/DiscoverResultCard.test.ts b/web/src/lib/components/DiscoverResultCard.test.ts index 86114a59..7057a7ef 100644 --- a/web/src/lib/components/DiscoverResultCard.test.ts +++ b/web/src/lib/components/DiscoverResultCard.test.ts @@ -168,4 +168,20 @@ describe('DiscoverResultCard', () => { expect(onUnsnooze).toHaveBeenCalledOnce(); expect(onRequest).not.toHaveBeenCalled(); }); + + test('an image that fails to load falls back to the icon', async () => { + const { container } = render(DiscoverResultCard, { + props: { + kind: 'artist', + title: 'Boards of Canada', + state: 'requestable', + imageUrl: '/api/catalogue/art/artist/x', + }, + }); + const img = container.querySelector('img'); + expect(img).not.toBeNull(); + await fireEvent.error(img!); + expect(container.querySelector('img')).toBeNull(); + expect(container.querySelector('.art svg')).not.toBeNull(); + }); }); diff --git a/web/src/lib/styles/error-copy.json b/web/src/lib/styles/error-copy.json index caf14ca5..1dfaea6c 100644 --- a/web/src/lib/styles/error-copy.json +++ b/web/src/lib/styles/error-copy.json @@ -30,6 +30,7 @@ "mbid_required": "An MBID is required for this lookup.", "system_playlist_readonly": "System playlists can't be edited directly.", "connection_refused": "Couldn't reach the server. Check the URL and try again.", + "musicbrainz_unreachable": "MusicBrainz can't be reached right now, so search is unavailable. Try again in a minute.", "lidarr_unreachable": "Lidarr is unreachable right now. Try again, or check Admin → Integrations.", "lidarr_disabled": "Lidarr integration is not enabled.", "lidarr_auth_failed": "Lidarr authentication failed.", diff --git a/web/src/routes/admin/integrations/+page.svelte b/web/src/routes/admin/integrations/+page.svelte index ae0a137c..f860cfa6 100644 --- a/web/src/routes/admin/integrations/+page.svelte +++ b/web/src/routes/admin/integrations/+page.svelte @@ -428,8 +428,8 @@

    Lidarr

    - Search Lidarr from /discover and - route approved requests to it. + Route approved requests from /discover + to Lidarr. Search itself now asks MusicBrainz.

    diff --git a/web/src/routes/discover/+page.svelte b/web/src/routes/discover/+page.svelte index c82bd2b7..1247c0d2 100644 --- a/web/src/routes/discover/+page.svelte +++ b/web/src/routes/discover/+page.svelte @@ -1,6 +1,6 @@
    {/if}
    - {#if state === 'kept'} + {#if cardState === 'kept'} Kept - {:else if state === 'snoozed'} + {:else if cardState === 'snoozed'} Not right now {/if}
    - {#if state === 'snoozed'} + {#if cardState === 'snoozed'} - {:else if state === 'requestable'} + {:else if cardState === 'requestable'}
    - {:else if state === 'kept'} + {:else if cardState === 'kept'} + + (open = false)}> +

    + New releases from {artistName} go on the wanted list. The profile says which kinds of + release are wanted. +

    + + + + + +
    + {#if following} + + {/if} + + +
    +     diff --git a/web/src/lib/components/FollowControl.test.ts b/web/src/lib/components/FollowControl.test.ts new file mode 100644 index 00000000..df9f2de5 --- /dev/null +++ b/web/src/lib/components/FollowControl.test.ts @@ -0,0 +1,67 @@ +import { afterEach, describe, expect, test, vi } from 'vitest'; +import { fireEvent, render, screen, waitFor } from '@testing-library/svelte'; +import { mockQuery } from '../../test-utils/query'; +import type { FollowState, ReleaseProfilesResponse } from '#lib/api/types.js'; + +vi.mock('#lib/api/wanted.js', () => ({ + createFollowQuery: vi.fn(), + createReleaseProfilesQuery: vi.fn(), + putFollow: vi.fn().mockResolvedValue({ following: true }), + deleteFollow: vi.fn().mockResolvedValue(undefined) +})); +vi.mock('#lib/stores/toast.svelte.js', () => ({ pushToast: vi.fn() })); + +import FollowControl from './FollowControl.svelte'; +import { + createFollowQuery, + createReleaseProfilesQuery, + deleteFollow, + putFollow +} from '#lib/api/wanted.js'; + +const profiles: ReleaseProfilesResponse = { + profiles: [ + { id: 'p-std', name: 'Standard', primary_types: ['Album'], secondary_types: [], is_default: true, follow_count: 0 }, + { id: 'p-live', name: 'With live', primary_types: ['Album'], secondary_types: ['Live'], is_default: false, follow_count: 0 } + ], + primary_types: ['Album'], + secondary_types: ['Live'] +}; + +afterEach(() => vi.clearAllMocks()); + +function setup(state: FollowState) { + vi.mocked(createFollowQuery).mockReturnValue( + mockQuery({ data: state }) as unknown as ReturnType + ); + vi.mocked(createReleaseProfilesQuery).mockReturnValue( + mockQuery({ data: profiles }) as unknown as ReturnType + ); + return render(FollowControl, { artistMbid: 'mb-blur', artistName: 'Blur' }); +} + +describe('FollowControl', () => { + test('follows on the default profile, wanting what is already released when asked', async () => { + setup({ following: false }); + await fireEvent.click(screen.getByRole('button', { name: /^follow$/i })); + expect((screen.getByLabelText(/release-type profile/i) as HTMLSelectElement).value).toBe('p-std'); + await fireEvent.click(screen.getByLabelText(/also want what they've released/i)); + const buttons = screen.getAllByRole('button', { name: /^follow$/i }); + await fireEvent.click(buttons[buttons.length - 1]); + await waitFor(() => + expect(putFollow).toHaveBeenCalledWith('mb-blur', { + artist_name: 'Blur', + profile_id: 'p-std', + want_existing: true + }) + ); + }); + + test('a followed artist shows its profile and can be unfollowed', async () => { + setup({ following: true, profile_id: 'p-live', profile_name: 'With live' }); + await fireEvent.click(screen.getByRole('button', { name: /following · with live/i })); + expect((screen.getByLabelText(/release-type profile/i) as HTMLSelectElement).value).toBe('p-live'); + await fireEvent.click(screen.getByRole('button', { name: /unfollow/i })); + await waitFor(() => expect(deleteFollow).toHaveBeenCalledWith('mb-blur')); + }); +}); diff --git a/web/src/lib/components/ReacquisitionSettingsCard.svelte b/web/src/lib/components/ReacquisitionSettingsCard.svelte index c9e16643..3c210dbc 100644 --- a/web/src/lib/components/ReacquisitionSettingsCard.svelte +++ b/web/src/lib/components/ReacquisitionSettingsCard.svelte @@ -9,10 +9,11 @@ import { errMessage } from '#lib/api/errors.js'; import { pushToast } from '#lib/stores/toast.svelte.js'; - // Policy for turning a missing file back into a Lidarr request - // (milestone #290). Lives on the missing-files page rather than under - // Integrations because this is where the operator meets the problem it - // solves; Lidarr is the mechanism, not the subject. + // Policy for putting an album whose files went missing back on the wanted + // list (milestone #290, M502 #5520). Lives on the missing-files page rather + // than under Integrations because this is where the operator meets the + // problem it solves. Searching for a want, and retrying, is the search + // schedule's job, so there is no retry policy here. let saved = $state(null); let form = $state(null); @@ -31,11 +32,7 @@ return { enabled: s.enabled, grace_hours: s.grace_hours, - backoff_base_hours: s.backoff_base_hours, - backoff_max_hours: s.backoff_max_hours, - max_attempts: s.max_attempts, - max_per_pass: s.max_per_pass, - auto_approve: s.auto_approve + max_per_pass: s.max_per_pass }; } @@ -70,28 +67,16 @@ } } - // Plain-language echo of what the numbers add up to. The individual fields - // are meaningless on their own — "6 hours" says nothing until you know it - // doubles — so the card states the resulting schedule. - const schedule = $derived.by(() => { - if (!form) return ''; - const steps: string[] = []; - let hours = form.backoff_base_hours; - for (let i = 0; i < form.max_attempts; i++) { - steps.push(`${hours}h`); - hours = Math.min(hours * 2, form.backoff_max_hours); - } - return steps.join(' → '); - });

    Automatic re-acquisition

    - When a file has been missing for a while, Minstrel can ask Lidarr for the album - again by itself. Requests are made per album, not per track — a whole folder - going missing is one request, not forty. + When a file has been missing for a while, Minstrel can put its album back on the + wanted list + by itself. It's wanted per album, not per track: a whole folder going missing is one + want, not forty.

    @@ -106,7 +91,7 @@

    - Leave fields blank to use the saved defaults. + The profile says which kinds of release following + {profileRow?.artist_name ?? 'this artist'} wants. Profiles are edited under Integrations.

    - - @@ -299,17 +274,17 @@
    diff --git a/web/src/routes/admin/requests/requests.test.ts b/web/src/routes/admin/requests/requests.test.ts index 8b24e597..251674ac 100644 --- a/web/src/routes/admin/requests/requests.test.ts +++ b/web/src/routes/admin/requests/requests.test.ts @@ -1,34 +1,30 @@ import { afterEach, describe, expect, test, vi } from 'vitest'; import { render, screen, fireEvent, waitFor } from '@testing-library/svelte'; import { mockQuery } from '../../../test-utils/query'; -import type { LidarrRequest } from '#lib/api/types.js'; +import type { MusicRequest } from '#lib/api/types.js'; // Wrap useQueryClient so the page can call invalidateQueries() without a // real QueryClient context. Everything else from svelte-query passes through. vi.mock('#lib/api/admin.js', () => ({ createAdminRequestsQuery: vi.fn(), - createQualityProfilesQuery: vi.fn(), - createRootFoldersQuery: vi.fn(), approveRequest: vi.fn(), rejectRequest: vi.fn() })); +vi.mock('#lib/api/wanted.js', () => ({ + createReleaseProfilesQuery: vi.fn() +})); import AdminRequestsPage from './+page.svelte'; -import { - createAdminRequestsQuery, - createQualityProfilesQuery, - createRootFoldersQuery, - approveRequest, - rejectRequest -} from '#lib/api/admin.js'; +import { createAdminRequestsQuery, approveRequest, rejectRequest } from '#lib/api/admin.js'; +import { createReleaseProfilesQuery } from '#lib/api/wanted.js'; -const baseRow: LidarrRequest = { +const baseRow: MusicRequest = { id: 'r1', user_id: 'u1234567-aaaa-bbbb-cccc-deadbeef0001', status: 'pending', kind: 'album', - lidarr_artist_mbid: 'a-mbid', - lidarr_album_mbid: 'al-mbid', + artist_mbid: 'a-mbid', + album_mbid: 'al-mbid', artist_name: 'Boards of Canada', album_title: 'Geogaddi', requested_at: '2026-04-29T10:00:00Z', @@ -39,15 +35,21 @@ const baseRow: LidarrRequest = { afterEach(() => vi.clearAllMocks()); -function setup(rows: LidarrRequest[] = [baseRow]) { +function setup(rows: MusicRequest[] = [baseRow]) { (createAdminRequestsQuery as ReturnType).mockReturnValue( mockQuery({ data: rows }) ); - (createQualityProfilesQuery as ReturnType).mockReturnValue( - mockQuery({ data: [{ id: 1, name: 'Standard' }] }) - ); - (createRootFoldersQuery as ReturnType).mockReturnValue( - mockQuery({ data: [{ path: '/music', accessible: true, free_space: 0 }] }) + (createReleaseProfilesQuery as ReturnType).mockReturnValue( + mockQuery({ + data: { + profiles: [ + { id: 'p-std', name: 'Standard', primary_types: ['Album'], secondary_types: [], is_default: true, follow_count: 0 }, + { id: 'p-live', name: 'With live', primary_types: ['Album'], secondary_types: ['Live'], is_default: false, follow_count: 0 } + ], + primary_types: ['Album'], + secondary_types: ['Live'] + } + }) ); return render(AdminRequestsPage); } @@ -59,30 +61,29 @@ describe('/admin/requests', () => { expect(createAdminRequestsQuery).toHaveBeenCalledWith('approved'); }); - test('Approve without override fires POST with no override body', async () => { + test('Approve sends no profile', async () => { setup(); (approveRequest as ReturnType).mockResolvedValueOnce(baseRow); await fireEvent.click(screen.getByRole('button', { name: /approve geogaddi/i })); - expect(approveRequest).toHaveBeenCalledWith('r1'); + expect(approveRequest).toHaveBeenCalledWith('r1', undefined); }); - test('Override modal Confirm sends chosen values', async () => { + test('An album request has no profile choice', () => { setup(); - await fireEvent.click(screen.getByRole('button', { name: /override geogaddi/i })); - await fireEvent.change(screen.getByLabelText(/quality profile/i), { - target: { value: '1' } - }); - await fireEvent.change(screen.getByLabelText(/root folder/i), { - target: { value: '/music' } - }); - (approveRequest as ReturnType).mockResolvedValueOnce(baseRow); + expect(screen.queryByRole('button', { name: /on a profile/i })).toBeNull(); + }); + + test('An artist request is approved on the chosen profile', async () => { + setup([{ ...baseRow, kind: 'artist', album_mbid: undefined, album_title: undefined }]); await fireEvent.click( - screen.getByRole('button', { name: /approve with override/i }) + screen.getByRole('button', { name: /approve boards of canada on a profile/i }) ); - expect(approveRequest).toHaveBeenCalledWith('r1', { - quality_profile_id: 1, - root_folder_path: '/music' - }); + const select = screen.getByLabelText(/release-type profile/i) as HTMLSelectElement; + expect(select.value).toBe('p-std'); + await fireEvent.change(select, { target: { value: 'p-live' } }); + (approveRequest as ReturnType).mockResolvedValueOnce(baseRow); + await fireEvent.click(screen.getByRole('button', { name: /approve and follow/i })); + expect(approveRequest).toHaveBeenCalledWith('r1', 'p-live'); }); test('Reject opens notes textarea; confirm sends notes', async () => { @@ -100,16 +101,16 @@ describe('/admin/requests', () => { expect(rejectRequest).toHaveBeenCalledWith('r1', 'duplicate'); }); - test('Lidarr-unreachable error shows toast', async () => { + test('An approve error shows a toast', async () => { setup(); (approveRequest as ReturnType).mockRejectedValueOnce({ - code: 'lidarr_unreachable', - message: 'unreachable', - status: 503 + code: 'request_not_pending', + message: 'not pending', + status: 409 }); await fireEvent.click(screen.getByRole('button', { name: /approve geogaddi/i })); await waitFor(() => - expect(screen.getByText(/lidarr is unreachable/i)).toBeInTheDocument() + expect(screen.getByText(/no longer pending/i)).toBeInTheDocument() ); }); @@ -122,6 +123,6 @@ describe('/admin/requests', () => { album_title: 'Geogaddi' } ]); - expect(screen.getByText(/will add the album/i)).toBeInTheDocument(); + expect(screen.getByText(/wants the album/i)).toBeInTheDocument(); }); }); diff --git a/web/src/routes/admin/wanted/+page.svelte b/web/src/routes/admin/wanted/+page.svelte new file mode 100644 index 00000000..1fed0c87 --- /dev/null +++ b/web/src/routes/admin/wanted/+page.svelte @@ -0,0 +1,268 @@ + + +{pageTitle('Admin · Wanted')} + +
    +
    +

    Wanted

    +

    + Albums Minstrel is to get: ones users requested, new releases from the artists it follows, + and albums whose files went missing. An album leaves the list once the library holds it. + Minstrel doesn't search indexers for these yet; that comes next. +

    +
    + + + + {#if tab === 'following'} +

    + Each artist is followed on a release-type profile, which says which kinds of release are + wanted. Profiles are edited under + Integrations. +

    + {#if follows.isPending} +

    Reading who's followed…

    + {:else if follows.isError} +

    Couldn't load the followed artists.

    + {:else if (follows.data ?? []).length === 0} +

    + No artists are followed. Follow one from its artist page. +

    + {:else} +
      + {#each follows.data ?? [] as f (f.artist_mbid)} +
    • + +
      +
      + {#if f.artist_id} + {f.artist_name} + {:else} + {f.artist_name} + {/if} +
      +
      + {f.open_wants === 1 ? '1 album wanted' : `${f.open_wants} albums wanted`} · + {f.checked_at ? `checked ${relativeTime(f.checked_at)}` : 'not checked yet'} +
      +
      + + +
      • + {/each} +
    + {/if} + {:else if wants.isPending} +

    Reading the wanted list…

    + {:else if wants.isError && !wants.isFetchNextPageError} +

    Couldn't load the wanted list.

    + {:else if items.length === 0} +
    + +

    + {#if tab === 'held'}Nothing on the list has been held yet. + {:else if tab === 'missing'}No missing albums are wanted back. + {:else}Nothing is wanted.{/if} +

    +

    + {#if tab === 'missing'} + An album whose files stay missing past the grace window on the Missing files page lands here. + {:else} + A request you approve, a new release from a followed artist, or an album whose files went + missing lands here. + {/if} +

    +
    + {:else} +
      + {#each items as w (w.id)} +
    • + +
      +
      + {#if w.album_id && w.held_at} + {w.title} + {:else} + {w.title} + {/if} +
      +
      + {#if w.artist_id} + {w.artist_name} + {:else} + {w.artist_name} + {/if} + {#if kind(w)} · {kind(w)}{/if} +
      +
      + {why(w)} · {w.held_at ? `held ${relativeTime(w.held_at)}` : `wanted ${relativeTime(w.wanted_at)}`} +
      +
      + {#if !w.held_at} + + {/if} +
      • + {/each} +
    + + wants.fetchNextPage()} + /> + {/if} +
    diff --git a/web/src/routes/admin/wanted/wanted.test.ts b/web/src/routes/admin/wanted/wanted.test.ts new file mode 100644 index 00000000..28ac7e2c --- /dev/null +++ b/web/src/routes/admin/wanted/wanted.test.ts @@ -0,0 +1,145 @@ +import { afterEach, describe, expect, test, vi } from 'vitest'; +import { fireEvent, render, screen, waitFor } from '@testing-library/svelte'; +import { mockInfiniteQuery, mockQuery } from '../../../test-utils/query'; +import type { Follow, ReleaseProfilesResponse, Want, WantsPage } from '#lib/api/types.js'; + +vi.mock('#lib/api/wanted.js', () => ({ + createWantsQuery: vi.fn(), + createFollowsQuery: vi.fn(), + createReleaseProfilesQuery: vi.fn(), + removeWant: vi.fn().mockResolvedValue(undefined), + putFollow: vi.fn().mockResolvedValue({ following: true }), + deleteFollow: vi.fn().mockResolvedValue(undefined) +})); + +import AdminWantedPage from './+page.svelte'; +import { + createFollowsQuery, + createReleaseProfilesQuery, + createWantsQuery, + deleteFollow, + putFollow, + removeWant +} from '#lib/api/wanted.js'; + +const DAY = 24 * 3_600_000; +const ago = (ms: number) => new Date(Date.now() - ms).toISOString(); + +function want(id: string, extra: Partial = {}): Want { + return { + id, + release_group_mbid: `rg-${id}`, + title: 'The Ballad of Darren', + artist_mbid: 'blur', + artist_name: 'Blur', + artist_id: 'art-1', + primary_type: 'Album', + secondary_types: [], + year: '2023', + reason: 'follow', + wanted_at: ago(2 * DAY), + held_at: null, + image_url: '', + ...extra + }; +} + +function page(items: Want[]): WantsPage { + return { + items, + total: items.length, + limit: 50, + offset: 0, + counts: { wanted: 2, missing: 1, held: 4, following: 1 } + }; +} + +const follow: Follow = { + artist_mbid: 'blur', + artist_name: 'Blur', + artist_id: 'art-1', + profile_id: 'p-std', + profile_name: 'Standard', + followed_at: ago(10 * DAY), + checked_at: ago(3 * 3_600_000), + open_wants: 2, + image_url: '' +}; + +const profiles: ReleaseProfilesResponse = { + profiles: [ + { id: 'p-std', name: 'Standard', primary_types: ['Album'], secondary_types: [], is_default: true, follow_count: 1 }, + { id: 'p-live', name: 'With live', primary_types: ['Album'], secondary_types: ['Live'], is_default: false, follow_count: 0 } + ], + primary_types: ['Album'], + secondary_types: ['Live'] +}; + +afterEach(() => vi.clearAllMocks()); + +function setup(items: Want[] = [want('w1'), want('w2', { reason: 'request', requested_by: 'sam', title: 'Parklife' })]) { + vi.mocked(createWantsQuery).mockReturnValue( + mockInfiniteQuery({ pages: [page(items)] }) as unknown as ReturnType + ); + vi.mocked(createFollowsQuery).mockReturnValue( + mockQuery({ data: [follow] }) as unknown as ReturnType + ); + vi.mocked(createReleaseProfilesQuery).mockReturnValue( + mockQuery({ data: profiles }) as unknown as ReturnType + ); + return render(AdminWantedPage); +} + +describe('/admin/wanted', () => { + test('each tab shows its count', () => { + setup(); + expect(screen.getByRole('tab', { name: /wanted/i }).textContent).toMatch(/2/); + expect(screen.getByRole('tab', { name: /held/i }).textContent).toMatch(/4/); + }); + + test('a want says why it is wanted', () => { + setup(); + const why = screen.getAllByTestId('want-why').map((e) => e.textContent ?? ''); + expect(why[0]).toMatch(/new from a followed artist/i); + expect(why[1]).toMatch(/requested by sam/i); + }); + + test('switching tab asks for that state', async () => { + setup(); + await fireEvent.click(screen.getByRole('tab', { name: /missing files/i })); + expect(createWantsQuery).toHaveBeenCalledWith('missing'); + }); + + test('remove takes a want off the list', async () => { + setup(); + await fireEvent.click(screen.getByRole('button', { name: /stop wanting parklife/i })); + await waitFor(() => expect(removeWant).toHaveBeenCalledWith('w2')); + }); + + test('a held want has no remove button', () => { + setup([want('w1', { held_at: ago(DAY), album_id: 'al-1' })]); + expect(screen.queryByRole('button', { name: /stop wanting/i })).toBeNull(); + expect(screen.getByRole('link', { name: /the ballad of darren/i }).getAttribute('href')).toBe('/albums/al-1'); + }); + + test('following lists artists; the profile can change and the artist be unfollowed', async () => { + setup(); + await fireEvent.click(screen.getByRole('tab', { name: /following/i })); + expect(screen.getAllByTestId('follow-row')).toHaveLength(1); + + const select = screen.getByLabelText(/profile for blur/i) as HTMLSelectElement; + expect(select.value).toBe('p-std'); + await fireEvent.change(select, { target: { value: 'p-live' } }); + await waitFor(() => + expect(putFollow).toHaveBeenCalledWith('blur', { artist_name: 'Blur', profile_id: 'p-live' }) + ); + + await fireEvent.click(screen.getByRole('button', { name: /unfollow blur/i })); + await waitFor(() => expect(deleteFollow).toHaveBeenCalledWith('blur')); + }); + + test('an empty list explains what lands there', () => { + setup([]); + expect(screen.getByText(/nothing is wanted/i)).toBeTruthy(); + }); +}); diff --git a/web/src/routes/artists/[id]/+page.svelte b/web/src/routes/artists/[id]/+page.svelte index a86463d5..ba564750 100644 --- a/web/src/routes/artists/[id]/+page.svelte +++ b/web/src/routes/artists/[id]/+page.svelte @@ -13,6 +13,8 @@ import CompactTrackCard from '#lib/components/CompactTrackCard.svelte'; import HorizontalScrollRow from '#lib/components/HorizontalScrollRow.svelte'; import LikeButton from '#lib/components/LikeButton.svelte'; + import FollowControl from '#lib/components/FollowControl.svelte'; + import { user } from '#lib/auth/store.svelte.js'; import LibrarySkeleton from '#lib/components/LibrarySkeleton.svelte'; import ApiErrorBanner from '#lib/components/ApiErrorBanner.svelte'; import { useDelayed } from '#lib/utils/useDelayed.svelte.js'; @@ -22,6 +24,7 @@ import type { ArtistRef, TrackRef } from '#lib/api/types.js'; const id = $derived(page.params.id ?? ''); + const isAdmin = $derived(user.value?.is_admin === true); const queryStore = $derived(createArtistQuery(id)); const query = $derived($queryStore); const showSkeleton = useDelayed(() => query.isPending); @@ -103,13 +106,18 @@ {/each}
{/if} -
+
+ + {#if isAdmin && detail.mbid} + + {/if}
diff --git a/web/src/routes/artists/[id]/artist.test.ts b/web/src/routes/artists/[id]/artist.test.ts index 640348e2..5962d425 100644 --- a/web/src/routes/artists/[id]/artist.test.ts +++ b/web/src/routes/artists/[id]/artist.test.ts @@ -25,6 +25,24 @@ vi.mock('#lib/api/queries.js', () => ({ vi.mock('#lib/api/likes.js', () => emptyLikesMock()); +const userState = vi.hoisted(() => ({ + current: null as { id: string; username: string; is_admin: boolean } | null +})); + +vi.mock('#lib/auth/store.svelte.js', () => ({ + user: { get value() { return userState.current; } } +})); + +vi.mock('#lib/api/wanted.js', async () => { + const { mockQuery } = await import('../../../test-utils/query'); + return { + createFollowQuery: vi.fn(() => mockQuery({ data: { following: false } })), + createReleaseProfilesQuery: vi.fn(() => mockQuery({ data: { profiles: [], primary_types: [], secondary_types: [] } })), + putFollow: vi.fn(), + deleteFollow: vi.fn() + }; +}); + vi.mock('#lib/api/client.js', () => apiClientMock()); vi.mock('#lib/player/store.svelte.js', () => ({ @@ -59,6 +77,7 @@ function album(id: string, title: string, year?: number): AlbumRef { afterEach(() => { vi.clearAllMocks(); state.pageParams = { id: 'abc' }; + userState.current = null; }); beforeEach(() => { @@ -73,7 +92,7 @@ describe('artist detail page', () => { const detail: ArtistDetail = { id: 'abc', name: 'Alice', sort_name: 'Alice', album_count: 2, cover_url: '', albums: [album('a1', 'First', 2020), album('a2', 'Second')], - genres: [] + genres: [], mbid: '' }; (createArtistQuery as ReturnType).mockReturnValue(mockQuery({ data: detail })); render(ArtistPage); @@ -87,7 +106,7 @@ describe('artist detail page', () => { const detail: ArtistDetail = { id: 'abc', name: 'Alice', sort_name: 'Alice', album_count: 1, cover_url: '', albums: [album('a1', 'First', 2020)], - genres: [] + genres: [], mbid: '' }; (createArtistQuery as ReturnType).mockReturnValue(mockQuery({ data: detail })); (createArtistTopTracksQuery as ReturnType).mockReturnValue(mockQuery({ @@ -154,7 +173,7 @@ describe('artist detail page', () => { describe('Play and Shuffle', () => { const detail: ArtistDetail = { id: 'abc', name: 'Alice', sort_name: 'Alice', album_count: 1, cover_url: '', - albums: [album('a1', 'First', 2020)], genres: [] + albums: [album('a1', 'First', 2020)], genres: [], mbid: '' }; const tracks = makeTracks(5); @@ -176,4 +195,34 @@ describe('artist detail page', () => { expect(queued.map((t: { id: string }) => t.id).sort()).toEqual(tracks.map((t) => t.id).sort()); }); }); + + // Following is for the wanted list (M502 #5520): an admin control, keyed by + // the artist's MusicBrainz id. + describe('Follow', () => { + const detail = (mbid: string): ArtistDetail => ({ + id: 'abc', name: 'Alice', sort_name: 'Alice', album_count: 1, cover_url: '', + albums: [album('a1', 'First', 2020)], genres: [], mbid + }); + + test('an admin can follow an artist with a MusicBrainz id', () => { + userState.current = { id: 'u1', username: 'admin', is_admin: true }; + (createArtistQuery as ReturnType).mockReturnValue(mockQuery({ data: detail('mb-alice') })); + render(ArtistPage); + expect(screen.getByRole('button', { name: /^follow$/i })).toBeInTheDocument(); + }); + + test('no follow control without a MusicBrainz id', () => { + userState.current = { id: 'u1', username: 'admin', is_admin: true }; + (createArtistQuery as ReturnType).mockReturnValue(mockQuery({ data: detail('') })); + render(ArtistPage); + expect(screen.queryByRole('button', { name: /^follow$/i })).toBeNull(); + }); + + test('no follow control for a non-admin', () => { + userState.current = { id: 'u2', username: 'sam', is_admin: false }; + (createArtistQuery as ReturnType).mockReturnValue(mockQuery({ data: detail('mb-alice') })); + render(ArtistPage); + expect(screen.queryByRole('button', { name: /^follow$/i })).toBeNull(); + }); + }); }); diff --git a/web/src/routes/discover/+page.svelte b/web/src/routes/discover/+page.svelte index 1247c0d2..f463afd7 100644 --- a/web/src/routes/discover/+page.svelte +++ b/web/src/routes/discover/+page.svelte @@ -11,7 +11,7 @@ import Modal from '#lib/components/Modal.svelte'; import type { CatalogueSearchResult, - LidarrRequestKind + RequestKind } from '#lib/api/types.js'; const queryClient = useQueryClient(); @@ -21,7 +21,7 @@ // pushing into `debouncedQ`, which is what the query factory observes. let inputValue = $state(''); let debouncedQ = $state(''); - let activeKind: LidarrRequestKind = $state('artist'); + let activeKind: RequestKind = $state('artist'); let debounceTimer: ReturnType | null = null; $effect(() => { @@ -56,19 +56,19 @@ // Backend validates kind→required-MBID-fields, not the human strings, so // imperfect derivations of artist_name from secondary_text won't block the // request. We pass the data we have and let the server take care of the rest. - function buildRequestParams(r: CatalogueSearchResult, kind: LidarrRequestKind) { + function buildRequestParams(r: CatalogueSearchResult, kind: RequestKind) { if (kind === 'artist') { return { kind, - lidarr_artist_mbid: r.artist_mbid || r.mbid, + artist_mbid: r.artist_mbid || r.mbid, artist_name: r.name }; } if (kind === 'album') { return { kind, - lidarr_artist_mbid: r.artist_mbid, - lidarr_album_mbid: r.album_mbid || r.mbid, + artist_mbid: r.artist_mbid, + album_mbid: r.album_mbid || r.mbid, artist_name: r.secondary_text, album_title: r.name }; @@ -76,9 +76,9 @@ // track return { kind, - lidarr_artist_mbid: r.artist_mbid, - lidarr_album_mbid: r.album_mbid, - lidarr_track_mbid: r.mbid, + artist_mbid: r.artist_mbid, + album_mbid: r.album_mbid, + track_mbid: r.mbid, artist_name: r.secondary_text, album_title: r.secondary_text, track_title: r.name @@ -116,7 +116,7 @@ modalResult = null; } - const tabs: { kind: LidarrRequestKind; label: string }[] = [ + const tabs: { kind: RequestKind; label: string }[] = [ { kind: 'artist', label: 'Artists' }, { kind: 'album', label: 'Albums' }, { kind: 'track', label: 'Tracks' } diff --git a/web/src/routes/discover/discover.test.ts b/web/src/routes/discover/discover.test.ts index 42dfe7e3..271365cb 100644 --- a/web/src/routes/discover/discover.test.ts +++ b/web/src/routes/discover/discover.test.ts @@ -263,9 +263,9 @@ describe('Discover page', () => { expect(mockedCreateRequest).toHaveBeenCalledWith( expect.objectContaining({ kind: 'track', - lidarr_track_mbid: 'tr-mbid', - lidarr_album_mbid: 'al-mbid', - lidarr_artist_mbid: 'art-mbid' + track_mbid: 'tr-mbid', + album_mbid: 'al-mbid', + artist_mbid: 'art-mbid' }) ); }); diff --git a/web/src/routes/requests/+page.svelte b/web/src/routes/requests/+page.svelte index 90fba3b8..84464ff3 100644 --- a/web/src/routes/requests/+page.svelte +++ b/web/src/routes/requests/+page.svelte @@ -7,11 +7,11 @@ import StatusPill from '#lib/components/StatusPill.svelte'; import ApiErrorBanner from '#lib/components/ApiErrorBanner.svelte'; import DiscoverTabs from '#lib/components/DiscoverTabs.svelte'; - import type { LidarrRequest, LidarrRequestKind } from '#lib/api/types.js'; + import type { MusicRequest, RequestKind } from '#lib/api/types.js'; const queryStore = createMyRequestsQuery(); const query = $derived($queryStore); - const requests = $derived((query.data ?? []) as LidarrRequest[]); + const requests = $derived((query.data ?? []) as MusicRequest[]); const client = useQueryClient(); @@ -25,13 +25,13 @@ } } - function fallbackIcon(kind: LidarrRequestKind) { + function fallbackIcon(kind: RequestKind) { if (kind === 'artist') return Disc3; if (kind === 'album') return Album; return Music2; } - function rowTitle(r: LidarrRequest): string { + function rowTitle(r: MusicRequest): string { if (r.kind === 'artist') return r.artist_name; if (r.kind === 'album') return r.album_title ?? '—'; return r.track_title ?? '—'; @@ -39,7 +39,7 @@ // Used in aria-labels — '—' is meaningless to screen readers, so when a // title is missing fall back to a generic-but-intelligible phrase. - function rowAccessibleName(r: LidarrRequest): string { + function rowAccessibleName(r: MusicRequest): string { if (r.kind === 'artist') return r.artist_name; if (r.kind === 'album') return r.album_title ?? `this album by ${r.artist_name}`; return r.track_title ?? `this track by ${r.artist_name}`; @@ -48,13 +48,13 @@ // Keep the meta line short and readable. We always lead with the artist, // then a relative-ish date — locale formatting is good enough for v1 // and keeps tests deterministic without pinning Intl.RelativeTimeFormat. - function rowMeta(r: LidarrRequest): string { + function rowMeta(r: MusicRequest): string { const when = new Date(r.requested_at).toLocaleDateString(); if (r.kind === 'artist') return `Requested ${when}`; return `by ${r.artist_name} · ${when}`; } - function listenHref(r: LidarrRequest): string | null { + function listenHref(r: MusicRequest): string | null { if (r.matched_track_id) return `/tracks/${r.matched_track_id}`; if (r.matched_album_id) return `/albums/${r.matched_album_id}`; if (r.matched_artist_id) return `/artists/${r.matched_artist_id}`; diff --git a/web/src/routes/requests/requests.test.ts b/web/src/routes/requests/requests.test.ts index 49b141ed..8dc11509 100644 --- a/web/src/routes/requests/requests.test.ts +++ b/web/src/routes/requests/requests.test.ts @@ -2,7 +2,7 @@ import { afterEach, describe, expect, test, vi } from 'vitest'; import { render, screen, fireEvent, waitFor } from '@testing-library/svelte'; import { mockQuery } from '../../test-utils/query'; import { apiClientMock } from '../../test-utils/mocks/client'; -import type { LidarrRequest } from '#lib/api/types.js'; +import type { MusicRequest } from '#lib/api/types.js'; import { pageUrlModule } from '../../test-utils/mocks/appState'; // $app/state / $app/navigation must be mocked or rendering the page @@ -43,20 +43,18 @@ import { qk } from '#lib/api/queries.js'; const mockedCreateMyRequestsQuery = createMyRequestsQuery as ReturnType; const mockedCancelRequest = cancelRequest as ReturnType; -function req(over: Partial = {}): LidarrRequest { +function req(over: Partial = {}): MusicRequest { return { id: 'r1', user_id: 'u1', status: 'pending', kind: 'album', - lidarr_artist_mbid: 'art-mbid', - lidarr_album_mbid: 'al-mbid', - lidarr_track_mbid: null, + artist_mbid: 'art-mbid', + album_mbid: 'al-mbid', + track_mbid: null, artist_name: 'Boards of Canada', album_title: 'Music Has The Right To Children', track_title: null, - quality_profile_id: null, - root_folder_path: null, decided_at: null, decided_by: null, notes: null, @@ -78,12 +76,12 @@ afterEach(() => { describe('Requests page', () => { test('renders one row per request from the API', () => { - const rows: LidarrRequest[] = [ + const rows: MusicRequest[] = [ req({ id: 'r1', kind: 'artist', artist_name: 'Boards of Canada' }), req({ id: 'r2', kind: 'album', album_title: 'Geogaddi', status: 'completed' }), req({ id: 'r3', kind: 'track', track_title: 'Roygbiv', status: 'rejected', notes: 'Already in library' }) ]; - mockedCreateMyRequestsQuery.mockReturnValue(mockQuery({ data: rows })); + mockedCreateMyRequestsQuery.mockReturnValue(mockQuery({ data: rows })); render(RequestsPage); expect(screen.getAllByTestId('request-row')).toHaveLength(3); expect(screen.getByText('Boards of Canada')).toBeInTheDocument(); @@ -92,10 +90,10 @@ describe('Requests page', () => { }); test('pending row exposes Cancel; clicking calls cancelRequest and invalidates myRequests', async () => { - const rows: LidarrRequest[] = [ + const rows: MusicRequest[] = [ req({ id: 'r-pending', status: 'pending', kind: 'album', album_title: 'Geogaddi' }) ]; - mockedCreateMyRequestsQuery.mockReturnValue(mockQuery({ data: rows })); + mockedCreateMyRequestsQuery.mockReturnValue(mockQuery({ data: rows })); render(RequestsPage); const cancelBtn = screen.getByRole('button', { name: /cancel request for geogaddi/i }); await fireEvent.click(cancelBtn); @@ -106,7 +104,7 @@ describe('Requests page', () => { }); test('completed row with matched_track_id renders Listen link to /tracks/', () => { - const rows: LidarrRequest[] = [ + const rows: MusicRequest[] = [ req({ id: 'r-track', status: 'completed', @@ -117,14 +115,14 @@ describe('Requests page', () => { matched_artist_id: 'art-9' }) ]; - mockedCreateMyRequestsQuery.mockReturnValue(mockQuery({ data: rows })); + mockedCreateMyRequestsQuery.mockReturnValue(mockQuery({ data: rows })); render(RequestsPage); const link = screen.getByRole('link', { name: /listen to roygbiv/i }); expect(link).toHaveAttribute('href', '/tracks/tr-9'); }); test('completed row with only matched_album_id renders Listen link to /albums/', () => { - const rows: LidarrRequest[] = [ + const rows: MusicRequest[] = [ req({ id: 'r-album', status: 'completed', @@ -135,14 +133,14 @@ describe('Requests page', () => { matched_artist_id: 'art-7' }) ]; - mockedCreateMyRequestsQuery.mockReturnValue(mockQuery({ data: rows })); + mockedCreateMyRequestsQuery.mockReturnValue(mockQuery({ data: rows })); render(RequestsPage); const link = screen.getByRole('link', { name: /listen to geogaddi/i }); expect(link).toHaveAttribute('href', '/albums/al-7'); }); test('rejected row renders notes and hides Cancel and Listen', () => { - const rows: LidarrRequest[] = [ + const rows: MusicRequest[] = [ req({ id: 'r-rej', status: 'rejected', @@ -151,7 +149,7 @@ describe('Requests page', () => { notes: 'Already kept in the library.' }) ]; - mockedCreateMyRequestsQuery.mockReturnValue(mockQuery({ data: rows })); + mockedCreateMyRequestsQuery.mockReturnValue(mockQuery({ data: rows })); render(RequestsPage); expect(screen.getByTestId('rejection-notes')).toHaveTextContent('Already kept in the library.'); expect(screen.queryByRole('button', { name: /cancel request/i })).not.toBeInTheDocument(); @@ -159,7 +157,7 @@ describe('Requests page', () => { }); test('empty list shows the voice-rule "Nothing requested yet." copy', () => { - mockedCreateMyRequestsQuery.mockReturnValue(mockQuery({ data: [] })); + mockedCreateMyRequestsQuery.mockReturnValue(mockQuery({ data: [] })); render(RequestsPage); expect(screen.getByText(/nothing requested yet\./i)).toBeInTheDocument(); expect(screen.queryAllByTestId('request-row')).toHaveLength(0); -- 2.54.0 From 1a1646a9205ef9c32c009df2245b5a354dc6a8e7 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Sat, 10 Oct 2026 00:54:01 -0400 Subject: [PATCH 009/117] fix: the wanted-list tests set a track's MBID with its source, and find the missing-files link in its row (M502 #5520) Co-Authored-By: Claude Opus 5.5 --- internal/requests/requests_test.go | 2 +- web/src/routes/admin/missing-files/missing-files.test.ts | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/internal/requests/requests_test.go b/internal/requests/requests_test.go index a0389a92..5b1b9004 100644 --- a/internal/requests/requests_test.go +++ b/internal/requests/requests_test.go @@ -227,7 +227,7 @@ func TestReconciler_CompletesWhenTheWantIsHeld(t *testing.T) { artist, album := seedAlbum(t, q, "Notify Artist", "artist-4", "Notify Album", "group-notify") _ = seedTrack(t, q, album, artist, "Other Track", "/music/notify/01.flac") wantedTrack := seedTrack(t, q, album, artist, "Wanted Track", "/music/notify/02.flac") - if _, err := pool.Exec(ctx, "UPDATE tracks SET mbid = 'rec-wanted', added_at = added_at - interval '1 hour' WHERE id = $1", wantedTrack.ID); err != nil { + if _, err := pool.Exec(ctx, "UPDATE tracks SET mbid = 'rec-wanted', mbid_source = 'tag', added_at = added_at - interval '1 hour' WHERE id = $1", wantedTrack.ID); err != nil { t.Fatal(err) } if _, err := pool.Exec(ctx, "UPDATE wants SET fulfilled_at = now(), album_id = $1 WHERE release_group_mbid = 'group-notify'", album.ID); err != nil { diff --git a/web/src/routes/admin/missing-files/missing-files.test.ts b/web/src/routes/admin/missing-files/missing-files.test.ts index 7ac39102..e8adf9b7 100644 --- a/web/src/routes/admin/missing-files/missing-files.test.ts +++ b/web/src/routes/admin/missing-files/missing-files.test.ts @@ -1,5 +1,5 @@ import { afterEach, describe, expect, test, vi } from 'vitest'; -import { render, screen } from '@testing-library/svelte'; +import { render, screen, within } from '@testing-library/svelte'; import { mockQuery } from '../../../test-utils/query'; import type { AdminMissingResponse } from '#lib/api/types.js'; @@ -137,7 +137,9 @@ describe('admin missing files', () => { renderWith(withState); const line = text('reacquisition-state'); expect(line).toMatch(/on the wanted list since 2d ago/i); - expect(screen.getByRole('link', { name: /wanted list/i }).getAttribute('href')).toBe( + expect( + within(screen.getByTestId('reacquisition-state')).getByRole('link', { name: /wanted list/i }).getAttribute('href') + ).toBe( '/admin/wanted?tab=missing' ); }); -- 2.54.0 From 89b3b44779e28c604a8e4b2d7916ff88e42798e3 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Sat, 10 Oct 2026 08:56:47 -0400 Subject: [PATCH 010/117] =?UTF-8?q?feat:=20search=20and=20grab=20=E2=80=94?= =?UTF-8?q?=20music=20release=20names,=20a=20quality=20ladder,=20a=20paced?= =?UTF-8?q?=20search=20schedule=20and=20one=20live=20grab=20per=20want=20(?= =?UTF-8?q?M502=20#5521)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - internal/release: reads a release name (format, bitrate, bit depth and sample rate, source, year, video, discography) and judges it against the album wanted: the artist and title must be in it; video rips, discographies, other versions (drumless, instrumental, karaoke, remix, live unless the album is live…) and singles or EPs for an album are refused. Quality tiers on Lidarr's ladder; a .torrent's file list is read for format, video files and a track count short of the album. - internal/acquire: fan-out across every search source with a narrowed search kept, a process-wide pacer per source, every candidate judged and kept with its reasons, and the grab: one live grab per want, tie-broken (a person's grab beats the schedule's; a better release replaces a worse one unless it is nearly done). Each download lands in its own folder under the client's downloads folder. - internal/wanted: the search schedule, every ten minutes, never-searched first, then requests, missing files and followed artists, taking the best release when auto-grab is on (the default). - Migration 0082: acquisition_settings (auto-grab, interval, allowed qualities seeded from Lidarr's "Any"), want search columns, want_candidates, grabs, release_blocks. - Web: the Wanted page shows each want's last search, a Search button and every release found with its verdict (Grab, Grab anyway, Never this one, Allow again). Integrations gains "Searching and grabbing". Co-Authored-By: Claude Opus 5.5 --- cmd/minstrel/main.go | 6 + internal/acquire/build.go | 45 ++ internal/acquire/fanout.go | 165 +++++ internal/acquire/grab.go | 307 +++++++++ internal/acquire/memory.go | 168 +++++ internal/acquire/pace.go | 145 +++++ internal/acquire/search.go | 390 ++++++++++++ internal/acquire/search_test.go | 241 ++++++++ internal/api/admin_search.go | 299 +++++++++ internal/api/admin_wanted.go | 35 ++ internal/api/api.go | 11 + internal/db/dbq/models.go | 67 ++ internal/db/dbq/reacquisition.sql.go | 8 +- internal/db/dbq/search.sql.go | 585 ++++++++++++++++++ internal/db/dbq/wanted.sql.go | 50 +- .../migrations/0082_search_and_grab.down.sql | 12 + .../db/migrations/0082_search_and_grab.up.sql | 102 +++ internal/db/queries/search.sql | 121 ++++ internal/db/queries/wanted.sql | 6 +- internal/dbtest/reset.go | 11 + internal/release/files.go | 76 +++ internal/release/match.go | 210 +++++++ internal/release/parse.go | 289 +++++++++ internal/release/quality.go | 169 +++++ internal/release/release_test.go | 251 ++++++++ internal/release/torrent.go | 178 ++++++ internal/wanted/search.go | 322 ++++++++++ internal/wanted/search_test.go | 195 ++++++ web/src/lib/api/queries.ts | 3 + web/src/lib/api/types.ts | 95 +++ web/src/lib/api/wanted.ts | 60 ++ .../lib/components/SearchSettingsCard.svelte | 162 +++++ .../lib/components/SearchSettingsCard.test.ts | 69 +++ web/src/lib/components/WantCandidates.svelte | 184 ++++++ web/src/lib/components/WantCandidates.test.ts | 90 +++ web/src/lib/styles/error-copy.json | 8 +- .../routes/admin/integrations/+page.svelte | 3 + .../admin/integrations/integrations.test.ts | 17 +- web/src/routes/admin/wanted/+page.svelte | 173 ++++-- web/src/routes/admin/wanted/wanted.test.ts | 53 +- 40 files changed, 5332 insertions(+), 49 deletions(-) create mode 100644 internal/acquire/build.go create mode 100644 internal/acquire/fanout.go create mode 100644 internal/acquire/grab.go create mode 100644 internal/acquire/memory.go create mode 100644 internal/acquire/pace.go create mode 100644 internal/acquire/search.go create mode 100644 internal/acquire/search_test.go create mode 100644 internal/api/admin_search.go create mode 100644 internal/db/dbq/search.sql.go create mode 100644 internal/db/migrations/0082_search_and_grab.down.sql create mode 100644 internal/db/migrations/0082_search_and_grab.up.sql create mode 100644 internal/db/queries/search.sql create mode 100644 internal/release/files.go create mode 100644 internal/release/match.go create mode 100644 internal/release/parse.go create mode 100644 internal/release/quality.go create mode 100644 internal/release/release_test.go create mode 100644 internal/release/torrent.go create mode 100644 internal/wanted/search.go create mode 100644 internal/wanted/search_test.go create mode 100644 web/src/lib/components/SearchSettingsCard.svelte create mode 100644 web/src/lib/components/SearchSettingsCard.test.ts create mode 100644 web/src/lib/components/WantCandidates.svelte create mode 100644 web/src/lib/components/WantCandidates.test.ts diff --git a/cmd/minstrel/main.go b/cmd/minstrel/main.go index 49afe10c..ef6a0e9d 100644 --- a/cmd/minstrel/main.go +++ b/cmd/minstrel/main.go @@ -322,6 +322,12 @@ func run() error { // daily for new releases and marks wants held once the library has them. go wanted.New(pool, cat, logger.With("component", "wanted")).Run(ctx) + // Search and grab (M502 #5521): every ten minutes, searches the + // configured indexers for wants due, paced by each source's + // searches-per-hour, and hands the best release to the download client + // when auto-grab is on. + go wanted.NewSearcher(pool, cat, logger.With("component", "search")).Run(ctx) + // Duplicate resolver (M498, M502): classifies the sweep's groups and, with // the operator's auto-resolve on, merges the copies that are safe to merge. // Started after SetNotifier so its summaries reach admins. See diff --git a/internal/acquire/build.go b/internal/acquire/build.go new file mode 100644 index 00000000..882d528c --- /dev/null +++ b/internal/acquire/build.go @@ -0,0 +1,45 @@ +package acquire + +import ( + "context" + "fmt" + + "github.com/jackc/pgx/v5/pgxpool" + + "git.fabledsword.com/bvandeusen/minstrel/internal/db/dbq" + "git.fabledsword.com/bvandeusen/minstrel/internal/release" +) + +// Build makes a Service from the settings as they stand: every enabled +// search source behind one paced Fanout, the first enabled download client, +// and the quality profile. Built afresh for each pass and each request, so a +// changed setting takes effect on the next one. +func Build(ctx context.Context, pool *pgxpool.Pool) (*Service, dbq.AcquisitionSetting, error) { + q := dbq.New(pool) + settings, err := q.GetAcquisitionSettings(ctx) + if err != nil { + return nil, settings, fmt.Errorf("acquire: read settings: %w", err) + } + sources, err := Sources(ctx, pool) + if err != nil { + return nil, settings, fmt.Errorf("acquire: read search sources: %w", err) + } + clients, err := q.ListUsableAcquisitionServices(ctx, JobDownload) + if err != nil { + return nil, settings, fmt.Errorf("acquire: read download clients: %w", err) + } + s := &Service{Profile: release.NewProfile(settings.AllowedQualities), Memory: NewMemory(pool)} + fan := &Fanout{Sources: sources, Pace: SearchPace} + if len(sources) > 0 { + s.Search = fan + } + if len(clients) > 0 { + row := clients[0] + client, err := DownloadClient(row) + if err != nil { + return nil, settings, fmt.Errorf("acquire: download client %s: %w", row.Code, err) + } + s.Grabber = &Downloader{Client: client, Code: row.Code, Landing: row.LocalPath, Fetch: fan} + } + return s, settings, nil +} diff --git a/internal/acquire/fanout.go b/internal/acquire/fanout.go new file mode 100644 index 00000000..3d0594b3 --- /dev/null +++ b/internal/acquire/fanout.go @@ -0,0 +1,165 @@ +package acquire + +import ( + "context" + "errors" + "fmt" + "strings" + "sync" + + "git.fabledsword.com/bvandeusen/minstrel/internal/indexer" +) + +// Searching every source at once (M502 #5521), ported from Fabled Librarian +// (#4182). A source that is down narrows a search rather than failing it: the +// results come back with the list of who did not answer. + +// SourceFailure is one source that could not be reached. +type SourceFailure struct { + Code string + Label string + Err error +} + +// PartialError is a search that reached some sources and not others. It is +// returned alongside the results, not instead of them. +type PartialError struct { + Failures []SourceFailure + // Answered is how many sources did answer. Zero means the search failed. + Answered int +} + +func (e *PartialError) Error() string { + if e == nil || len(e.Failures) == 0 { + return "some sources did not answer" + } + parts := make([]string, 0, len(e.Failures)) + for _, f := range e.Failures { + parts = append(parts, fmt.Sprintf("%s: %s", f.Label, indexer.FirstLine(f.Err.Error()))) + } + return "some sources did not answer — " + strings.Join(parts, "; ") +} + +// Unwrap lets errors.Is find indexer.ErrUnreachable through a partial failure. +func (e *PartialError) Unwrap() []error { + if e == nil { + return nil + } + out := make([]error, 0, len(e.Failures)) + for _, f := range e.Failures { + out = append(out, f.Err) + } + return out +} + +// Fanout searches every source and merges what comes back. It is both the +// Searcher and the Fetcher, routing a .torrent fetch to the source that +// offered the result. +type Fanout struct { + // Sources in code order: the fold keeps a duplicate's first arrival, so + // the order decides which source a grab is fetched through. + Sources []Source + // Pace spaces each source's searches. Nil paces nothing. + Pace *Pacer +} + +// Search asks every source concurrently. Each client bounds its own call by +// its configured timeout, so there is no shared deadline here. +func (f *Fanout) Search(ctx context.Context, query indexer.Query) ([]indexer.Result, error) { + if f == nil || len(f.Sources) == 0 { + return nil, indexer.ErrUnconfigured + } + found := make([][]indexer.Result, len(f.Sources)) + errs := make([]error, len(f.Sources)) + var wg sync.WaitGroup + for i, src := range f.Sources { + if src.Search == nil { + continue + } + f.Pace.spend(src) + wg.Add(1) + go func(i int, src Source) { + defer wg.Done() + found[i], errs[i] = src.Search.Search(ctx, query) + }(i, src) + } + wg.Wait() + + out := make([]indexer.Result, 0, 32) + var failures []SourceFailure + answered, unconfigured := 0, 0 + for i, src := range f.Sources { + if src.Search == nil { + continue + } + if errors.Is(errs[i], indexer.ErrUnconfigured) { + unconfigured++ + continue + } + if rl, ok := indexer.AsRateLimited(errs[i]); ok { + f.Pace.hold(src.Code, rl) + } + if errs[i] != nil { + failures = append(failures, SourceFailure{Code: src.Code, Label: src.Label, Err: errs[i]}) + continue + } + answered++ + for j := range found[i] { + if found[i][j].Source == "" { + found[i][j].Source = src.Code + } + } + out = append(out, found[i]...) + } + switch { + case answered == 0 && len(failures) > 0: + return nil, &PartialError{Failures: failures} + case len(failures) > 0: + return out, &PartialError{Failures: failures, Answered: answered} + case answered == 0 && unconfigured > 0: + return nil, indexer.ErrUnconfigured + } + return out, nil +} + +// Download fetches a result's .torrent from the source that offered it: a +// Prowlarr link carries Prowlarr's key and only Prowlarr can redeem it. +func (f *Fanout) Download(ctx context.Context, r indexer.Result) ([]byte, error) { + if f == nil { + return nil, indexer.ErrUnconfigured + } + for _, src := range f.Sources { + if src.Code != r.Source { + continue + } + if src.Fetch == nil { + return nil, fmt.Errorf("%w: %s cannot fetch torrent files", indexer.ErrUnconfigured, src.Label) + } + return src.Fetch.Download(ctx, r) + } + if r.Source == "" { + return nil, fmt.Errorf("%w: this result does not say which source it came from", indexer.ErrUnconfigured) + } + return nil, fmt.Errorf("%w: %q is no longer a configured source", indexer.ErrUnconfigured, r.Source) +} + +// Narrowing is one source that did not answer, as a page shows it. +type Narrowing struct { + Source string `json:"source"` + Label string `json:"label"` + Reason string `json:"reason"` +} + +// narrowings reads a PartialError out of err, and whether any source +// answered. Nil when err is not a partial failure. +func narrowings(err error) ([]Narrowing, bool) { + var partial *PartialError + if !errors.As(err, &partial) || len(partial.Failures) == 0 { + return nil, false + } + out := make([]Narrowing, 0, len(partial.Failures)) + for _, f := range partial.Failures { + out = append(out, Narrowing{Source: f.Code, Label: f.Label, Reason: indexer.FirstLine(f.Err.Error())}) + } + return out, partial.Answered > 0 +} diff --git a/internal/acquire/grab.go b/internal/acquire/grab.go new file mode 100644 index 00000000..6186ebd9 --- /dev/null +++ b/internal/acquire/grab.go @@ -0,0 +1,307 @@ +package acquire + +import ( + "context" + "errors" + "path" + "strings" + "sync" + + "github.com/jackc/pgx/v5/pgtype" + + "git.fabledsword.com/bvandeusen/minstrel/internal/db/dbq" + "git.fabledsword.com/bvandeusen/minstrel/internal/downloader" + "git.fabledsword.com/bvandeusen/minstrel/internal/indexer" + "git.fabledsword.com/bvandeusen/minstrel/internal/release" +) + +// Taking a release (M502 #5521), ported from Fabled Librarian's grab.go and +// rivals.go. One live grab per want: a second is tie-broken against the +// download already running (FL #4470). + +// Tag goes on every torrent Minstrel adds, so the operator can tell them +// from their own in the client. +const Tag = "minstrel" + +// MissingClient is Taken.Missing when no download client is set up. +const MissingClient = "download_client" + +// nearlyDone is how far along a running download stays whatever would +// replace it. +const nearlyDone = 0.9 + +// Downloader is the grab half of a Service. +type Downloader struct { + Client downloader.Driver + // Code is the client's acquisition_services row. + Code string + // Landing is the client's downloads folder as Minstrel sees it; each + // grab lands in a folder of its own under it. + Landing string + // Fetch gets a result's .torrent from the source that offered it. + Fetch Fetcher +} + +// Origin says who took a release. +type Origin struct { + By string // OriginSchedule or OriginOperator + User pgtype.UUID +} + +// Contest is how a grab fared against the download already running. +type Contest struct { + // Kept: the running one stays and this grab was not made. Replaced: + // this grab took its place. + Kept, Replaced bool + Reason string + Rival dbq.Grab +} + +// Taken is what came of a grab. +type Taken struct { + Chosen *Candidate + // Added is the only field that means a download started. + Added bool + SavePath string + Grab dbq.Grab + // Gone: the search no longer offers the release. + Gone bool + // Refused: not taken; the reason is on Chosen. + Refused bool + // Unconfigured: Missing names what is not set up ("" for search). + Unconfigured bool + Missing string + Contest *Contest + Error string +} + +// Take searches again and grabs the candidate named by key: a person's +// grab. Searching again keeps the indexer's download link (which can carry +// its key) on the server, and finds a release gone since it was shown. +// force takes what the verdict would not. +func (s *Service) Take(ctx context.Context, want Want, key string, force bool, origin Origin) Taken { + if s.Grabber == nil { + return Taken{Unconfigured: true, Missing: MissingClient} + } + found := s.For(ctx, want) + switch { + case found.Unconfigured: + return Taken{Unconfigured: true} + case found.Error != "": + return Taken{Error: found.Error} + } + chosen := pick(found.Candidates, key) + if chosen == nil { + return Taken{Gone: true} + } + return s.Grab(ctx, want, chosen, found.Candidates, force, origin) +} + +// Grab hands a judged candidate to the download client. The schedule calls +// it with the candidate it has just judged; Take calls it after searching +// again. +func (s *Service) Grab(ctx context.Context, want Want, chosen *Candidate, all []Candidate, force bool, origin Origin) Taken { + out := Taken{Chosen: chosen} + if s.Grabber == nil || s.Grabber.Client == nil { + out.Unconfigured, out.Missing = true, MissingClient + return out + } + if chosen.Verdict == Blocked || (chosen.Verdict != Grab && !force) { + out.Refused = true + return out + } + + unlock := lockWant(want.ID) + defer unlock() + contest := s.contest(ctx, want, chosen, all, origin) + if contest != nil && contest.Kept { + out.Contest = contest + return out + } + + add := downloader.Add{Magnet: chosen.Source.Magnet, Tags: []string{Tag}} + hash := chosen.InfoHash + if add.Magnet == "" { + if s.Grabber.Fetch == nil { + out.Error = "This release is a .torrent link, and nothing is set up to fetch it." + return out + } + body, err := s.Grabber.Fetch.Download(ctx, chosen.Source) + if magnet, ok := indexer.AsMagnet(err); ok { + add.Magnet, err = magnet, nil + } + if err != nil { + out.Error = indexer.FirstLine(err.Error()) + return out + } + if add.Magnet == "" { + add.Torrent = body + if meta, err := release.ParseTorrent(body); err == nil { + if hash == "" { + hash = meta.InfoHash + } + if refused := s.judgeFiles(ctx, want, chosen, hash, meta.Files, force); refused { + out.Refused = true + return out + } + } + } + } + if hash == "" && add.Magnet != "" { + hash = indexer.MagnetHash(add.Magnet) + } + add.SavePath = landingFor(s.Grabber.Landing, hash, chosen.Name) + + if err := s.Grabber.Client.AddTorrent(ctx, add); err != nil && !errors.Is(err, downloader.ErrDuplicate) { + if errors.Is(err, downloader.ErrUnconfigured) { + out.Unconfigured, out.Missing = true, MissingClient + } else { + out.Error = indexer.FirstLine(err.Error()) + } + return out + } + out.Added, out.SavePath = true, add.SavePath + if s.Memory == nil { + return out + } + + var replaced pgtype.UUID + why := "" + if contest != nil && contest.Replaced { + out.Contest = contest + replaced, why = contest.Rival.ID, "Replaced by "+chosen.Name+": "+contest.Reason+"." + } + g, err := s.Memory.Grabbed(ctx, dbq.InsertGrabParams{ + WantID: want.ID, InfoHash: hash, Name: chosen.Name, Key: Key(*chosen), Indexer: chosen.Indexer, + Source: chosen.Source.Source, Client: s.Grabber.Code, SavePath: add.SavePath, + Quality: string(chosen.Tier), Origin: origin.By, Forced: force, GrabbedBy: origin.User, + }, replaced, why) + if err != nil { + out.Error = "The download started, but could not be recorded: " + indexer.FirstLine(err.Error()) + return out + } + out.Grab = g + // Removed only after the new download was added and recorded, so a + // failure never leaves the want with nothing running. + if replaced.Valid && contest.Rival.InfoHash != "" { + if err := s.Grabber.Client.Delete(ctx, true, contest.Rival.InfoHash); err != nil { + contest.Reason += ". The download it replaced could not be removed from the client: " + indexer.FirstLine(err.Error()) + } + } + return out +} + +// judgeFiles judges a chosen release again on its .torrent's file list, and +// sets it aside when the files say it is not the album, so no later search +// offers it again. force still takes it. +func (s *Service) judgeFiles(ctx context.Context, want Want, chosen *Candidate, hash string, files []release.File, force bool) bool { + chosen.Files = files + if chosen.FilesCount == 0 { + chosen.FilesCount = len(files) + } + veto, reason, _ := release.JudgeFiles(want.Want, files) + if veto == "" || force { + return false + } + set(chosen, Rejected, Hard, veto, reason) + if s.Memory != nil && hash != "" { + _ = s.Memory.Block(ctx, dbq.BlockReleaseParams{ + InfoHash: hash, Name: chosen.Name, Reason: BlockFiles, Detail: reason, WantID: want.ID, + }) + } + return true +} + +var wantLocks sync.Map // want id → *sync.Mutex + +// lockWant settles two grabs for one want one after the other. The +// one-live-grab index is the backstop for anything outside this process. +func lockWant(id pgtype.UUID) func() { + if !id.Valid { + return func() {} + } + m, _ := wantLocks.LoadOrStore(id.Bytes, &sync.Mutex{}) + mu := m.(*sync.Mutex) + mu.Lock() + return mu.Unlock +} + +// contest settles a grab against the download running for its want: a +// person's grab beats the schedule's; between two of a kind the better +// candidate wins; a tie, a download nearly done, or one that cannot be +// removed stays. Nil when nothing is running. +func (s *Service) contest(ctx context.Context, want Want, chosen *Candidate, all []Candidate, origin Origin) *Contest { + if s.Memory == nil || !want.ID.Valid { + return nil + } + running, ok, err := s.Memory.LiveGrab(ctx, want.ID) + if err != nil || !ok { + return nil + } + c := &Contest{Rival: running} + if running.InfoHash != "" && running.InfoHash == chosen.InfoHash { + c.Kept, c.Reason = true, "That release is already downloading for this album." + return c + } + switch { + case origin.By == OriginOperator && running.Origin == OriginSchedule: + c.Reason = "you chose it over the one the schedule started" + case origin.By == OriginSchedule && running.Origin == OriginOperator: + c.Kept, c.Reason = true, "A download you started for this album is already running." + return c + default: + var theirs *Candidate + for i := range all { + if (running.InfoHash != "" && all[i].InfoHash == running.InfoHash) || Key(all[i]) == running.Key { + theirs = &all[i] + break + } + } + if theirs == nil || !better(*chosen, *theirs) { + c.Kept, c.Reason = true, "A download for this album is already running, and this release is no better." + return c + } + c.Reason = "it ranks above the one that was running" + } + if running.InfoHash == "" { + c.Kept, c.Reason = true, "A download for this album is already running, and it cannot be identified to replace it." + return c + } + if ts, err := s.Grabber.Client.Torrents(ctx, running.InfoHash); err == nil && len(ts) > 0 && ts[0].Progress >= nearlyDone { + c.Kept, c.Reason = true, "A download for this album is already running and nearly done." + return c + } + c.Replaced = true + return c +} + +// landingFor is the folder a download lands in: one per torrent, named by +// its infohash, so two grabs never unpack into the same place. +func landingFor(root, hash, name string) string { + dir := hash + if dir == "" { + dir = slug(name) + } + if dir == "" { + dir = "unnamed" + } + return path.Join(root, dir) +} + +// slug keeps a folder name to characters every filesystem in the chain +// takes. +func slug(s string) string { + var b strings.Builder + for _, r := range strings.ToLower(strings.TrimSpace(s)) { + if b.Len() >= 80 { + break + } + switch { + case r >= 'a' && r <= 'z', r >= '0' && r <= '9': + b.WriteRune(r) + case r == '-', r == '_', r == ' ', r == '.': + b.WriteRune('-') + } + } + return strings.Trim(b.String(), "-") +} diff --git a/internal/acquire/memory.go b/internal/acquire/memory.go new file mode 100644 index 00000000..a9b0354a --- /dev/null +++ b/internal/acquire/memory.go @@ -0,0 +1,168 @@ +package acquire + +import ( + "context" + "encoding/json" + "errors" + "fmt" + + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgtype" + "github.com/jackc/pgx/v5/pgxpool" + + "git.fabledsword.com/bvandeusen/minstrel/internal/db/dbq" + "git.fabledsword.com/bvandeusen/minstrel/internal/indexer" + "git.fabledsword.com/bvandeusen/minstrel/internal/release" +) + +// Block reasons, matching release_blocks_reason_check. +const ( + BlockOperator = "operator" + BlockFiles = "files" +) + +// Grab origins, matching grabs_origin_check. +const ( + OriginSchedule = "schedule" + OriginOperator = "operator" +) + +// Memory is what searches and grabs leave behind: every candidate of a +// want's last search, the grabs, and the releases set aside. +type Memory struct { + pool *pgxpool.Pool +} + +// NewMemory returns a Memory over pool. +func NewMemory(pool *pgxpool.Pool) *Memory { return &Memory{pool: pool} } + +// Keep replaces a want's candidates with this search's, in order. +func (m *Memory) Keep(ctx context.Context, wantID pgtype.UUID, cands []Candidate) error { + return pgx.BeginFunc(ctx, m.pool, func(tx pgx.Tx) error { + q := dbq.New(tx) + if err := q.DeleteWantCandidates(ctx, wantID); err != nil { + return err + } + for i, c := range cands { + reasons, err := json.Marshal(c.Reasons) + if err != nil { + return err + } + if c.Reasons == nil { + reasons = []byte("[]") + } + folded := c.FoldedFrom + if folded == nil { + folded = []string{} + } + if err := q.InsertWantCandidate(ctx, dbq.InsertWantCandidateParams{ + WantID: wantID, Key: Key(c), Name: c.Name, InfoHash: c.InfoHash, Indexer: c.Indexer, + Source: c.Source.Source, SizeBytes: c.SizeBytes, Seeders: int32(c.Seeders), + FilesCount: int32(c.FilesCount), Format: release.Describe(c.Release), Quality: string(c.Tier), + Year: int32(c.Year), Verdict: string(c.Verdict), Reason: c.Reason, Reasons: reasons, + FoldedFrom: folded, Rank: int32(i), + }); err != nil { + return err + } + } + return nil + }) +} + +// blocked is every release set aside, by infohash, with the sentence a +// candidate shows. A failed read blocks nothing: offering a release set +// aside costs one grab a person can undo, while blocking everything on a +// database blip would hide every result. +func (m *Memory) blocked(ctx context.Context) map[string]string { + if m == nil { + return nil + } + rows, err := dbq.New(m.pool).ListReleaseBlocks(ctx) + if err != nil { + return nil + } + out := make(map[string]string, len(rows)) + for _, r := range rows { + out[indexer.NormalizeHash(r.InfoHash)] = BlockSentence(r.Reason, r.Detail) + } + return out +} + +// BlockSentence is what a blocked candidate says. +func BlockSentence(reason, detail string) string { + if reason == BlockFiles { + if detail != "" { + return "Set aside for what its files hold: " + detail + } + return "Set aside for what its files hold." + } + return "You set this release aside." +} + +// Block sets a release aside and marks every kept candidate of it blocked. +func (m *Memory) Block(ctx context.Context, p dbq.BlockReleaseParams) error { + p.InfoHash = indexer.NormalizeHash(p.InfoHash) + if p.InfoHash == "" { + return errors.New("a release without an infohash cannot be set aside") + } + return pgx.BeginFunc(ctx, m.pool, func(tx pgx.Tx) error { + q := dbq.New(tx) + if err := q.BlockRelease(ctx, p); err != nil { + return err + } + return q.MarkCandidatesBlocked(ctx, dbq.MarkCandidatesBlockedParams{ + InfoHash: p.InfoHash, Reason: BlockSentence(p.Reason, p.Detail), + }) + }) +} + +// Unblock lets a release be taken again. Its kept candidates wait for a +// person until the next search judges them afresh. +func (m *Memory) Unblock(ctx context.Context, hash string) (bool, error) { + hash = indexer.NormalizeHash(hash) + var n int64 + err := pgx.BeginFunc(ctx, m.pool, func(tx pgx.Tx) error { + q := dbq.New(tx) + var err error + if n, err = q.UnblockRelease(ctx, hash); err != nil { + return err + } + return q.MarkCandidatesUnblocked(ctx, dbq.MarkCandidatesUnblockedParams{ + InfoHash: hash, Reason: "Allowed again. The next search judges it afresh.", + }) + }) + return n > 0, err +} + +// LiveGrab is the download running for a want, if there is one. +func (m *Memory) LiveGrab(ctx context.Context, wantID pgtype.UUID) (dbq.Grab, bool, error) { + g, err := dbq.New(m.pool).GetLiveGrab(ctx, wantID) + if errors.Is(err, pgx.ErrNoRows) { + return g, false, nil + } + if err != nil { + return g, false, err + } + return g, true, nil +} + +// Grabbed records a download started, ending the grab it replaced first so +// the one-live-grab index holds. +func (m *Memory) Grabbed(ctx context.Context, p dbq.InsertGrabParams, replaced pgtype.UUID, why string) (dbq.Grab, error) { + var g dbq.Grab + err := pgx.BeginFunc(ctx, m.pool, func(tx pgx.Tx) error { + q := dbq.New(tx) + if replaced.Valid { + if err := q.EndGrab(ctx, dbq.EndGrabParams{ID: replaced, EndReason: "superseded", EndDetail: why}); err != nil { + return err + } + } + var err error + g, err = q.InsertGrab(ctx, p) + return err + }) + if err != nil { + return g, fmt.Errorf("record grab: %w", err) + } + return g, nil +} diff --git a/internal/acquire/pace.go b/internal/acquire/pace.go new file mode 100644 index 00000000..53900e85 --- /dev/null +++ b/internal/acquire/pace.go @@ -0,0 +1,145 @@ +package acquire + +import ( + "context" + "sync" + "time" + + "git.fabledsword.com/bvandeusen/minstrel/internal/indexer" +) + +// SearchPace is the one Pacer every search shares, ported from Fabled +// Librarian (#4448). Process-wide because a Fanout is built fresh for each +// search; a Pacer per Fanout would start empty and pace nothing. +var SearchPace = &Pacer{} + +// holdWithoutRetryAfter is how long a source that answered 429 without saying +// for how long is left alone: one schedule tick. +const holdWithoutRetryAfter = 10 * time.Minute + +// Pacer spaces the searches sent to each source by its searches_per_hour: a +// spacing, not a burst bucket, because Prowlarr answers 429 to a few requests +// in quick succession (FL #4304). Every search spends, the schedule's and a +// person's alike; only the schedule waits. +type Pacer struct { + // Now and Sleep are the clock, replaceable in tests. + Now func() time.Time + Sleep func(context.Context, time.Duration) error + + mu sync.Mutex + next map[string]time.Time +} + +func (p *Pacer) now() time.Time { + if p.Now != nil { + return p.Now() + } + return time.Now() +} + +func (p *Pacer) sleep(ctx context.Context, d time.Duration) error { + if p.Sleep != nil { + return p.Sleep(ctx, d) + } + t := time.NewTimer(d) + defer t.Stop() + select { + case <-t.C: + return nil + case <-ctx.Done(): + return ctx.Err() + } +} + +// gap is the spacing a source's number asks for; zero is unpaced. +func gap(perHour int) time.Duration { + if perHour <= 0 { + return 0 + } + return time.Hour / time.Duration(perHour) +} + +// readyAt is when every source may next be asked. +func (p *Pacer) readyAt(sources []Source) time.Time { + p.mu.Lock() + defer p.mu.Unlock() + var at time.Time + for _, s := range sources { + if s.Search == nil { + continue + } + if n := p.next[s.Code]; n.After(at) { + at = n + } + } + return at +} + +// Wait holds until every source may be asked, and answers false, without +// waiting, when that is after the deadline or ctx ends first. All of them, +// not any: a search while one is held back would be narrowed, and the want +// recorded as looked for. +func (p *Pacer) Wait(ctx context.Context, sources []Source, deadline time.Time) bool { + if p == nil { + return true + } + at := p.readyAt(sources) + if !deadline.IsZero() && at.After(deadline) { + return false + } + if d := at.Sub(p.now()); d > 0 { + return p.sleep(ctx, d) == nil + } + return ctx.Err() == nil +} + +// spend takes one search's slot from a source. +func (p *Pacer) spend(s Source) { + g := gap(s.PerHour) + if p == nil || g == 0 { + return + } + p.mu.Lock() + defer p.mu.Unlock() + if p.next == nil { + p.next = map[string]time.Time{} + } + at := p.next[s.Code] + if t := p.now(); at.Before(t) { + at = t + } + p.next[s.Code] = at.Add(g) +} + +// hold leaves a source alone after it answered 429, for as long as it asked, +// whatever its searches_per_hour says. +func (p *Pacer) hold(code string, rl *indexer.RateLimited) { + if p == nil { + return + } + d := rl.RetryAfter + if d <= 0 { + d = holdWithoutRetryAfter + } + p.mu.Lock() + defer p.mu.Unlock() + if p.next == nil { + p.next = map[string]time.Time{} + } + if until := p.now().Add(d); until.After(p.next[code]) { + p.next[code] = until + } +} + +// Ready holds until every source this service searches may be asked again, +// and answers false when that is past the deadline. The schedule calls it +// before each want; a person's search never does. +func (s *Service) Ready(ctx context.Context, deadline time.Time) bool { + if s == nil { + return ctx.Err() == nil + } + if f, ok := s.Search.(*Fanout); ok && f != nil { + return f.Pace.Wait(ctx, f.Sources, deadline) + } + return ctx.Err() == nil +} diff --git a/internal/acquire/search.go b/internal/acquire/search.go new file mode 100644 index 00000000..d3edf564 --- /dev/null +++ b/internal/acquire/search.go @@ -0,0 +1,390 @@ +package acquire + +import ( + "context" + "errors" + "sort" + "strings" + + "github.com/jackc/pgx/v5/pgtype" + + "git.fabledsword.com/bvandeusen/minstrel/internal/indexer" + "git.fabledsword.com/bvandeusen/minstrel/internal/release" +) + +// One search for one want, and what could be done about each result (M502 +// #5521), ported from Fabled Librarian's internal/acquire. internal/release +// says whether a name is the album and what quality it is; this package +// asks in the right order and says why in words a person can act on. + +// Verdict is what can be done about one result. +type Verdict string + +const ( + // Grab: the album, in a quality the profile takes. + Grab Verdict = "grab" + // Review: maybe the album; a person decides. + Review Verdict = "review" + // Refused: the album, in a quality the profile does not take. The only + // verdict that names a setting. + Refused Verdict = "refused" + // Rejected: not the album. + Rejected Verdict = "rejected" + // Blocked: a release set aside, by a person or by its files. + Blocked Verdict = "blocked" +) + +// order sorts verdicts: what can be taken first, what was given up on last. +var order = map[Verdict]int{Grab: 0, Review: 1, Refused: 2, Rejected: 3, Blocked: 4} + +// Reason kinds: a hard reason is a fact about the bytes, a soft one about the +// moment or a setting. +const ( + Hard = "hard" + Soft = "soft" +) + +// Reason codes beyond internal/release's vetoes. +const ( + CodeBlocked = "blocked" + CodeProfile = "profile" + CodeNoSeeders = "no_seeders" + CodeDoubt = "doubt" +) + +// Reason is one thing a verdict rests on. +type Reason struct { + Kind string `json:"kind"` + Code string `json:"code"` + Text string `json:"text"` +} + +// Want is the album searched for: its wants row and how the catalogue +// describes it. +type Want struct { + ID pgtype.UUID + release.Want +} + +// Candidate is one result with everything decided about it. +type Candidate struct { + release.Release + // Source is the indexer's own result. It never leaves the server: its + // DownloadURL can carry the indexer's key. + Source indexer.Result + InfoHash string + Indexer string + SizeBytes int64 + Seeders int + SwarmKnown bool + Category int + FilesCount int + // Files is the torrent's file list once its .torrent has been read. + Files []release.File + FoldedFrom []string + + Tier release.Tier + YearAgrees bool + Verdict Verdict + Reason string + Reasons []Reason +} + +// Key is how a candidate is named across a search and a grab: its infohash, +// or its name when the source gave none. +func Key(c Candidate) string { + if c.InfoHash != "" { + return c.InfoHash + } + return c.Name +} + +// Result is one search. +type Result struct { + Query string + // Unconfigured: no search source is set up. + Unconfigured bool + // Error is a search that failed outright. + Error string + // Narrowed names sources that did not answer on a search that otherwise + // did. + Narrowed []Narrowing + Considered int + Folded int + // Looked is how many .torrent files were read for their file lists. + Looked int + Candidates []Candidate + // KeepError: the candidates could not be stored. + KeepError string +} + +// Grabbable is what may be taken, best first. +func (r Result) Grabbable() []Candidate { + var out []Candidate + for _, c := range r.Candidates { + if c.Verdict == Grab { + out = append(out, c) + } + } + return out +} + +// ForAPerson says whether anything found is worth a person's look: a +// release that may be the album, whatever its quality. +func (r Result) ForAPerson() bool { + for _, c := range r.Candidates { + if c.Verdict == Grab || c.Verdict == Review || c.Verdict == Refused { + return true + } + } + return false +} + +// Service searches and grabs. +type Service struct { + // Search is nil when no source is configured. + Search Searcher + // Grabber is nil when no download client is configured. + Grabber *Downloader + Profile release.Profile + // Memory stores candidates, grabs and blocks. Nil keeps nothing. + Memory *Memory +} + +// Query is the text sent to the indexers: the artist and the title, without +// a trailing bracketed part the release names usually leave off. +func Query(w release.Want) string { + title := w.Title + if i := strings.IndexAny(title, "(["); i > 0 { + title = title[:i] + } + return strings.Join(strings.Fields(strings.TrimSpace(w.Artist)+" "+strings.TrimSpace(title)), " ") +} + +// For searches for one want and judges everything that comes back. It never +// returns an error: a search that could not be made is a Result saying so. +func (s *Service) For(ctx context.Context, want Want) Result { + r := Result{Query: Query(want.Want)} + if s == nil || s.Search == nil { + r.Unconfigured = true + return r + } + found, err := s.Search.Search(ctx, indexer.Query{Text: r.Query, Categories: indexer.MusicCategories}) + // A partial search arrives as results and an error together. + if narrowed, answered := narrowings(err); narrowed != nil && answered { + r.Narrowed, err = narrowed, nil + } + switch { + case errors.Is(err, indexer.ErrUnconfigured): + r.Unconfigured = true + return r + case err != nil: + r.Error = indexer.FirstLine(err.Error()) + return r + } + r.Considered = len(found) + cands, folded := fold(found) + r.Folded = folded + + blocked := s.Memory.blocked(ctx) + for i := range cands { + decide(&cands[i], want.Want, s.Profile, blocked) + } + sortCandidates(cands) + for _, i := range s.peek(ctx, cands) { + decide(&cands[i], want.Want, s.Profile, blocked) + r.Looked++ + } + sortCandidates(cands) + r.Candidates = cands + if s.Memory != nil && want.ID.Valid { + if err := s.Memory.Keep(ctx, want.ID, cands); err != nil { + r.KeepError = indexer.FirstLine(err.Error()) + } + } + return r +} + +// decide works out one candidate's verdict, in order: given up on, then +// whether it is the album, then whether the profile takes its quality, then +// whether anything leaves it to a person. A quality refusal said about the +// wrong album would be noise, so identity comes first. +func decide(c *Candidate, w release.Want, profile release.Profile, blocked map[string]string) { + c.Verdict, c.Reason, c.Reasons = "", "", nil + if why, ok := blocked[c.InfoHash]; ok && c.InfoHash != "" { + set(c, Blocked, Hard, CodeBlocked, why) + return + } + m := release.Judge(w, c.Release) + c.YearAgrees = m.YearAgrees + if m.Veto != "" { + set(c, Rejected, Hard, m.Veto, m.Reason) + return + } + doubt := m.Doubt + if len(c.Files) > 0 { + if in := release.Inspect(c.Files); in.Format != "" { + c.Format = in.Format + } + veto, reason, fileDoubt := release.JudgeFiles(w, c.Files) + if veto != "" { + set(c, Rejected, Hard, veto, reason) + return + } + if doubt == "" { + doubt = fileDoubt + } + } + c.Tier = release.TierOf(c.Release, c.Category) + if !profile.Allows(c.Tier) { + set(c, Refused, Soft, CodeProfile, "Your quality settings don’t take "+strings.ToLower(release.Label(c.Tier))+".") + return + } + if doubt != "" { + set(c, Review, Soft, CodeDoubt, doubt) + return + } + if c.Seeders == 0 { + set(c, Review, Soft, CodeNoSeeders, "Nobody is sharing it right now.") + return + } + c.Verdict = Grab + c.Reason = "The album, " + strings.ToLower(release.Label(c.Tier)) + "." + if d := release.Describe(c.Release); d != "" { + c.Reason = "The album, " + d + "." + } +} + +func set(c *Candidate, v Verdict, kind, code, text string) { + c.Verdict, c.Reason = v, text + c.Reasons = append(c.Reasons, Reason{Kind: kind, Code: code, Text: text}) +} + +// sortCandidates puts the best first, stably so the same search proposes the +// same grab twice. +func sortCandidates(cands []Candidate) { + sort.SliceStable(cands, func(a, b int) bool { return better(cands[a], cands[b]) }) +} + +// better says whether x ranks above y: verdict, then quality, then how many +// share it, then whether its year agrees. +func better(x, y Candidate) bool { + if order[x.Verdict] != order[y.Verdict] { + return order[x.Verdict] < order[y.Verdict] + } + if rx, ry := release.Rank(x.Tier), release.Rank(y.Tier); rx != ry { + return rx > ry + } + if x.Seeders != y.Seeders { + return x.Seeders > y.Seeders + } + return x.YearAgrees && !y.YearAgrees +} + +// fold merges results that are the same torrent, by infohash. Prowlarr +// aggregates, so one release arrives once per tracker that lists it. A +// result without a hash is never folded: an absent hash is not a shared one. +func fold(found []indexer.Result) ([]Candidate, int) { + out := make([]Candidate, 0, len(found)) + at := make(map[string]int, len(found)) + folded := 0 + for _, f := range found { + if strings.TrimSpace(f.Title) == "" { + continue + } + hash := indexer.NormalizeHash(f.InfoHash) + if hash == "" && f.Magnet != "" { + hash = indexer.MagnetHash(f.Magnet) + } + if hash != "" { + if i, seen := at[hash]; seen { + folded++ + if f.Indexer != "" { + out[i].FoldedFrom = append(out[i].FoldedFrom, f.Indexer) + } + // A measured swarm beats a reported one; among like, the + // highest count. + switch { + case f.SwarmKnown && !out[i].SwarmKnown: + out[i].Seeders, out[i].SwarmKnown = f.Seeders, true + case f.SwarmKnown == out[i].SwarmKnown && f.Seeders > out[i].Seeders: + out[i].Seeders = f.Seeders + } + if f.FilesCount > 0 && out[i].FilesCount == 0 { + out[i].FilesCount = f.FilesCount + } + continue + } + at[hash] = len(out) + } + out = append(out, Candidate{ + Release: release.Parse(f.Title), Source: f, InfoHash: hash, Indexer: f.Indexer, + SizeBytes: f.SizeBytes, Seeders: f.Seeders, SwarmKnown: f.SwarmKnown, + Category: f.Category, FilesCount: f.FilesCount, + }) + } + return out, folded +} + +// maxPeeks is how many .torrent files one search reads. Few: each goes +// through Prowlarr to a tracker, and Prowlarr answers 429 to a burst. +const maxPeeks = 3 + +// peek reads the .torrent of candidates that are the album but whose name +// states no format, so their files can say what they are; it returns which +// now carry a file list. A failed fetch leaves the candidate as it was. +func (s *Service) peek(ctx context.Context, cands []Candidate) []int { + f, ok := s.Search.(Fetcher) + if !ok { + return nil + } + var looked []int + for i := range cands { + if len(looked) >= maxPeeks || ctx.Err() != nil { + break + } + c := &cands[i] + if c.Tier != release.TierUnknown || (c.Verdict != Grab && c.Verdict != Refused) || + c.Files != nil || c.Source.DownloadURL == "" || c.Source.Magnet != "" { + continue + } + body, err := f.Download(ctx, c.Source) + if err != nil { + continue + } + meta, err := release.ParseTorrent(body) + if err != nil { + continue + } + c.Files = meta.Files + if c.InfoHash == "" { + c.InfoHash = meta.InfoHash + } + if c.FilesCount == 0 { + c.FilesCount = len(meta.Files) + } + looked = append(looked, i) + } + return looked +} + +// pick finds a candidate by key: the infohash first, the exact name for a +// result that has none. +func pick(cands []Candidate, key string) *Candidate { + key = strings.TrimSpace(key) + if key == "" { + return nil + } + hash := indexer.NormalizeHash(key) + for i := range cands { + if cands[i].InfoHash != "" && cands[i].InfoHash == hash { + return &cands[i] + } + } + for i := range cands { + if cands[i].InfoHash == "" && cands[i].Name == key { + return &cands[i] + } + } + return nil +} diff --git a/internal/acquire/search_test.go b/internal/acquire/search_test.go new file mode 100644 index 00000000..f158b345 --- /dev/null +++ b/internal/acquire/search_test.go @@ -0,0 +1,241 @@ +package acquire + +import ( + "context" + "errors" + "strings" + "testing" + "time" + + "git.fabledsword.com/bvandeusen/minstrel/internal/downloader" + "git.fabledsword.com/bvandeusen/minstrel/internal/indexer" + "git.fabledsword.com/bvandeusen/minstrel/internal/release" +) + +type fakeSearch struct { + results []indexer.Result + err error + asked []indexer.Query +} + +func (f *fakeSearch) Search(_ context.Context, q indexer.Query) ([]indexer.Result, error) { + f.asked = append(f.asked, q) + return f.results, f.err +} + +var ram = Want{Want: release.Want{ + Artist: "Daft Punk", Title: "Random Access Memories", Year: 2013, PrimaryType: "Album", + MinTracks: 13, MaxTracks: 15, +}} + +var anyQuality = release.NewProfile([]string{ + "lossless_hires", "lossless", "lossy_high", "lossy_standard", "lossy_low", "lossy_poor", "lossy_trash", +}) + +func result(title, hash string, seeders int) indexer.Result { + return indexer.Result{Title: title, InfoHash: hash, Seeders: seeders, Indexer: "Knaben", Source: "prowlarr", + Magnet: "magnet:?xt=urn:btih:" + hash} +} + +func TestFor_JudgesSortsAndFolds(t *testing.T) { + search := &fakeSearch{results: []indexer.Result{ + result("Daft Punk - Random Access Memories (2013) [MP3 320]", "aaaa", 40), + result("Daft Punk Random Access Memories (Drumless Edition 2023) [Flac 24-88]", "bbbb", 90), + result("Daft Punk - Random Access Memories (2013) [FLAC]", "cccc", 12), + result("Daft Punk - Random Access Memories (2013) [FLAC]", "CCCC", 30), // the same torrent from another tracker + result("Daft Punk - Discovery (2001) [FLAC]", "dddd", 50), + }} + s := &Service{Search: search, Profile: anyQuality} + r := s.For(context.Background(), ram) + + if len(search.asked) != 1 || search.asked[0].Text != "Daft Punk Random Access Memories" { + t.Fatalf("asked %+v", search.asked) + } + if r.Considered != 5 || r.Folded != 1 || len(r.Candidates) != 4 { + t.Fatalf("considered %d folded %d candidates %d", r.Considered, r.Folded, len(r.Candidates)) + } + // Lossless before high-quality lossy among what can be taken; the + // folded copy keeps the higher seeder count. + first, second := r.Candidates[0], r.Candidates[1] + if first.InfoHash != "cccc" || first.Verdict != Grab || first.Seeders != 30 || first.Tier != release.TierLossless { + t.Errorf("first = %s %s %d %s", first.Name, first.Verdict, first.Seeders, first.Tier) + } + if second.InfoHash != "aaaa" || second.Verdict != Grab { + t.Errorf("second = %s %s", second.Name, second.Verdict) + } + for _, c := range r.Candidates[2:] { + if c.Verdict != Rejected || len(c.Reasons) != 1 || c.Reasons[0].Kind != Hard { + t.Errorf("%s: %s %+v", c.Name, c.Verdict, c.Reasons) + } + } + if len(r.Grabbable()) != 2 || !r.ForAPerson() { + t.Errorf("grabbable %d", len(r.Grabbable())) + } +} + +func TestFor_ProfileSeedersAndDoubt(t *testing.T) { + search := &fakeSearch{results: []indexer.Result{ + result("Daft Punk - Random Access Memories [MP3 128]", "aaaa", 5), + result("Daft Punk - Random Access Memories [FLAC]", "bbbb", 0), + result("Daft Punk - Random Access Memories", "cccc", 9), + }} + s := &Service{Search: search, Profile: release.NewProfile([]string{"lossless", "lossy_high"})} + r := s.For(context.Background(), ram) + verdicts := map[string]Verdict{} + for _, c := range r.Candidates { + verdicts[c.InfoHash] = c.Verdict + } + if verdicts["aaaa"] != Refused || verdicts["bbbb"] != Review || verdicts["cccc"] != Refused { + t.Errorf("verdicts = %v", verdicts) + } + if len(r.Grabbable()) != 0 || !r.ForAPerson() { + t.Error("nothing grabbable, but something for a person") + } +} + +func TestFor_BlockedReleasesAreSetAside(t *testing.T) { + c := Candidate{Release: release.Parse("Daft Punk - Random Access Memories [FLAC]"), InfoHash: "aaaa", Seeders: 3} + decide(&c, ram.Want, anyQuality, map[string]string{"aaaa": "You set this release aside."}) + if c.Verdict != Blocked || c.Reason != "You set this release aside." { + t.Errorf("verdict %s %q", c.Verdict, c.Reason) + } +} + +func TestFor_UnconfiguredAndFailedSearches(t *testing.T) { + if r := (&Service{}).For(context.Background(), ram); !r.Unconfigured { + t.Error("no source is unconfigured") + } + r := (&Service{Search: &fakeSearch{err: indexer.ErrUnconfigured}}).For(context.Background(), ram) + if !r.Unconfigured { + t.Error("ErrUnconfigured is unconfigured") + } + r = (&Service{Search: &fakeSearch{err: errors.New("dial tcp: refused")}}).For(context.Background(), ram) + if r.Error == "" { + t.Error("a failed search says so") + } +} + +func TestFanout_ANarrowedSearchKeepsWhatAnswered(t *testing.T) { + up := &fakeSearch{results: []indexer.Result{result("Daft Punk - Random Access Memories [FLAC]", "aaaa", 3)}} + down := &fakeSearch{err: indexer.ErrUnreachable} + f := &Fanout{Sources: []Source{{Code: "prowlarr", Label: "Prowlarr", Search: up}, {Code: "torznab", Label: "Feed", Search: down}}} + r := (&Service{Search: f, Profile: anyQuality}).For(context.Background(), ram) + if r.Error != "" || len(r.Candidates) != 1 || len(r.Narrowed) != 1 || r.Narrowed[0].Source != "torznab" { + t.Errorf("narrowed search: error %q, %d candidates, narrowed %+v", r.Error, len(r.Candidates), r.Narrowed) + } + f.Sources = f.Sources[1:] + if r := (&Service{Search: f}).For(context.Background(), ram); r.Error == "" { + t.Error("every source down is a failed search") + } +} + +func TestPacer_SpacesSearchesAndHoldsAfterA429(t *testing.T) { + now := time.Unix(0, 0) + var slept time.Duration + p := &Pacer{Now: func() time.Time { return now }, Sleep: func(_ context.Context, d time.Duration) error { slept += d; return nil }} + src := []Source{{Code: "prowlarr", PerHour: 120, Search: &fakeSearch{}}} + p.spend(src[0]) + if !p.Wait(context.Background(), src, now.Add(time.Hour)) || slept != 30*time.Second { + t.Errorf("slept %s, want 30s", slept) + } + if p.Wait(context.Background(), src, now.Add(time.Second)) { + t.Error("waited past the deadline") + } + p.hold("prowlarr", &indexer.RateLimited{RetryAfter: 5 * time.Minute}) + if at := p.readyAt(src); at.Sub(now) != 5*time.Minute { + t.Errorf("held until %s", at.Sub(now)) + } +} + +type fakeClient struct { + downloader.Driver + added []downloader.Add + addErr error + deleted []string + progress float64 +} + +func (c *fakeClient) AddTorrent(_ context.Context, a downloader.Add) error { + c.added = append(c.added, a) + return c.addErr +} + +func (c *fakeClient) Torrents(_ context.Context, hashes ...string) ([]downloader.Torrent, error) { + return []downloader.Torrent{{Hash: hashes[0], Progress: c.progress}}, nil +} + +func (c *fakeClient) Delete(_ context.Context, _ bool, hashes ...string) error { + c.deleted = append(c.deleted, hashes...) + return nil +} + +func TestGrab_AddsAMagnetToItsOwnFolder(t *testing.T) { + client := &fakeClient{} + s := &Service{Search: &fakeSearch{results: []indexer.Result{ + result("Daft Punk - Random Access Memories (2013) [FLAC]", "ABCD", 8), + }}, Profile: anyQuality, Grabber: &Downloader{Client: client, Code: "qbittorrent", Landing: "/downloads"}} + r := s.For(context.Background(), ram) + took := s.Grab(context.Background(), ram, &r.Candidates[0], r.Candidates, false, Origin{By: OriginSchedule}) + if !took.Added || took.SavePath != "/downloads/abcd" { + t.Fatalf("took %+v", took) + } + if len(client.added) != 1 || !strings.Contains(strings.ToLower(client.added[0].Magnet), "abcd") || client.added[0].Tags[0] != Tag { + t.Errorf("added %+v", client.added) + } +} + +func TestGrab_RefusesWhatTheVerdictWouldNotTake(t *testing.T) { + client := &fakeClient{} + s := &Service{Grabber: &Downloader{Client: client, Landing: "/downloads"}} + review := Candidate{Verdict: Review, InfoHash: "aaaa", Source: indexer.Result{Magnet: "magnet:?xt=urn:btih:aaaa"}} + if took := s.Grab(context.Background(), ram, &review, nil, false, Origin{By: OriginSchedule}); !took.Refused { + t.Error("a review was taken unforced") + } + if took := s.Grab(context.Background(), ram, &review, nil, true, Origin{By: OriginOperator}); !took.Added { + t.Errorf("a forced review was not taken: %+v", took) + } + blocked := Candidate{Verdict: Blocked, InfoHash: "bbbb"} + if took := s.Grab(context.Background(), ram, &blocked, nil, true, Origin{By: OriginOperator}); !took.Refused { + t.Error("forcing took a release set aside") + } + if took := (&Service{}).Grab(context.Background(), ram, &review, nil, true, Origin{}); !took.Unconfigured || took.Missing != MissingClient { + t.Errorf("no client: %+v", took) + } +} + +func TestGrab_AClientThatAlreadyHasItCounts(t *testing.T) { + client := &fakeClient{addErr: downloader.ErrDuplicate} + s := &Service{Grabber: &Downloader{Client: client, Landing: "/downloads"}} + c := Candidate{Verdict: Grab, InfoHash: "aaaa", Source: indexer.Result{Magnet: "magnet:?xt=urn:btih:aaaa"}} + if took := s.Grab(context.Background(), ram, &c, nil, false, Origin{By: OriginSchedule}); !took.Added { + t.Errorf("a duplicate: %+v", took) + } +} + +func TestBetter_QualityThenSeedersThenYear(t *testing.T) { + flac := Candidate{Verdict: Grab, Tier: release.TierLossless, Seeders: 2} + mp3 := Candidate{Verdict: Grab, Tier: release.TierLossyHigh, Seeders: 90} + if !better(flac, mp3) || better(mp3, flac) { + t.Error("quality first among grabs") + } + busy := Candidate{Verdict: Grab, Tier: release.TierLossless, Seeders: 20} + if !better(busy, flac) { + t.Error("then seeders") + } + dated := Candidate{Verdict: Grab, Tier: release.TierLossless, Seeders: 2, YearAgrees: true} + if !better(dated, flac) { + t.Error("then the year") + } + if better(Candidate{Verdict: Review}, flac) || !better(flac, Candidate{Verdict: Review}) { + t.Error("a grab ranks above a review") + } +} + +func TestLandingFor_OneFolderPerTorrent(t *testing.T) { + if got := landingFor("/downloads", "abcd", "x"); got != "/downloads/abcd" { + t.Error(got) + } + if got := landingFor("/downloads", "", "Daft Punk: R.A.M. [FLAC]"); got != "/downloads/daft-punk-r-a-m--flac" { + t.Error(got) + } +} diff --git a/internal/api/admin_search.go b/internal/api/admin_search.go new file mode 100644 index 00000000..53b7dbe0 --- /dev/null +++ b/internal/api/admin_search.go @@ -0,0 +1,299 @@ +package api + +import ( + "context" + "encoding/json" + "errors" + "net/http" + "time" + + "github.com/go-chi/chi/v5" + "github.com/jackc/pgx/v5" + + "git.fabledsword.com/bvandeusen/minstrel/internal/acquire" + "git.fabledsword.com/bvandeusen/minstrel/internal/apierror" + "git.fabledsword.com/bvandeusen/minstrel/internal/db/dbq" + "git.fabledsword.com/bvandeusen/minstrel/internal/release" + "git.fabledsword.com/bvandeusen/minstrel/internal/wanted" +) + +// Searching for wants and grabbing releases (M502 #5521): the schedule's +// settings, each want's candidates, and a person's Search, Grab and "never +// this one". + +// searchTimeout bounds a search a person waits on: a Prowlarr fanning out to +// many trackers can take most of a minute. +const searchTimeout = 2 * time.Minute + +type searchSettingsView struct { + AutoGrab bool `json:"auto_grab"` + SearchIntervalHours int32 `json:"search_interval_hours"` + AllowedQualities []string `json:"allowed_qualities"` + Tiers []release.TierInfo `json:"tiers"` + LastPassAt *string `json:"last_pass_at"` + LastPass json.RawMessage `json:"last_pass"` +} + +func searchSettings(s dbq.AcquisitionSetting) searchSettingsView { + v := searchSettingsView{ + AutoGrab: s.AutoGrab, SearchIntervalHours: s.SearchIntervalHours, + AllowedQualities: s.AllowedQualities, Tiers: release.Tiers, LastPass: s.LastPass, + } + if v.AllowedQualities == nil { + v.AllowedQualities = []string{} + } + if len(v.LastPass) == 0 { + v.LastPass = json.RawMessage("{}") + } + if s.LastPassAt.Valid { + at := formatTimestamp(s.LastPassAt) + v.LastPassAt = &at + } + return v +} + +// handleGetSearchSettings implements GET /api/admin/acquisition/settings. +func (h *handlers) handleGetSearchSettings(w http.ResponseWriter, r *http.Request) { + s, err := dbq.New(h.pool).GetAcquisitionSettings(r.Context()) + if err != nil { + writeErrWithLog(w, h.logger, "admin search settings: get", apierror.Internal(err)) + return + } + writeJSON(w, http.StatusOK, searchSettings(s)) +} + +type putSearchSettingsBody struct { + AutoGrab bool `json:"auto_grab"` + SearchIntervalHours int32 `json:"search_interval_hours"` + AllowedQualities []string `json:"allowed_qualities"` +} + +// handlePutSearchSettings implements PUT /api/admin/acquisition/settings. +func (h *handlers) handlePutSearchSettings(w http.ResponseWriter, r *http.Request) { + var body putSearchSettingsBody + if !decodeBody(w, r, &body) { + return + } + if body.SearchIntervalHours < 0 || body.SearchIntervalHours > 720 { + writeErr(w, apierror.BadRequest("bad_interval", "search_interval_hours must be between 0 and 720")) + return + } + if len(body.AllowedQualities) == 0 { + writeErr(w, apierror.BadRequest("bad_quality", "allow at least one quality")) + return + } + for _, t := range body.AllowedQualities { + if !release.KnownTier(t) { + writeErr(w, apierror.BadRequest("bad_quality", "unknown quality "+t)) + return + } + } + s, err := dbq.New(h.pool).UpdateAcquisitionSettings(r.Context(), dbq.UpdateAcquisitionSettingsParams{ + AutoGrab: body.AutoGrab, SearchIntervalHours: body.SearchIntervalHours, AllowedQualities: body.AllowedQualities, + }) + if err != nil { + writeErrWithLog(w, h.logger, "admin search settings: put", apierror.Internal(err)) + return + } + writeJSON(w, http.StatusOK, searchSettings(s)) +} + +// candidateView is one release a want's last search found. +type candidateView struct { + Key string `json:"key"` + Name string `json:"name"` + InfoHash string `json:"info_hash"` + Indexer string `json:"indexer"` + SizeBytes int64 `json:"size_bytes"` + Seeders int32 `json:"seeders"` + FilesCount int32 `json:"files_count"` + Format string `json:"format"` + Quality string `json:"quality"` + Year int32 `json:"year"` + Verdict string `json:"verdict"` + Reason string `json:"reason"` + Reasons json.RawMessage `json:"reasons"` + FoldedFrom []string `json:"folded_from"` + SearchedAt string `json:"searched_at"` + Downloading bool `json:"downloading"` +} + +// handleListCandidates implements GET /api/admin/wanted/{id}/candidates. +func (h *handlers) handleListCandidates(w http.ResponseWriter, r *http.Request) { + id, ok := requireURLUUID(w, r, "id") + if !ok { + return + } + q := dbq.New(h.pool) + rows, err := q.ListWantCandidates(r.Context(), id) + if err != nil { + writeErrWithLog(w, h.logger, "admin wanted: candidates", apierror.Internal(err)) + return + } + live, err := q.GetLiveGrab(r.Context(), id) + if err != nil && !errors.Is(err, pgx.ErrNoRows) { + writeErrWithLog(w, h.logger, "admin wanted: live grab", apierror.Internal(err)) + return + } + out := make([]candidateView, 0, len(rows)) + for _, c := range rows { + v := candidateView{ + Key: c.Key, Name: c.Name, InfoHash: c.InfoHash, Indexer: c.Indexer, SizeBytes: c.SizeBytes, + Seeders: c.Seeders, FilesCount: c.FilesCount, Format: c.Format, Quality: c.Quality, Year: c.Year, + Verdict: c.Verdict, Reason: c.Reason, Reasons: c.Reasons, FoldedFrom: c.FoldedFrom, + SearchedAt: formatTimestamp(c.SearchedAt), + Downloading: live.ID.Valid && live.Key == c.Key, + } + if v.FoldedFrom == nil { + v.FoldedFrom = []string{} + } + if len(v.Reasons) == 0 { + v.Reasons = json.RawMessage("[]") + } + out = append(out, v) + } + writeJSON(w, http.StatusOK, map[string]any{"items": out}) +} + +// handleSearchWant implements POST /api/admin/wanted/{id}/search: search +// now, as the schedule would, taking the best release when auto-grab is on. +func (h *handlers) handleSearchWant(w http.ResponseWriter, r *http.Request) { + id, ok := requireURLUUID(w, r, "id") + if !ok { + return + } + ctx, cancel := context.WithTimeout(r.Context(), searchTimeout) + defer cancel() + res, err := h.searcher.SearchNow(ctx, id) + if h.writeSearchErr(w, err) { + return + } + writeJSON(w, http.StatusOK, res) +} + +type grabBody struct { + Key string `json:"key"` + Force bool `json:"force"` +} + +// grabView is what came of a person's grab. +type grabView struct { + Added bool `json:"added"` + Refused bool `json:"refused"` + Gone bool `json:"gone"` + Unconfigured bool `json:"unconfigured"` + Missing string `json:"missing,omitempty"` + Kept bool `json:"kept"` + Replaced bool `json:"replaced"` + Verdict string `json:"verdict,omitempty"` + Reason string `json:"reason,omitempty"` + Error string `json:"error,omitempty"` +} + +// handleGrabCandidate implements POST /api/admin/wanted/{id}/grab: take the +// release named by key. The search runs again first, so the indexer's link +// never leaves the server; force takes what the verdict would not. +func (h *handlers) handleGrabCandidate(w http.ResponseWriter, r *http.Request) { + admin, ok := requireUser(w, r) + if !ok { + return + } + id, ok := requireURLUUID(w, r, "id") + if !ok { + return + } + var body grabBody + if !decodeBody(w, r, &body) { + return + } + ctx, cancel := context.WithTimeout(r.Context(), searchTimeout) + defer cancel() + took, err := h.searcher.Take(ctx, id, body.Key, body.Force, admin.ID) + if h.writeSearchErr(w, err) { + return + } + v := grabView{ + Added: took.Added, Refused: took.Refused, Gone: took.Gone, Unconfigured: took.Unconfigured, + Missing: took.Missing, Error: took.Error, + } + if took.Chosen != nil { + v.Verdict, v.Reason = string(took.Chosen.Verdict), took.Chosen.Reason + } + if took.Contest != nil { + v.Kept, v.Replaced = took.Contest.Kept, took.Contest.Replaced + if took.Contest.Kept { + v.Reason = took.Contest.Reason + } + } + writeJSON(w, http.StatusOK, v) +} + +type blockBody struct { + Key string `json:"key"` +} + +// handleBlockCandidate implements POST /api/admin/wanted/{id}/block: never +// take this release again, for any want. +func (h *handlers) handleBlockCandidate(w http.ResponseWriter, r *http.Request) { + admin, ok := requireUser(w, r) + if !ok { + return + } + id, ok := requireURLUUID(w, r, "id") + if !ok { + return + } + var body blockBody + if !decodeBody(w, r, &body) { + return + } + c, err := dbq.New(h.pool).GetWantCandidate(r.Context(), dbq.GetWantCandidateParams{WantID: id, Key: body.Key}) + if errors.Is(err, pgx.ErrNoRows) { + writeErr(w, apierror.NotFound("candidate")) + return + } + if err != nil { + writeErrWithLog(w, h.logger, "admin wanted: block", apierror.Internal(err)) + return + } + if c.InfoHash == "" { + writeErr(w, apierror.BadRequest("no_infohash", "this release has no infohash to set aside by")) + return + } + if err := acquire.NewMemory(h.pool).Block(r.Context(), dbq.BlockReleaseParams{ + InfoHash: c.InfoHash, Name: c.Name, Reason: acquire.BlockOperator, WantID: id, BlockedBy: admin.ID, + }); err != nil { + writeErrWithLog(w, h.logger, "admin wanted: block", apierror.Internal(err)) + return + } + w.WriteHeader(http.StatusNoContent) +} + +// handleUnblockRelease implements DELETE /api/admin/release-blocks/{hash}. +func (h *handlers) handleUnblockRelease(w http.ResponseWriter, r *http.Request) { + removed, err := acquire.NewMemory(h.pool).Unblock(r.Context(), chi.URLParam(r, "hash")) + if err != nil { + writeErrWithLog(w, h.logger, "admin wanted: unblock", apierror.Internal(err)) + return + } + if !removed { + writeErr(w, apierror.NotFound("block")) + return + } + w.WriteHeader(http.StatusNoContent) +} + +// writeSearchErr answers a Searcher error, and says whether there was one. +func (h *handlers) writeSearchErr(w http.ResponseWriter, err error) bool { + switch { + case err == nil: + return false + case errors.Is(err, pgx.ErrNoRows): + writeErr(w, apierror.NotFound("want")) + case errors.Is(err, wanted.ErrWantHeld): + writeErr(w, apierror.Conflict("want_held", "the library already holds this album")) + default: + writeErrWithLog(w, h.logger, "admin wanted: search", apierror.Internal(err)) + } + return true +} diff --git a/internal/api/admin_wanted.go b/internal/api/admin_wanted.go index ab4a3eb2..6c740ab6 100644 --- a/internal/api/admin_wanted.go +++ b/internal/api/admin_wanted.go @@ -35,6 +35,25 @@ type wantView struct { WantedAt string `json:"wanted_at"` HeldAt *string `json:"held_at"` ImageURL string `json:"image_url"` + // Search is the schedule's last look; Grab the download running (#5521). + Search *wantSearchView `json:"search"` + Grab *wantGrabView `json:"grab"` +} + +type wantSearchView struct { + SearchedAt string `json:"searched_at"` + Outcome string `json:"outcome"` + Considered int32 `json:"considered"` + BestRelease string `json:"best_release"` + BestVerdict string `json:"best_verdict"` + Error string `json:"error"` +} + +type wantGrabView struct { + Name string `json:"name"` + GrabbedAt string `json:"grabbed_at"` + Origin string `json:"origin"` + Quality string `json:"quality"` } // wantCounts sizes each tab of the Wanted page. @@ -118,6 +137,22 @@ func (h *handlers) handleListWants(w http.ResponseWriter, r *http.Request) { if v.SecondaryTypes == nil { v.SecondaryTypes = []string{} } + if row.SearchedAt.Valid { + v.Search = &wantSearchView{ + SearchedAt: formatTimestamp(row.SearchedAt), Outcome: row.SearchOutcome, + Considered: row.SearchConsidered, BestRelease: row.BestRelease, + BestVerdict: row.BestVerdict, Error: row.SearchError, + } + } + if row.GrabName != nil { + v.Grab = &wantGrabView{Name: *row.GrabName, GrabbedAt: formatTimestamp(row.GrabbedAt)} + if row.GrabOrigin != nil { + v.Grab.Origin = *row.GrabOrigin + } + if row.GrabQuality != nil { + v.Grab.Quality = *row.GrabQuality + } + } out.Items = append(out.Items, v) } writeJSON(w, http.StatusOK, out) diff --git a/internal/api/api.go b/internal/api/api.go index 9d3e2721..08d2d231 100644 --- a/internal/api/api.go +++ b/internal/api/api.go @@ -77,6 +77,7 @@ func Mount(r chi.Router, pool *pgxpool.Pool, logger *slog.Logger, events *playev acquisition: acquire.NewStore(pool), catalogue: cat, wanted: wanted.New(pool, cat, logger), + searcher: wanted.NewSearcher(pool, cat, logger), catalogueSearches: newSearchCache(), catalogueArt: catalogue.NewArt(dataDir, artSources(coverSettings)), leveled: newLeveledRenderer(dataDir, loudSettings, logger), @@ -248,6 +249,14 @@ func Mount(r chi.Router, pool *pgxpool.Pool, logger *slog.Logger, events *playev // profiles that decide which of their releases count. admin.Get("/wanted", h.handleListWants) admin.Delete("/wanted/{id}", h.handleRemoveWant) + // Searching and grabbing (M502 #5521). + admin.Get("/acquisition/settings", h.handleGetSearchSettings) + admin.Put("/acquisition/settings", h.handlePutSearchSettings) + admin.Get("/wanted/{id}/candidates", h.handleListCandidates) + admin.Post("/wanted/{id}/search", h.handleSearchWant) + admin.Post("/wanted/{id}/grab", h.handleGrabCandidate) + admin.Post("/wanted/{id}/block", h.handleBlockCandidate) + admin.Delete("/release-blocks/{hash}", h.handleUnblockRelease) admin.Get("/follows", h.handleListFollows) admin.Get("/follows/{mbid}", h.handleGetFollow) admin.Put("/follows/{mbid}", h.handlePutFollow) @@ -416,6 +425,8 @@ type handlers struct { // wanted is the wanted list: follows, release-type profiles and wants // (M502 #5520). Its worker runs in main; the API reads and edits. wanted *wanted.Service + // searcher searches for wants and grabs releases for them (#5521). + searcher *wanted.Searcher // catalogueSearches reuses recent MusicBrainz answers; nil asks every // time, which is what handler tests get. catalogueSearches *searchCache diff --git a/internal/db/dbq/models.go b/internal/db/dbq/models.go index 75d88561..0d4214dd 100644 --- a/internal/db/dbq/models.go +++ b/internal/db/dbq/models.go @@ -217,6 +217,16 @@ type AcquisitionService struct { UpdatedAt pgtype.Timestamptz } +type AcquisitionSetting struct { + ID bool + AutoGrab bool + SearchIntervalHours int32 + AllowedQualities []string + LastPassAt pgtype.Timestamptz + LastPass []byte + UpdatedAt pgtype.Timestamptz +} + type Album struct { ID pgtype.UUID Title string @@ -420,6 +430,26 @@ type GeneralLikesArtist struct { LikedAt pgtype.Timestamptz } +type Grab struct { + ID pgtype.UUID + WantID pgtype.UUID + InfoHash string + Name string + Key string + Indexer string + Source string + Client string + SavePath string + Quality string + Origin string + Forced bool + GrabbedBy pgtype.UUID + GrabbedAt pgtype.Timestamptz + EndedAt pgtype.Timestamptz + EndReason string + EndDetail string +} + type LibraryChange struct { ID int64 EntityType string @@ -652,6 +682,16 @@ type RegistrationSetting struct { Mode string } +type ReleaseBlock struct { + InfoHash string + Name string + Reason string + Detail string + WantID pgtype.UUID + BlockedBy pgtype.UUID + BlockedAt pgtype.Timestamptz +} + type ReleaseTypeProfile struct { ID pgtype.UUID Name string @@ -976,6 +1016,33 @@ type Want struct { FulfilledAt pgtype.Timestamptz CreatedAt pgtype.Timestamptz UpdatedAt pgtype.Timestamptz + SearchedAt pgtype.Timestamptz + SearchOutcome string + SearchConsidered int32 + BestRelease string + BestVerdict string + SearchError string +} + +type WantCandidate struct { + WantID pgtype.UUID + Key string + Name string + InfoHash string + Indexer string + Source string + SizeBytes int64 + Seeders int32 + FilesCount int32 + Format string + Quality string + Year int32 + Verdict string + Reason string + Reasons []byte + FoldedFrom []string + Rank int32 + SearchedAt pgtype.Timestamptz } type YouMightLikeAlbum struct { diff --git a/internal/db/dbq/reacquisition.sql.go b/internal/db/dbq/reacquisition.sql.go index c549005b..c4ea7208 100644 --- a/internal/db/dbq/reacquisition.sql.go +++ b/internal/db/dbq/reacquisition.sql.go @@ -212,7 +212,7 @@ ON CONFLICT (release_group_mbid) DO UPDATE fulfilled_at = NULL, updated_at = now() WHERE wants.fulfilled_at IS NOT NULL -RETURNING id, release_group_mbid, artist_mbid, artist_name, title, primary_type, first_release_date, reason, album_id, created_by, wanted_at, fulfilled_at, created_at, updated_at +RETURNING id, release_group_mbid, artist_mbid, artist_name, title, primary_type, first_release_date, reason, album_id, created_by, wanted_at, fulfilled_at, created_at, updated_at, searched_at, search_outcome, search_considered, best_release, best_verdict, search_error ` type WantMissingAlbumParams struct { @@ -249,6 +249,12 @@ func (q *Queries) WantMissingAlbum(ctx context.Context, arg WantMissingAlbumPara &i.FulfilledAt, &i.CreatedAt, &i.UpdatedAt, + &i.SearchedAt, + &i.SearchOutcome, + &i.SearchConsidered, + &i.BestRelease, + &i.BestVerdict, + &i.SearchError, ) return i, err } diff --git a/internal/db/dbq/search.sql.go b/internal/db/dbq/search.sql.go new file mode 100644 index 00000000..b1043de2 --- /dev/null +++ b/internal/db/dbq/search.sql.go @@ -0,0 +1,585 @@ +// Code generated by sqlc. DO NOT EDIT. +// versions: +// sqlc v1.31.1 +// source: search.sql + +package dbq + +import ( + "context" + + "github.com/jackc/pgx/v5/pgtype" +) + +const blockRelease = `-- name: BlockRelease :exec +INSERT INTO release_blocks (info_hash, name, reason, detail, want_id, blocked_by) +VALUES ($1, $2, $3, $4, + $5, $6) +ON CONFLICT (info_hash) DO UPDATE + SET reason = EXCLUDED.reason, detail = EXCLUDED.detail, blocked_at = now() +` + +type BlockReleaseParams struct { + InfoHash string + Name string + Reason string + Detail string + WantID pgtype.UUID + BlockedBy pgtype.UUID +} + +func (q *Queries) BlockRelease(ctx context.Context, arg BlockReleaseParams) error { + _, err := q.db.Exec(ctx, blockRelease, + arg.InfoHash, + arg.Name, + arg.Reason, + arg.Detail, + arg.WantID, + arg.BlockedBy, + ) + return err +} + +const deleteWantCandidates = `-- name: DeleteWantCandidates :exec +DELETE FROM want_candidates WHERE want_id = $1 +` + +func (q *Queries) DeleteWantCandidates(ctx context.Context, wantID pgtype.UUID) error { + _, err := q.db.Exec(ctx, deleteWantCandidates, wantID) + return err +} + +const editionTrackSpan = `-- name: EditionTrackSpan :one +SELECT COALESCE(MIN(track_count) FILTER (WHERE status = 'Official'), MIN(track_count), 0)::int AS min_tracks, + COALESCE(MAX(track_count), 0)::int AS max_tracks + FROM mb_releases + WHERE release_group_mbid = $1 AND track_count > 0 +` + +type EditionTrackSpanRow struct { + MinTracks int32 + MaxTracks int32 +} + +// The fewest tracks an official edition of the group has (any edition when +// none is official), and the most any has. Zero when none is catalogued. +func (q *Queries) EditionTrackSpan(ctx context.Context, releaseGroupMbid string) (EditionTrackSpanRow, error) { + row := q.db.QueryRow(ctx, editionTrackSpan, releaseGroupMbid) + var i EditionTrackSpanRow + err := row.Scan(&i.MinTracks, &i.MaxTracks) + return i, err +} + +const endGrab = `-- name: EndGrab :exec +UPDATE grabs + SET ended_at = now(), end_reason = $1, end_detail = $2 + WHERE id = $3 AND ended_at IS NULL +` + +type EndGrabParams struct { + EndReason string + EndDetail string + ID pgtype.UUID +} + +func (q *Queries) EndGrab(ctx context.Context, arg EndGrabParams) error { + _, err := q.db.Exec(ctx, endGrab, arg.EndReason, arg.EndDetail, arg.ID) + return err +} + +const getAcquisitionSettings = `-- name: GetAcquisitionSettings :one + +SELECT id, auto_grab, search_interval_hours, allowed_qualities, last_pass_at, last_pass, updated_at FROM acquisition_settings WHERE id = true +` + +// Searching for wants and grabbing releases (M502 #5521). +func (q *Queries) GetAcquisitionSettings(ctx context.Context) (AcquisitionSetting, error) { + row := q.db.QueryRow(ctx, getAcquisitionSettings) + var i AcquisitionSetting + err := row.Scan( + &i.ID, + &i.AutoGrab, + &i.SearchIntervalHours, + &i.AllowedQualities, + &i.LastPassAt, + &i.LastPass, + &i.UpdatedAt, + ) + return i, err +} + +const getLiveGrab = `-- name: GetLiveGrab :one +SELECT id, want_id, info_hash, name, key, indexer, source, client, save_path, quality, origin, forced, grabbed_by, grabbed_at, ended_at, end_reason, end_detail FROM grabs WHERE want_id = $1 AND ended_at IS NULL +` + +func (q *Queries) GetLiveGrab(ctx context.Context, wantID pgtype.UUID) (Grab, error) { + row := q.db.QueryRow(ctx, getLiveGrab, wantID) + var i Grab + err := row.Scan( + &i.ID, + &i.WantID, + &i.InfoHash, + &i.Name, + &i.Key, + &i.Indexer, + &i.Source, + &i.Client, + &i.SavePath, + &i.Quality, + &i.Origin, + &i.Forced, + &i.GrabbedBy, + &i.GrabbedAt, + &i.EndedAt, + &i.EndReason, + &i.EndDetail, + ) + return i, err +} + +const getWantCandidate = `-- name: GetWantCandidate :one +SELECT want_id, key, name, info_hash, indexer, source, size_bytes, seeders, files_count, format, quality, year, verdict, reason, reasons, folded_from, rank, searched_at FROM want_candidates WHERE want_id = $1 AND key = $2 +` + +type GetWantCandidateParams struct { + WantID pgtype.UUID + Key string +} + +func (q *Queries) GetWantCandidate(ctx context.Context, arg GetWantCandidateParams) (WantCandidate, error) { + row := q.db.QueryRow(ctx, getWantCandidate, arg.WantID, arg.Key) + var i WantCandidate + err := row.Scan( + &i.WantID, + &i.Key, + &i.Name, + &i.InfoHash, + &i.Indexer, + &i.Source, + &i.SizeBytes, + &i.Seeders, + &i.FilesCount, + &i.Format, + &i.Quality, + &i.Year, + &i.Verdict, + &i.Reason, + &i.Reasons, + &i.FoldedFrom, + &i.Rank, + &i.SearchedAt, + ) + return i, err +} + +const getWantForSearch = `-- name: GetWantForSearch :one +SELECT w.id, w.release_group_mbid, w.artist_name, w.title, w.reason, w.fulfilled_at, + COALESCE(NULLIF(w.primary_type, ''), g.primary_type, '')::text AS primary_type, + COALESCE(g.secondary_types, '{}')::text[] AS secondary_types, + COALESCE(NULLIF(w.first_release_date, ''), g.first_release_date, '')::text AS first_release_date + FROM wants w + LEFT JOIN mb_release_groups g ON g.mbid = w.release_group_mbid + WHERE w.id = $1 +` + +type GetWantForSearchRow struct { + ID pgtype.UUID + ReleaseGroupMbid string + ArtistName string + Title string + Reason string + FulfilledAt pgtype.Timestamptz + PrimaryType string + SecondaryTypes []string + FirstReleaseDate string +} + +func (q *Queries) GetWantForSearch(ctx context.Context, id pgtype.UUID) (GetWantForSearchRow, error) { + row := q.db.QueryRow(ctx, getWantForSearch, id) + var i GetWantForSearchRow + err := row.Scan( + &i.ID, + &i.ReleaseGroupMbid, + &i.ArtistName, + &i.Title, + &i.Reason, + &i.FulfilledAt, + &i.PrimaryType, + &i.SecondaryTypes, + &i.FirstReleaseDate, + ) + return i, err +} + +const insertGrab = `-- name: InsertGrab :one +INSERT INTO grabs (want_id, info_hash, name, key, indexer, source, client, save_path, quality, + origin, forced, grabbed_by) +VALUES ($1, $2, $3, $4, $5, + $6, $7, $8, $9, + $10, $11, $12) +RETURNING id, want_id, info_hash, name, key, indexer, source, client, save_path, quality, origin, forced, grabbed_by, grabbed_at, ended_at, end_reason, end_detail +` + +type InsertGrabParams struct { + WantID pgtype.UUID + InfoHash string + Name string + Key string + Indexer string + Source string + Client string + SavePath string + Quality string + Origin string + Forced bool + GrabbedBy pgtype.UUID +} + +func (q *Queries) InsertGrab(ctx context.Context, arg InsertGrabParams) (Grab, error) { + row := q.db.QueryRow(ctx, insertGrab, + arg.WantID, + arg.InfoHash, + arg.Name, + arg.Key, + arg.Indexer, + arg.Source, + arg.Client, + arg.SavePath, + arg.Quality, + arg.Origin, + arg.Forced, + arg.GrabbedBy, + ) + var i Grab + err := row.Scan( + &i.ID, + &i.WantID, + &i.InfoHash, + &i.Name, + &i.Key, + &i.Indexer, + &i.Source, + &i.Client, + &i.SavePath, + &i.Quality, + &i.Origin, + &i.Forced, + &i.GrabbedBy, + &i.GrabbedAt, + &i.EndedAt, + &i.EndReason, + &i.EndDetail, + ) + return i, err +} + +const insertWantCandidate = `-- name: InsertWantCandidate :exec +INSERT INTO want_candidates (want_id, key, name, info_hash, indexer, source, size_bytes, seeders, + files_count, format, quality, year, verdict, reason, reasons, folded_from, rank) +VALUES ($1, $2, $3, $4, $5, + $6, $7, $8, $9, $10, + $11, $12, $13, $14, $15, + $16, $17) +ON CONFLICT (want_id, key) DO NOTHING +` + +type InsertWantCandidateParams struct { + WantID pgtype.UUID + Key string + Name string + InfoHash string + Indexer string + Source string + SizeBytes int64 + Seeders int32 + FilesCount int32 + Format string + Quality string + Year int32 + Verdict string + Reason string + Reasons []byte + FoldedFrom []string + Rank int32 +} + +func (q *Queries) InsertWantCandidate(ctx context.Context, arg InsertWantCandidateParams) error { + _, err := q.db.Exec(ctx, insertWantCandidate, + arg.WantID, + arg.Key, + arg.Name, + arg.InfoHash, + arg.Indexer, + arg.Source, + arg.SizeBytes, + arg.Seeders, + arg.FilesCount, + arg.Format, + arg.Quality, + arg.Year, + arg.Verdict, + arg.Reason, + arg.Reasons, + arg.FoldedFrom, + arg.Rank, + ) + return err +} + +const listReleaseBlocks = `-- name: ListReleaseBlocks :many +SELECT info_hash, name, reason, detail, want_id, blocked_by, blocked_at FROM release_blocks ORDER BY blocked_at DESC +` + +func (q *Queries) ListReleaseBlocks(ctx context.Context) ([]ReleaseBlock, error) { + rows, err := q.db.Query(ctx, listReleaseBlocks) + if err != nil { + return nil, err + } + defer rows.Close() + var items []ReleaseBlock + for rows.Next() { + var i ReleaseBlock + if err := rows.Scan( + &i.InfoHash, + &i.Name, + &i.Reason, + &i.Detail, + &i.WantID, + &i.BlockedBy, + &i.BlockedAt, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const listWantCandidates = `-- name: ListWantCandidates :many +SELECT want_id, key, name, info_hash, indexer, source, size_bytes, seeders, files_count, format, quality, year, verdict, reason, reasons, folded_from, rank, searched_at FROM want_candidates WHERE want_id = $1 ORDER BY rank, key +` + +func (q *Queries) ListWantCandidates(ctx context.Context, wantID pgtype.UUID) ([]WantCandidate, error) { + rows, err := q.db.Query(ctx, listWantCandidates, wantID) + if err != nil { + return nil, err + } + defer rows.Close() + var items []WantCandidate + for rows.Next() { + var i WantCandidate + if err := rows.Scan( + &i.WantID, + &i.Key, + &i.Name, + &i.InfoHash, + &i.Indexer, + &i.Source, + &i.SizeBytes, + &i.Seeders, + &i.FilesCount, + &i.Format, + &i.Quality, + &i.Year, + &i.Verdict, + &i.Reason, + &i.Reasons, + &i.FoldedFrom, + &i.Rank, + &i.SearchedAt, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const listWantsDueForSearch = `-- name: ListWantsDueForSearch :many +SELECT w.id, w.release_group_mbid, w.artist_name, w.title, w.reason, + COALESCE(NULLIF(w.primary_type, ''), g.primary_type, '')::text AS primary_type, + COALESCE(g.secondary_types, '{}')::text[] AS secondary_types, + COALESCE(NULLIF(w.first_release_date, ''), g.first_release_date, '')::text AS first_release_date + FROM wants w + LEFT JOIN mb_release_groups g ON g.mbid = w.release_group_mbid + WHERE w.fulfilled_at IS NULL + AND NOT EXISTS (SELECT 1 FROM grabs gr WHERE gr.want_id = w.id AND gr.ended_at IS NULL) + AND (w.searched_at IS NULL OR w.searched_at < now() - make_interval(hours => $1::int)) + ORDER BY w.searched_at IS NOT NULL, + CASE w.reason WHEN 'request' THEN 0 WHEN 'missing' THEN 1 ELSE 2 END, + w.searched_at NULLS FIRST, w.wanted_at, w.id + LIMIT $2 +` + +type ListWantsDueForSearchParams struct { + Hours int32 + MaxRows int32 +} + +type ListWantsDueForSearchRow struct { + ID pgtype.UUID + ReleaseGroupMbid string + ArtistName string + Title string + Reason string + PrimaryType string + SecondaryTypes []string + FirstReleaseDate string +} + +// Open wants with no download running that were never searched or not since +// @hours ago: never-searched first, then requests, then missing files, then +// followed artists' releases. +func (q *Queries) ListWantsDueForSearch(ctx context.Context, arg ListWantsDueForSearchParams) ([]ListWantsDueForSearchRow, error) { + rows, err := q.db.Query(ctx, listWantsDueForSearch, arg.Hours, arg.MaxRows) + if err != nil { + return nil, err + } + defer rows.Close() + var items []ListWantsDueForSearchRow + for rows.Next() { + var i ListWantsDueForSearchRow + if err := rows.Scan( + &i.ID, + &i.ReleaseGroupMbid, + &i.ArtistName, + &i.Title, + &i.Reason, + &i.PrimaryType, + &i.SecondaryTypes, + &i.FirstReleaseDate, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const markCandidatesBlocked = `-- name: MarkCandidatesBlocked :exec +UPDATE want_candidates + SET verdict = 'blocked', reason = $1, + reasons = jsonb_build_array(jsonb_build_object('kind', 'hard', 'code', 'blocked', 'text', $1::text)) + WHERE info_hash = $2 +` + +type MarkCandidatesBlockedParams struct { + Reason string + InfoHash string +} + +func (q *Queries) MarkCandidatesBlocked(ctx context.Context, arg MarkCandidatesBlockedParams) error { + _, err := q.db.Exec(ctx, markCandidatesBlocked, arg.Reason, arg.InfoHash) + return err +} + +const markCandidatesUnblocked = `-- name: MarkCandidatesUnblocked :exec +UPDATE want_candidates + SET verdict = 'review', reason = $1, + reasons = jsonb_build_array(jsonb_build_object('kind', 'soft', 'code', 'unblocked', 'text', $1::text)) + WHERE info_hash = $2 AND verdict = 'blocked' +` + +type MarkCandidatesUnblockedParams struct { + Reason string + InfoHash string +} + +func (q *Queries) MarkCandidatesUnblocked(ctx context.Context, arg MarkCandidatesUnblockedParams) error { + _, err := q.db.Exec(ctx, markCandidatesUnblocked, arg.Reason, arg.InfoHash) + return err +} + +const recordSearchPass = `-- name: RecordSearchPass :exec +UPDATE acquisition_settings SET last_pass_at = now(), last_pass = $1 WHERE id = true +` + +func (q *Queries) RecordSearchPass(ctx context.Context, report []byte) error { + _, err := q.db.Exec(ctx, recordSearchPass, report) + return err +} + +const recordWantSearch = `-- name: RecordWantSearch :exec +UPDATE wants + SET searched_at = now(), + search_outcome = $1, + search_considered = $2, + best_release = $3, + best_verdict = $4, + search_error = $5, + updated_at = now() + WHERE id = $6 +` + +type RecordWantSearchParams struct { + Outcome string + Considered int32 + BestRelease string + BestVerdict string + SearchError string + ID pgtype.UUID +} + +func (q *Queries) RecordWantSearch(ctx context.Context, arg RecordWantSearchParams) error { + _, err := q.db.Exec(ctx, recordWantSearch, + arg.Outcome, + arg.Considered, + arg.BestRelease, + arg.BestVerdict, + arg.SearchError, + arg.ID, + ) + return err +} + +const unblockRelease = `-- name: UnblockRelease :execrows +DELETE FROM release_blocks WHERE info_hash = $1 +` + +func (q *Queries) UnblockRelease(ctx context.Context, infoHash string) (int64, error) { + result, err := q.db.Exec(ctx, unblockRelease, infoHash) + if err != nil { + return 0, err + } + return result.RowsAffected(), nil +} + +const updateAcquisitionSettings = `-- name: UpdateAcquisitionSettings :one +UPDATE acquisition_settings + SET auto_grab = $1, + search_interval_hours = $2, + allowed_qualities = $3, + updated_at = now() + WHERE id = true +RETURNING id, auto_grab, search_interval_hours, allowed_qualities, last_pass_at, last_pass, updated_at +` + +type UpdateAcquisitionSettingsParams struct { + AutoGrab bool + SearchIntervalHours int32 + AllowedQualities []string +} + +func (q *Queries) UpdateAcquisitionSettings(ctx context.Context, arg UpdateAcquisitionSettingsParams) (AcquisitionSetting, error) { + row := q.db.QueryRow(ctx, updateAcquisitionSettings, arg.AutoGrab, arg.SearchIntervalHours, arg.AllowedQualities) + var i AcquisitionSetting + err := row.Scan( + &i.ID, + &i.AutoGrab, + &i.SearchIntervalHours, + &i.AllowedQualities, + &i.LastPassAt, + &i.LastPass, + &i.UpdatedAt, + ) + return i, err +} diff --git a/internal/db/dbq/wanted.sql.go b/internal/db/dbq/wanted.sql.go index 38414108..1374189c 100644 --- a/internal/db/dbq/wanted.sql.go +++ b/internal/db/dbq/wanted.sql.go @@ -206,7 +206,7 @@ func (q *Queries) GetReleaseTypeProfile(ctx context.Context, id pgtype.UUID) (Re } const getWantByReleaseGroup = `-- name: GetWantByReleaseGroup :one -SELECT id, release_group_mbid, artist_mbid, artist_name, title, primary_type, first_release_date, reason, album_id, created_by, wanted_at, fulfilled_at, created_at, updated_at FROM wants WHERE release_group_mbid = $1 +SELECT id, release_group_mbid, artist_mbid, artist_name, title, primary_type, first_release_date, reason, album_id, created_by, wanted_at, fulfilled_at, created_at, updated_at, searched_at, search_outcome, search_considered, best_release, best_verdict, search_error FROM wants WHERE release_group_mbid = $1 ` func (q *Queries) GetWantByReleaseGroup(ctx context.Context, releaseGroupMbid string) (Want, error) { @@ -227,6 +227,12 @@ func (q *Queries) GetWantByReleaseGroup(ctx context.Context, releaseGroupMbid st &i.FulfilledAt, &i.CreatedAt, &i.UpdatedAt, + &i.SearchedAt, + &i.SearchOutcome, + &i.SearchConsidered, + &i.BestRelease, + &i.BestVerdict, + &i.SearchError, ) return i, err } @@ -406,7 +412,7 @@ func (q *Queries) ListFollowsDue(ctx context.Context, arg ListFollowsDueParams) } const listOpenWants = `-- name: ListOpenWants :many -SELECT id, release_group_mbid, artist_mbid, artist_name, title, primary_type, first_release_date, reason, album_id, created_by, wanted_at, fulfilled_at, created_at, updated_at FROM wants +SELECT id, release_group_mbid, artist_mbid, artist_name, title, primary_type, first_release_date, reason, album_id, created_by, wanted_at, fulfilled_at, created_at, updated_at, searched_at, search_outcome, search_considered, best_release, best_verdict, search_error FROM wants WHERE fulfilled_at IS NULL ORDER BY wanted_at, id LIMIT $1 @@ -436,6 +442,12 @@ func (q *Queries) ListOpenWants(ctx context.Context, limit int32) ([]Want, error &i.FulfilledAt, &i.CreatedAt, &i.UpdatedAt, + &i.SearchedAt, + &i.SearchOutcome, + &i.SearchConsidered, + &i.BestRelease, + &i.BestVerdict, + &i.SearchError, ); err != nil { return nil, err } @@ -505,10 +517,14 @@ SELECT w.id, w.release_group_mbid, w.artist_mbid, w.artist_name, w.title, w.reas COALESCE(g.secondary_types, '{}')::text[] AS secondary_types, COALESCE(NULLIF(w.first_release_date, ''), g.first_release_date, '')::text AS first_release_date, u.username AS requested_by, - (SELECT ar.id FROM artists ar WHERE ar.mbid = w.artist_mbid) AS library_artist_id + (SELECT ar.id FROM artists ar WHERE ar.mbid = w.artist_mbid) AS library_artist_id, + -- The search schedule's last look (#5521), and the download running. + w.searched_at, w.search_outcome, w.search_considered, w.best_release, w.best_verdict, w.search_error, + gr.name AS grab_name, gr.grabbed_at, gr.origin AS grab_origin, gr.quality AS grab_quality FROM wants w LEFT JOIN users u ON u.id = w.created_by LEFT JOIN mb_release_groups g ON g.mbid = w.release_group_mbid + LEFT JOIN grabs gr ON gr.want_id = w.id AND gr.ended_at IS NULL WHERE CASE $1::text WHEN 'held' THEN w.fulfilled_at IS NOT NULL WHEN 'missing' THEN w.fulfilled_at IS NULL AND w.reason = 'missing' @@ -539,6 +555,16 @@ type ListWantsRow struct { FirstReleaseDate string RequestedBy *string LibraryArtistID pgtype.UUID + SearchedAt pgtype.Timestamptz + SearchOutcome string + SearchConsidered int32 + BestRelease string + BestVerdict string + SearchError string + GrabName *string + GrabbedAt pgtype.Timestamptz + GrabOrigin *string + GrabQuality *string } // The Wanted page: open wants (all but missing ones, or only missing ones), @@ -567,6 +593,16 @@ func (q *Queries) ListWants(ctx context.Context, arg ListWantsParams) ([]ListWan &i.FirstReleaseDate, &i.RequestedBy, &i.LibraryArtistID, + &i.SearchedAt, + &i.SearchOutcome, + &i.SearchConsidered, + &i.BestRelease, + &i.BestVerdict, + &i.SearchError, + &i.GrabName, + &i.GrabbedAt, + &i.GrabOrigin, + &i.GrabQuality, ); err != nil { return nil, err } @@ -791,7 +827,7 @@ INSERT INTO wants (release_group_mbid, artist_mbid, artist_name, title, primary_ VALUES ($1, $2, $3, $4, $5, $6, $7, $8) ON CONFLICT (release_group_mbid) DO UPDATE SET updated_at = wants.updated_at -RETURNING id, release_group_mbid, artist_mbid, artist_name, title, primary_type, first_release_date, reason, album_id, created_by, wanted_at, fulfilled_at, created_at, updated_at +RETURNING id, release_group_mbid, artist_mbid, artist_name, title, primary_type, first_release_date, reason, album_id, created_by, wanted_at, fulfilled_at, created_at, updated_at, searched_at, search_outcome, search_considered, best_release, best_verdict, search_error ` type WantReleaseGroupParams struct { @@ -834,6 +870,12 @@ func (q *Queries) WantReleaseGroup(ctx context.Context, arg WantReleaseGroupPara &i.FulfilledAt, &i.CreatedAt, &i.UpdatedAt, + &i.SearchedAt, + &i.SearchOutcome, + &i.SearchConsidered, + &i.BestRelease, + &i.BestVerdict, + &i.SearchError, ) return i, err } diff --git a/internal/db/migrations/0082_search_and_grab.down.sql b/internal/db/migrations/0082_search_and_grab.down.sql new file mode 100644 index 00000000..630d58e6 --- /dev/null +++ b/internal/db/migrations/0082_search_and_grab.down.sql @@ -0,0 +1,12 @@ +DROP TABLE release_blocks; +DROP TABLE grabs; +DROP TABLE want_candidates; +DROP INDEX wants_search_due_idx; +ALTER TABLE wants + DROP COLUMN searched_at, + DROP COLUMN search_outcome, + DROP COLUMN search_considered, + DROP COLUMN best_release, + DROP COLUMN best_verdict, + DROP COLUMN search_error; +DROP TABLE acquisition_settings; diff --git a/internal/db/migrations/0082_search_and_grab.up.sql b/internal/db/migrations/0082_search_and_grab.up.sql new file mode 100644 index 00000000..4c20ef3f --- /dev/null +++ b/internal/db/migrations/0082_search_and_grab.up.sql @@ -0,0 +1,102 @@ +-- 0082_search_and_grab.up.sql — searching for wants and grabbing releases +-- (Scribe milestone #502, #5521). +-- +-- acquisition_settings one row: whether the schedule takes the best release +-- by itself, how often a want is searched again, which +-- quality tiers are taken, and the last pass's report. +-- wants (columns) when each want was last searched and what came of it. +-- want_candidates every release the last search found for a want, with +-- its verdict and the reasons behind it. +-- grabs downloads started for wants; at most one live per want. +-- release_blocks releases never to take again, by infohash. + +CREATE TABLE acquisition_settings ( + id boolean PRIMARY KEY DEFAULT true CONSTRAINT acquisition_settings_one_row CHECK (id), + auto_grab boolean NOT NULL DEFAULT true, + -- 0 switches the schedule off. + search_interval_hours integer NOT NULL DEFAULT 24 + CONSTRAINT acquisition_settings_interval_check CHECK (search_interval_hours BETWEEN 0 AND 720), + -- Lidarr's "Any" profile (note #5527): trash lossy to lossless; unknown off. + allowed_qualities text[] NOT NULL DEFAULT ARRAY['lossless_hires', 'lossless', 'lossy_high', + 'lossy_standard', 'lossy_low', 'lossy_poor', 'lossy_trash'], + last_pass_at timestamptz, + last_pass jsonb NOT NULL DEFAULT '{}', + updated_at timestamptz NOT NULL DEFAULT now() +); + +INSERT INTO acquisition_settings DEFAULT VALUES; + +ALTER TABLE wants + ADD COLUMN searched_at timestamptz, + ADD COLUMN search_outcome text NOT NULL DEFAULT '' + CONSTRAINT wants_search_outcome_check CHECK (search_outcome IN ('', 'grabbed', 'waiting', 'nothing', 'error')), + ADD COLUMN search_considered integer NOT NULL DEFAULT 0, + ADD COLUMN best_release text NOT NULL DEFAULT '', + ADD COLUMN best_verdict text NOT NULL DEFAULT '', + ADD COLUMN search_error text NOT NULL DEFAULT ''; + +CREATE INDEX wants_search_due_idx ON wants (searched_at NULLS FIRST) WHERE fulfilled_at IS NULL; + +CREATE TABLE want_candidates ( + want_id uuid NOT NULL REFERENCES wants (id) ON DELETE CASCADE, + -- The infohash, or the name when the source gave none. + key text NOT NULL, + name text NOT NULL, + info_hash text NOT NULL DEFAULT '', + indexer text NOT NULL DEFAULT '', + source text NOT NULL DEFAULT '', + size_bytes bigint NOT NULL DEFAULT 0, + seeders integer NOT NULL DEFAULT 0, + files_count integer NOT NULL DEFAULT 0, + format text NOT NULL DEFAULT '', + quality text NOT NULL DEFAULT '', + year integer NOT NULL DEFAULT 0, + verdict text NOT NULL + CONSTRAINT want_candidates_verdict_check CHECK (verdict IN ('grab', 'review', 'refused', 'rejected', 'blocked')), + reason text NOT NULL DEFAULT '', + reasons jsonb NOT NULL DEFAULT '[]', + folded_from text[] NOT NULL DEFAULT '{}', + -- Position in the search's order, best first. + rank integer NOT NULL DEFAULT 0, + searched_at timestamptz NOT NULL DEFAULT now(), + PRIMARY KEY (want_id, key) +); + +CREATE INDEX want_candidates_info_hash_idx ON want_candidates (info_hash) WHERE info_hash <> ''; + +CREATE TABLE grabs ( + id uuid PRIMARY KEY DEFAULT gen_random_uuid(), + want_id uuid REFERENCES wants (id) ON DELETE SET NULL, + info_hash text NOT NULL DEFAULT '', + name text NOT NULL, + key text NOT NULL, + indexer text NOT NULL DEFAULT '', + source text NOT NULL DEFAULT '', + -- The acquisition_services row of the download client it went to. + client text NOT NULL, + save_path text NOT NULL, + quality text NOT NULL DEFAULT '', + origin text NOT NULL CONSTRAINT grabs_origin_check CHECK (origin IN ('schedule', 'operator')), + forced boolean NOT NULL DEFAULT false, + grabbed_by uuid REFERENCES users (id) ON DELETE SET NULL, + grabbed_at timestamptz NOT NULL DEFAULT now(), + ended_at timestamptz, + end_reason text NOT NULL DEFAULT '' + CONSTRAINT grabs_end_reason_check CHECK (end_reason IN ('', 'superseded')), + end_detail text NOT NULL DEFAULT '' +); + +-- One live grab per want; the service decides which, this is the backstop. +CREATE UNIQUE INDEX grabs_one_live_per_want ON grabs (want_id) WHERE ended_at IS NULL AND want_id IS NOT NULL; +CREATE INDEX grabs_info_hash_idx ON grabs (info_hash) WHERE info_hash <> ''; + +CREATE TABLE release_blocks ( + info_hash text PRIMARY KEY, + name text NOT NULL DEFAULT '', + -- operator: "never this one"; files: its .torrent held the wrong thing. + reason text NOT NULL CONSTRAINT release_blocks_reason_check CHECK (reason IN ('operator', 'files')), + detail text NOT NULL DEFAULT '', + want_id uuid REFERENCES wants (id) ON DELETE SET NULL, + blocked_by uuid REFERENCES users (id) ON DELETE SET NULL, + blocked_at timestamptz NOT NULL DEFAULT now() +); diff --git a/internal/db/queries/search.sql b/internal/db/queries/search.sql new file mode 100644 index 00000000..832e35f2 --- /dev/null +++ b/internal/db/queries/search.sql @@ -0,0 +1,121 @@ +-- Searching for wants and grabbing releases (M502 #5521). + +-- name: GetAcquisitionSettings :one +SELECT * FROM acquisition_settings WHERE id = true; + +-- name: UpdateAcquisitionSettings :one +UPDATE acquisition_settings + SET auto_grab = sqlc.arg(auto_grab), + search_interval_hours = sqlc.arg(search_interval_hours), + allowed_qualities = sqlc.arg(allowed_qualities), + updated_at = now() + WHERE id = true +RETURNING *; + +-- name: RecordSearchPass :exec +UPDATE acquisition_settings SET last_pass_at = now(), last_pass = sqlc.arg(report) WHERE id = true; + +-- name: ListWantsDueForSearch :many +-- Open wants with no download running that were never searched or not since +-- @hours ago: never-searched first, then requests, then missing files, then +-- followed artists' releases. +SELECT w.id, w.release_group_mbid, w.artist_name, w.title, w.reason, + COALESCE(NULLIF(w.primary_type, ''), g.primary_type, '')::text AS primary_type, + COALESCE(g.secondary_types, '{}')::text[] AS secondary_types, + COALESCE(NULLIF(w.first_release_date, ''), g.first_release_date, '')::text AS first_release_date + FROM wants w + LEFT JOIN mb_release_groups g ON g.mbid = w.release_group_mbid + WHERE w.fulfilled_at IS NULL + AND NOT EXISTS (SELECT 1 FROM grabs gr WHERE gr.want_id = w.id AND gr.ended_at IS NULL) + AND (w.searched_at IS NULL OR w.searched_at < now() - make_interval(hours => sqlc.arg(hours)::int)) + ORDER BY w.searched_at IS NOT NULL, + CASE w.reason WHEN 'request' THEN 0 WHEN 'missing' THEN 1 ELSE 2 END, + w.searched_at NULLS FIRST, w.wanted_at, w.id + LIMIT sqlc.arg(max_rows); + +-- name: GetWantForSearch :one +SELECT w.id, w.release_group_mbid, w.artist_name, w.title, w.reason, w.fulfilled_at, + COALESCE(NULLIF(w.primary_type, ''), g.primary_type, '')::text AS primary_type, + COALESCE(g.secondary_types, '{}')::text[] AS secondary_types, + COALESCE(NULLIF(w.first_release_date, ''), g.first_release_date, '')::text AS first_release_date + FROM wants w + LEFT JOIN mb_release_groups g ON g.mbid = w.release_group_mbid + WHERE w.id = $1; + +-- name: EditionTrackSpan :one +-- The fewest tracks an official edition of the group has (any edition when +-- none is official), and the most any has. Zero when none is catalogued. +SELECT COALESCE(MIN(track_count) FILTER (WHERE status = 'Official'), MIN(track_count), 0)::int AS min_tracks, + COALESCE(MAX(track_count), 0)::int AS max_tracks + FROM mb_releases + WHERE release_group_mbid = $1 AND track_count > 0; + +-- name: RecordWantSearch :exec +UPDATE wants + SET searched_at = now(), + search_outcome = sqlc.arg(outcome), + search_considered = sqlc.arg(considered), + best_release = sqlc.arg(best_release), + best_verdict = sqlc.arg(best_verdict), + search_error = sqlc.arg(search_error), + updated_at = now() + WHERE id = sqlc.arg(id); + +-- name: DeleteWantCandidates :exec +DELETE FROM want_candidates WHERE want_id = $1; + +-- name: InsertWantCandidate :exec +INSERT INTO want_candidates (want_id, key, name, info_hash, indexer, source, size_bytes, seeders, + files_count, format, quality, year, verdict, reason, reasons, folded_from, rank) +VALUES (sqlc.arg(want_id), sqlc.arg(key), sqlc.arg(name), sqlc.arg(info_hash), sqlc.arg(indexer), + sqlc.arg(source), sqlc.arg(size_bytes), sqlc.arg(seeders), sqlc.arg(files_count), sqlc.arg(format), + sqlc.arg(quality), sqlc.arg(year), sqlc.arg(verdict), sqlc.arg(reason), sqlc.arg(reasons), + sqlc.arg(folded_from), sqlc.arg(rank)) +ON CONFLICT (want_id, key) DO NOTHING; + +-- name: ListWantCandidates :many +SELECT * FROM want_candidates WHERE want_id = $1 ORDER BY rank, key; + +-- name: GetWantCandidate :one +SELECT * FROM want_candidates WHERE want_id = $1 AND key = $2; + +-- name: MarkCandidatesBlocked :exec +UPDATE want_candidates + SET verdict = 'blocked', reason = sqlc.arg(reason), + reasons = jsonb_build_array(jsonb_build_object('kind', 'hard', 'code', 'blocked', 'text', sqlc.arg(reason)::text)) + WHERE info_hash = sqlc.arg(info_hash); + +-- name: MarkCandidatesUnblocked :exec +UPDATE want_candidates + SET verdict = 'review', reason = sqlc.arg(reason), + reasons = jsonb_build_array(jsonb_build_object('kind', 'soft', 'code', 'unblocked', 'text', sqlc.arg(reason)::text)) + WHERE info_hash = sqlc.arg(info_hash) AND verdict = 'blocked'; + +-- name: GetLiveGrab :one +SELECT * FROM grabs WHERE want_id = $1 AND ended_at IS NULL; + +-- name: InsertGrab :one +INSERT INTO grabs (want_id, info_hash, name, key, indexer, source, client, save_path, quality, + origin, forced, grabbed_by) +VALUES (sqlc.narg(want_id), sqlc.arg(info_hash), sqlc.arg(name), sqlc.arg(key), sqlc.arg(indexer), + sqlc.arg(source), sqlc.arg(client), sqlc.arg(save_path), sqlc.arg(quality), + sqlc.arg(origin), sqlc.arg(forced), sqlc.narg(grabbed_by)) +RETURNING *; + +-- name: EndGrab :exec +UPDATE grabs + SET ended_at = now(), end_reason = sqlc.arg(end_reason), end_detail = sqlc.arg(end_detail) + WHERE id = sqlc.arg(id) AND ended_at IS NULL; + +-- name: ListReleaseBlocks :many +SELECT * FROM release_blocks ORDER BY blocked_at DESC; + +-- name: BlockRelease :exec +INSERT INTO release_blocks (info_hash, name, reason, detail, want_id, blocked_by) +VALUES (sqlc.arg(info_hash), sqlc.arg(name), sqlc.arg(reason), sqlc.arg(detail), + sqlc.narg(want_id), sqlc.narg(blocked_by)) +ON CONFLICT (info_hash) DO UPDATE + SET reason = EXCLUDED.reason, detail = EXCLUDED.detail, blocked_at = now(); + +-- name: UnblockRelease :execrows +DELETE FROM release_blocks WHERE info_hash = $1; diff --git a/internal/db/queries/wanted.sql b/internal/db/queries/wanted.sql index e28463d9..bde3219d 100644 --- a/internal/db/queries/wanted.sql +++ b/internal/db/queries/wanted.sql @@ -186,10 +186,14 @@ SELECT w.id, w.release_group_mbid, w.artist_mbid, w.artist_name, w.title, w.reas COALESCE(g.secondary_types, '{}')::text[] AS secondary_types, COALESCE(NULLIF(w.first_release_date, ''), g.first_release_date, '')::text AS first_release_date, u.username AS requested_by, - (SELECT ar.id FROM artists ar WHERE ar.mbid = w.artist_mbid) AS library_artist_id + (SELECT ar.id FROM artists ar WHERE ar.mbid = w.artist_mbid) AS library_artist_id, + -- The search schedule's last look (#5521), and the download running. + w.searched_at, w.search_outcome, w.search_considered, w.best_release, w.best_verdict, w.search_error, + gr.name AS grab_name, gr.grabbed_at, gr.origin AS grab_origin, gr.quality AS grab_quality FROM wants w LEFT JOIN users u ON u.id = w.created_by LEFT JOIN mb_release_groups g ON g.mbid = w.release_group_mbid + LEFT JOIN grabs gr ON gr.want_id = w.id AND gr.ended_at IS NULL WHERE CASE sqlc.arg(state)::text WHEN 'held' THEN w.fulfilled_at IS NOT NULL WHEN 'missing' THEN w.fulfilled_at IS NULL AND w.reason = 'missing' diff --git a/internal/dbtest/reset.go b/internal/dbtest/reset.go index 540276b8..fd1b7865 100644 --- a/internal/dbtest/reset.go +++ b/internal/dbtest/reset.go @@ -109,6 +109,9 @@ var dataTables = []string{ "mb_release_groups", // M502 #5519 "mb_artists", // M502 #5519 "requests", // M502 #5520 (was lidarr_requests) + "release_blocks", // M502 #5521 + "grabs", // M502 #5521 + "want_candidates", // M502 #5521 "wants", // M502 #5520 "follow_seen_groups", // M502 #5520 "follows", // M502 #5520 @@ -180,6 +183,14 @@ func ResetDB(t *testing.T, pool *pgxpool.Pool) { ); err != nil { t.Fatalf("dbtest.ResetDB reset release-type profiles: %v", err) } + // Search and grab settings (M502 #5521), a singleton reset the same way. + if _, err := pool.Exec(ctx, ` + UPDATE acquisition_settings + SET auto_grab = DEFAULT, search_interval_hours = DEFAULT, allowed_qualities = DEFAULT, + last_pass_at = NULL, last_pass = DEFAULT, updated_at = DEFAULT`, + ); err != nil { + t.Fatalf("dbtest.ResetDB reset acquisition settings: %v", err) + } // Notification email settings (M489 #5346), reset the same way. if _, err := pool.Exec(ctx, ` UPDATE notification_email_settings diff --git a/internal/release/files.go b/internal/release/files.go new file mode 100644 index 00000000..2bfeca9f --- /dev/null +++ b/internal/release/files.go @@ -0,0 +1,76 @@ +package release + +import ( + "path" + "strconv" + "strings" +) + +// File is one file inside a torrent. +type File struct { + Index int + Path string + Size int64 +} + +// Contents is what a torrent's file list holds. +type Contents struct { + Audio int + Video int + // Format is the format most of the audio files are in, "" when none. + Format string +} + +var videoExt = map[string]bool{ + "mkv": true, "mp4": true, "avi": true, "m2ts": true, "vob": true, + "mov": true, "webm": true, "wmv": true, "m4v": true, "ts": true, +} + +// Inspect counts a file list's audio and video files. +func Inspect(files []File) Contents { + var c Contents + formats := map[string]int{} + for _, f := range files { + ext := strings.TrimPrefix(strings.ToLower(path.Ext(f.Path)), ".") + if fm := formatOf(ext); fm != "" { + c.Audio++ + formats[fm]++ + } else if videoExt[ext] { + c.Video++ + } + } + best := 0 + for fm, n := range formats { + if n > best || (n == best && formatRank[fm] > formatRank[c.Format]) { + c.Format, best = fm, n + } + } + return c +} + +// JudgeFiles says what a torrent's files say about whether it is the album: +// a veto (code and sentence), or a doubt for a person, or nothing. +func JudgeFiles(w Want, files []File) (veto, reason, doubt string) { + if len(files) == 0 { + return "", "", "" + } + c := Inspect(files) + switch { + case c.Audio == 0 && c.Video > 0: + return VetoVideo, "It holds video files and no audio.", "" + case c.Audio == 0: + return VetoNoAudio, "It holds no audio files Minstrel can read.", "" + case w.MinTracks > 0 && c.Audio < w.MinTracks: + return VetoIncomplete, "It holds " + tracks(c.Audio) + "; the shortest edition has " + tracks(w.MinTracks) + ".", "" + case w.MaxTracks > 0 && c.Audio > 2*w.MaxTracks+4: + return "", "", "It holds " + tracks(c.Audio) + ", more than any edition of the album has: probably more than one album." + } + return "", "", "" +} + +func tracks(n int) string { + if n == 1 { + return "1 track" + } + return strconv.Itoa(n) + " tracks" +} diff --git a/internal/release/match.go b/internal/release/match.go new file mode 100644 index 00000000..77ae1575 --- /dev/null +++ b/internal/release/match.go @@ -0,0 +1,210 @@ +package release + +import "strings" + +// Want is the album a search is for, as the catalogue describes it. +type Want struct { + Artist string + Title string + Year int + PrimaryType string // Album, EP, Single, Broadcast, Other + SecondaryTypes []string + // MinTracks and MaxTracks span the group's editions' track counts; zero + // when the catalogue has not read them. + MinTracks int + MaxTracks int +} + +// Veto codes: why a name is not the album. Kept on a candidate as its +// reason's code. +const ( + VetoArtist = "wrong_artist" + VetoTitle = "wrong_title" + VetoVideo = "video" + VetoDiscography = "discography" + VetoVersion = "other_version" + VetoType = "wrong_type" + VetoNoAudio = "no_audio" + VetoIncomplete = "incomplete" +) + +// Match is what a name says about whether it is the album wanted. +type Match struct { + // Veto is a code from above when the name is not the album, with Reason + // saying why in a sentence. + Veto string + Reason string + // Doubt is set when the name may be the album and cannot say for sure: a + // person decides. It is a sentence. + Doubt string + // YearAgrees: the name states the album's year, give or take one. + YearAgrees bool +} + +// versions are words that make a release a different recording of the +// album, with what to call it. A word the wanted title or artist holds does +// not count ("Live Through This", "Instrumental Core"). +var versions = []struct { + words []string + what string + // typ is the MusicBrainz secondary type that wants this version. + typ string +}{ + {[]string{"instrumental", "instrumentals"}, "an instrumental version", ""}, + {[]string{"drumless"}, "a drumless edition", ""}, + {[]string{"karaoke"}, "a karaoke version", ""}, + {[]string{"acapella", "acappella", "accapella", "acapellas"}, "an a cappella version", ""}, + {[]string{"remix", "remixes", "remixed"}, "a remix release", "Remix"}, + {[]string{"live", "bootleg"}, "a live recording", "Live"}, + {[]string{"demo", "demos"}, "demo recordings", "Demo"}, + // Not "covers": in a release name that is usually the artwork scans. + {[]string{"tribute", "lullaby", "lullabies"}, "a tribute release", ""}, + {[]string{"nightcore", "slowed", "sped"}, "a slowed or sped-up edit", ""}, + {[]string{"commentary", "interview", "interviews"}, "commentary or interviews", "Interview"}, +} + +// collections are words naming more than one album. +var collections = []string{"collection", "anthology", "albums", "boxset", "discography"} + +// Judge says whether a release name is the album wanted. It looks only at +// the name; JudgeFiles looks at what a torrent holds. +func Judge(w Want, r Release) Match { + var m Match + m.YearAgrees = r.Year != 0 && w.Year != 0 && abs(r.Year-w.Year) <= 1 + if r.Video != "" { + m.Veto, m.Reason = VetoVideo, "A video, not audio (“"+r.Video+"”)." + return m + } + artist := Tokens(w.Artist) + title := Tokens(w.Title) + if len(artist) > 0 && !contains(r.Tokens, artist) && !contains(r.Tokens, withoutThe(artist)) { + m.Veto, m.Reason = VetoArtist, "The name doesn’t mention "+w.Artist+"." + return m + } + if len(title) > 0 && !contains(r.Tokens, title) && !contains(r.Tokens, bare(w.Title)) { + m.Veto, m.Reason = VetoTitle, "The name doesn’t contain the title “"+w.Title+"”." + return m + } + own := map[string]bool{} + for _, t := range append(append([]string{}, title...), artist...) { + own[t] = true + } + if r.Discography != "" && !own[r.Discography] { + m.Veto, m.Reason = VetoDiscography, "A discography or collection ("+r.Discography+"), and one album is wanted." + return m + } + for _, c := range collections { + if !own[c] && has(r.Tokens, c) { + m.Veto, m.Reason = VetoDiscography, "A collection of albums (“"+c+"”), and one album is wanted." + return m + } + } + for _, v := range versions { + if v.typ != "" && hasType(w.SecondaryTypes, v.typ) { + continue + } + for _, word := range v.words { + if !own[word] && has(r.Tokens, word) { + m.Veto, m.Reason = VetoVersion, "This is "+v.what+" (“"+word+"”)." + return m + } + } + } + if w.PrimaryType == "Album" || w.PrimaryType == "EP" { + if !own["single"] && has(r.Tokens, "single") { + m.Veto, m.Reason = VetoType, "A single, and the "+strings.ToLower(w.PrimaryType)+" is wanted." + return m + } + } + if w.PrimaryType == "Album" && !own["ep"] && has(r.Tokens, "ep") { + m.Veto, m.Reason = VetoType, "An EP, and the album is wanted." + return m + } + // A self-titled album: every release of the artist names it once. Two + // mentions, or its year, say it is this one. + if len(title) > 0 && equal(title, artist) && count(r.Tokens, title) < 2 && !m.YearAgrees { + m.Doubt = "The album shares the artist’s name, and nothing else in the name says it is this album." + } + return m +} + +func abs(n int) int { + if n < 0 { + return -n + } + return n +} + +func withoutThe(t []string) []string { + if len(t) > 1 && t[0] == "the" { + return t[1:] + } + return t +} + +// bare is a title without a trailing bracketed part ("Album (Remastered)"), +// for names that leave it off. +func bare(title string) []string { + if i := strings.IndexAny(title, "(["); i > 0 { + return Tokens(title[:i]) + } + return nil +} + +// contains says whether want's words appear in name in order and together, +// or, for two or more words, run together ("ACDC" for AC/DC). A single word +// is never looked for run together: "muse" is inside "museum". +func contains(name, want []string) bool { + if len(want) == 0 { + return false + } + if count(name, want) > 0 { + return true + } + joined := strings.Join(want, "") + if len(want) < 2 || len(joined) < 4 { + return false + } + return strings.Contains(strings.Join(name, ""), joined) +} + +// count is how often want's words appear in name in order and together. +func count(name, want []string) int { + n := 0 + for i := 0; i+len(want) <= len(name); i++ { + if equal(name[i:i+len(want)], want) { + n++ + } + } + return n +} + +func equal(a, b []string) bool { + if len(a) != len(b) { + return false + } + for i := range a { + if a[i] != b[i] { + return false + } + } + return true +} + +func has(tokens []string, word string) bool { + for _, t := range tokens { + if t == word { + return true + } + } + return false +} + +func hasType(types []string, t string) bool { + for _, x := range types { + if strings.EqualFold(x, t) { + return true + } + } + return false +} diff --git a/internal/release/parse.go b/internal/release/parse.go new file mode 100644 index 00000000..956d548d --- /dev/null +++ b/internal/release/parse.go @@ -0,0 +1,289 @@ +// Package release reads music release names and judges them against a +// wanted album (M502 #5521): what a name says it is (format, bitrate, bit +// depth, source, year, edition), whether it is the album wanted, and which +// quality tier it sits in. Nothing here touches the network or the database, +// so every rule is testable on a string. +package release + +import ( + "regexp" + "strconv" + "strings" +) + +// Formats a name can state. +const ( + FormatFLAC = "FLAC" + FormatALAC = "ALAC" + FormatAPE = "APE" + FormatWV = "WavPack" + FormatWAV = "WAV" + FormatAIFF = "AIFF" + FormatDSD = "DSD" + FormatMP3 = "MP3" + FormatAAC = "AAC" + FormatVorbis = "Vorbis" + FormatOpus = "Opus" +) + +// Sources a name can state. +const ( + SourceWEB = "WEB" + SourceCD = "CD" + SourceVinyl = "Vinyl" + SourceSACD = "SACD" + SourceCassette = "Cassette" +) + +// Release is what a release name says about itself. Zero values mean the +// name does not say. +type Release struct { + Name string + Year int + Format string + // Bitrate is a constant bitrate in kbps; VBR is a LAME preset (V0, V1, V2). + Bitrate int + VBR string + // BitDepth (16, 24) and SampleRate (kHz) for lossless. + BitDepth int + SampleRate float64 + Source string + // Video is the marker that says this is a video ("x264", "1080p"), or "". + Video string + // Discography: a year range or a word naming many albums. + Discography string + // Tokens are the name's words, normalised for matching. + Tokens []string +} + +var ( + formatRe = regexp.MustCompile(`\b(flac|alac|ape|wavpack|wv|wav|aiff|dsd|dsf|dff|mp3|aac|m4a|ogg|vorbis|opus)\b`) + // "24-96", "24/192", "16-44 1" (44.1 after dots become spaces). + depthRateRe = regexp.MustCompile(`\b(16|24|32) ?[-/ ] ?(44 1|44|48|88 2|88|96|176 4|176|192|352 8|384)\b`) + depthRe = regexp.MustCompile(`\b(16|24|32) ?-?bits?\b`) + rateRe = regexp.MustCompile(`\b(44 1|48|88 2|96|176 4|192) ?khz\b`) + hiResRe = regexp.MustCompile(`\bhi ?-?res\b`) + codecRateRe = regexp.MustCompile(`\b(?:mp3|aac|m4a|cbr|ogg|opus)[ \-@]*(320|256|224|192|160|128|112|96|64)\b`) + kbpsRe = regexp.MustCompile(`\b(320|256|224|192|160|128|112|96|64) ?(?:kbps|kbit|kb/s|k)\b`) + vbrRe = regexp.MustCompile(`\bv([0-2])\b`) + webRe = regexp.MustCompile(`\bweb(?: ?-?dl)?\b`) + cdRe = regexp.MustCompile(`\bcd ?(?:rip)?\b`) + vinylRe = regexp.MustCompile(`\b(?:vinyl|vinylrip|lp ?rip|needle ?drop)\b`) + sacdRe = regexp.MustCompile(`\bsacd\b`) + cassetteRe = regexp.MustCompile(`\b(?:cassette|tape ?rip)\b`) + yearRe = regexp.MustCompile(`\b(19[3-9]\d|20[0-4]\d)\b`) + bracketYear = regexp.MustCompile(`[(\[{] ?(19[3-9]\d|20[0-4]\d) ?[)\]}]`) + yearRangeRe = regexp.MustCompile(`\b(19[3-9]\d|20[0-4]\d) ?(?:-|–|to) ?(19[3-9]\d|20[0-4]\d)\b`) + videoRe = regexp.MustCompile(`\b(x264|x265|h ?264|h ?265|hevc|xvid|divx|1080p|1080i|720p|2160p|bluray|blu ?ray|bdrip|brrip|bdremux|dvdrip|dvd9|dvd5|webrip|hdtv|mkv|avi|m2ts)\b`) + discogRe = regexp.MustCompile(`\b(discography|discografia|discographie|diskografie|discografie)\b`) +) + +// Parse reads what a release name says. +func Parse(name string) Release { + r := Release{Name: name} + s := spaced(name) + + r.Format = bestFormat(formatRe.FindAllStringSubmatch(s, -1)) + + if m := depthRateRe.FindStringSubmatch(s); m != nil { + r.BitDepth, _ = strconv.Atoi(m[1]) + r.SampleRate = rate(m[2]) + } + if r.BitDepth == 0 { + if m := depthRe.FindStringSubmatch(s); m != nil { + r.BitDepth, _ = strconv.Atoi(m[1]) + } + } + if r.SampleRate == 0 { + if m := rateRe.FindStringSubmatch(s); m != nil { + r.SampleRate = rate(m[1]) + } + } + if r.BitDepth == 0 && hiResRe.MatchString(s) { + r.BitDepth = 24 + } + + if m := codecRateRe.FindStringSubmatch(s); m != nil { + r.Bitrate, _ = strconv.Atoi(m[1]) + } else if m := kbpsRe.FindStringSubmatch(s); m != nil { + r.Bitrate, _ = strconv.Atoi(m[1]) + } + // A bare "V0" is a LAME preset only beside MP3 or VBR; elsewhere it can + // be part of a title. + if m := vbrRe.FindStringSubmatch(s); m != nil && (r.Format == FormatMP3 || strings.Contains(s, "vbr")) { + r.VBR = "V" + m[1] + } + if r.Format == "" && (r.Bitrate > 0 || r.VBR != "") { + r.Format = FormatMP3 + } + + switch { + case sacdRe.MatchString(s): + r.Source = SourceSACD + case vinylRe.MatchString(s): + r.Source = SourceVinyl + case webRe.MatchString(s): + r.Source = SourceWEB + case cassetteRe.MatchString(s): + r.Source = SourceCassette + case cdRe.MatchString(s): + r.Source = SourceCD + } + + if m := videoRe.FindStringSubmatch(s); m != nil { + r.Video = m[1] + } + if m := discogRe.FindStringSubmatch(s); m != nil { + r.Discography = m[1] + } else if m := yearRangeRe.FindStringSubmatch(s); m != nil && m[2] > m[1] { + r.Discography = m[1] + "–" + m[2] + } + r.Year = year(s) + r.Tokens = Tokens(name) + return r +} + +// spaced lowercases a name and turns its separators into spaces, keeping +// the slash in "24/96" and "kb/s". +func spaced(name string) string { + var b strings.Builder + for _, c := range strings.ToLower(name) { + switch c { + case '_', '.', ',': + b.WriteByte(' ') + default: + b.WriteRune(c) + } + } + return b.String() +} + +// rate reads a sample rate in kHz; "44", "88" and "176" are the short +// spellings of 44.1, 88.2 and 176.4. +func rate(s string) float64 { + switch s { + case "44": + return 44.1 + case "88": + return 88.2 + case "176": + return 176.4 + } + f, _ := strconv.ParseFloat(strings.ReplaceAll(s, " ", "."), 64) + return f +} + +// formatRank orders formats a name might state together ("FLAC + MP3"): +// the best is the one it is described by. +var formatRank = map[string]int{ + FormatDSD: 11, FormatFLAC: 10, FormatALAC: 9, FormatWV: 8, FormatAPE: 7, FormatWAV: 6, FormatAIFF: 5, + FormatOpus: 4, FormatAAC: 3, FormatVorbis: 2, FormatMP3: 1, +} + +func bestFormat(found [][]string) string { + best := "" + for _, m := range found { + f := formatOf(m[1]) + if formatRank[f] > formatRank[best] { + best = f + } + } + return best +} + +// formatOf names the format a word or file extension stands for. +func formatOf(word string) string { + switch word { + case "flac": + return FormatFLAC + case "alac": + return FormatALAC + case "ape": + return FormatAPE + case "wavpack", "wv": + return FormatWV + case "wav": + return FormatWAV + case "aiff", "aif": + return FormatAIFF + case "dsd", "dsf", "dff": + return FormatDSD + case "mp3": + return FormatMP3 + case "aac", "m4a": + return FormatAAC + case "ogg", "vorbis": + return FormatVorbis + case "opus": + return FormatOpus + } + return "" +} + +// year is the release year a name states: one in brackets when there is +// one (so the title "1989" is not read as its year), else the last. +func year(s string) int { + if m := bracketYear.FindStringSubmatch(s); m != nil { + y, _ := strconv.Atoi(m[1]) + return y + } + all := yearRe.FindAllString(s, -1) + if len(all) == 0 { + return 0 + } + y, _ := strconv.Atoi(all[len(all)-1]) + return y +} + +// fold maps the accented letters music names carry to plain ones. +var fold = map[rune]string{ + 'à': "a", 'á': "a", 'â': "a", 'ã': "a", 'ä': "a", 'å': "a", 'ā': "a", 'ą': "a", + 'æ': "ae", 'ç': "c", 'ć': "c", 'č': "c", + 'è': "e", 'é': "e", 'ê': "e", 'ë': "e", 'ē': "e", 'ė': "e", 'ę': "e", 'ě': "e", + 'ì': "i", 'í': "i", 'î': "i", 'ï': "i", 'ī': "i", 'ı': "i", + 'ñ': "n", 'ń': "n", 'ň': "n", + 'ò': "o", 'ó': "o", 'ô': "o", 'õ': "o", 'ö': "o", 'ø': "o", 'ō': "o", 'ő': "o", + 'œ': "oe", 'ù': "u", 'ú': "u", 'û': "u", 'ü': "u", 'ū': "u", 'ů': "u", 'ű': "u", + 'ý': "y", 'ÿ': "y", 'ß': "ss", 'š': "s", 'ś': "s", 'ş': "s", + 'ž': "z", 'ź': "z", 'ż': "z", 'ł': "l", 'đ': "d", 'ð': "d", 'þ': "th", 'ř': "r", +} + +// abbreviations are spelled out on both sides before comparing. +var abbreviations = map[string]string{"vol": "volume", "pt": "part", "n": "and"} + +// Tokens are a name's words for matching: lowercase, accents folded, "&" +// and "+" read as "and", apostrophes dropped (so "Don't" is "dont"), every +// other mark a word break. +func Tokens(s string) []string { + var b strings.Builder + for _, c := range strings.ToLower(s) { + switch { + case c >= 'a' && c <= 'z', c >= '0' && c <= '9': + b.WriteRune(c) + case c == '\'' || c == '’' || c == '`' || c == '´': + case c == '&' || c == '+': + b.WriteString(" and ") + default: + if f, ok := fold[c]; ok { + b.WriteString(f) + } else if c > 127 && isLetter(c) { + b.WriteRune(c) + } else { + b.WriteByte(' ') + } + } + } + words := strings.Fields(b.String()) + for i, w := range words { + if full, ok := abbreviations[w]; ok { + words[i] = full + } + } + return words +} + +// isLetter keeps letters of scripts the fold table does not cover (Cyrillic, +// CJK) as letters rather than breaks. +func isLetter(c rune) bool { + return (c >= 0x0400 && c <= 0x04FF) || (c >= 0x0370 && c <= 0x03FF) || c >= 0x3040 +} diff --git a/internal/release/quality.go b/internal/release/quality.go new file mode 100644 index 00000000..66590f9a --- /dev/null +++ b/internal/release/quality.go @@ -0,0 +1,169 @@ +package release + +import ( + "fmt" + "strconv" + "strings" +) + +// Tier is a rung of the quality ladder, Lidarr's names for them. +type Tier string + +// The ladder, best first. +const ( + TierLosslessHiRes Tier = "lossless_hires" + TierLossless Tier = "lossless" + TierLossyHigh Tier = "lossy_high" + TierLossyStandard Tier = "lossy_standard" + TierLossyLow Tier = "lossy_low" + TierLossyPoor Tier = "lossy_poor" + TierLossyTrash Tier = "lossy_trash" + TierUnknown Tier = "unknown" +) + +// losslessCategory is Newznab's Audio/Lossless. +const losslessCategory = 3040 + +// TierInfo is a rung as the settings page shows it. +type TierInfo struct { + Tier Tier `json:"tier"` + Label string `json:"label"` + What string `json:"what"` +} + +// Tiers is the ladder, best first. +var Tiers = []TierInfo{ + {TierLosslessHiRes, "Lossless, hi-res", "24-bit or above 48 kHz: FLAC, ALAC, DSD"}, + {TierLossless, "Lossless", "FLAC, ALAC, APE, WavPack, WAV at CD quality"}, + {TierLossyHigh, "High quality lossy", "MP3 320 or V0, AAC 256 and up"}, + {TierLossyStandard, "Standard quality lossy", "MP3 224–256 or V1/V2, AAC 192, Vorbis, Opus, or a lossy format with no bitrate stated"}, + {TierLossyLow, "Low quality lossy", "160–192 kbps"}, + {TierLossyPoor, "Poor quality lossy", "128 kbps"}, + {TierLossyTrash, "Trash quality lossy", "Under 128 kbps"}, + {TierUnknown, "Unknown", "Nothing says what format it is"}, +} + +// DefaultTiers is what a new install allows: Lidarr's "Any" profile, every +// format from trash lossy to lossless (note #5527). Unknown is left out; a +// release that names no format has its .torrent read to find out. +var DefaultTiers = []Tier{ + TierLosslessHiRes, TierLossless, TierLossyHigh, TierLossyStandard, + TierLossyLow, TierLossyPoor, TierLossyTrash, +} + +// KnownTier says whether t is a rung of the ladder. +func KnownTier(t string) bool { + for _, info := range Tiers { + if string(info.Tier) == t { + return true + } + } + return false +} + +// Rank orders tiers: higher is better, Unknown lowest. +func Rank(t Tier) int { + for i, info := range Tiers { + if info.Tier == t { + return len(Tiers) - i + } + } + return 0 +} + +// Label is a tier's name for a person. +func Label(t Tier) string { + for _, info := range Tiers { + if info.Tier == t { + return info.Label + } + } + return string(t) +} + +// TierOf places a release on the ladder. category is the indexer's; 3040 +// (Lossless) stands in for a name that states no format. +func TierOf(r Release, category int) Tier { + switch r.Format { + case FormatDSD: + return TierLosslessHiRes + case FormatFLAC, FormatALAC, FormatAPE, FormatWV, FormatWAV, FormatAIFF: + if r.BitDepth >= 24 || r.SampleRate > 48 { + return TierLosslessHiRes + } + return TierLossless + case FormatMP3: + switch r.VBR { + case "V0": + return TierLossyHigh + case "V1", "V2": + return TierLossyStandard + } + return byBitrate(r.Bitrate, 320) + case FormatAAC, FormatVorbis, FormatOpus: + return byBitrate(r.Bitrate, 256) + } + if category == losslessCategory { + return TierLossless + } + return TierUnknown +} + +// byBitrate places a lossy bitrate; high is the bitrate from which the codec +// counts as high quality. An unstated bitrate is standard. +func byBitrate(kbps, high int) Tier { + switch { + case kbps == 0: + return TierLossyStandard + case kbps >= high: + return TierLossyHigh + case kbps >= 224 || (high == 256 && kbps >= 192): + return TierLossyStandard + case kbps >= 160: + return TierLossyLow + case kbps >= 128: + return TierLossyPoor + } + return TierLossyTrash +} + +// Describe is a release's format in a few words: "FLAC 24-bit/96 kHz", +// "MP3 V0", "AAC 256". Empty when the name states none. +func Describe(r Release) string { + if r.Format == "" { + return "" + } + parts := []string{r.Format} + switch { + case r.BitDepth > 0 && r.SampleRate > 0: + parts = append(parts, fmt.Sprintf("%d-bit/%s kHz", r.BitDepth, khz(r.SampleRate))) + case r.BitDepth > 0: + parts = append(parts, fmt.Sprintf("%d-bit", r.BitDepth)) + case r.VBR != "": + parts = append(parts, r.VBR) + case r.Bitrate > 0: + parts = append(parts, strconv.Itoa(r.Bitrate)) + } + return strings.Join(parts, " ") +} + +func khz(f float64) string { + return strings.TrimSuffix(strconv.FormatFloat(f, 'f', 1, 64), ".0") +} + +// Profile is which tiers the operator takes. +type Profile struct { + Allowed map[Tier]bool +} + +// NewProfile builds a profile from the stored tier list. +func NewProfile(tiers []string) Profile { + p := Profile{Allowed: map[Tier]bool{}} + for _, t := range tiers { + p.Allowed[Tier(t)] = true + } + return p +} + +// Allows says whether the profile takes a tier. +func (p Profile) Allows(t Tier) bool { return p.Allowed[t] } diff --git a/internal/release/release_test.go b/internal/release/release_test.go new file mode 100644 index 00000000..a5c07015 --- /dev/null +++ b/internal/release/release_test.go @@ -0,0 +1,251 @@ +package release + +import ( + "crypto/sha1" + "encoding/hex" + "fmt" + "strings" + "testing" +) + +func TestParse_ReadsWhatTheNameSays(t *testing.T) { + for _, c := range []struct { + name string + format, vbr, src string + bitrate, depth, yr int + rate float64 + tier Tier + }{ + // The wrong edition Lidarr grabbed on 2026-10-09 (note #5527). + {"Daft Punk Random Access Memories (Drumless Edition 2023) [Flac 24-88]", FormatFLAC, "", "", 0, 24, 2023, 88.2, TierLosslessHiRes}, + {"Gorillaz - Humanz - (Japanese Edition) - [CD-FLAC] 2017", FormatFLAC, "", SourceCD, 0, 0, 2017, 0, TierLossless}, + {"Radiohead - OK Computer (1997) [MP3 320]", FormatMP3, "", "", 320, 0, 1997, 0, TierLossyHigh}, + {"Artist - Album 2019 MP3 V0", FormatMP3, "V0", "", 0, 0, 2019, 0, TierLossyHigh}, + {"Artist - Album (2020) [WEB] [AAC 256]", FormatAAC, "", SourceWEB, 256, 0, 2020, 0, TierLossyHigh}, + {"Artist_-_Album-(2004)-192kbps", FormatMP3, "", "", 192, 0, 2004, 0, TierLossyLow}, + {"Artist - Album [24bit-96kHz] FLAC Vinyl", FormatFLAC, "", SourceVinyl, 0, 24, 0, 96, TierLosslessHiRes}, + {"Taylor Swift - 1989 (2014) [FLAC]", FormatFLAC, "", "", 0, 0, 2014, 0, TierLossless}, + {"Artist - Album", "", "", "", 0, 0, 0, 0, TierUnknown}, + } { + r := Parse(c.name) + got := fmt.Sprint(r.Format, r.VBR, r.Source, r.Bitrate, r.BitDepth, r.Year, r.SampleRate, TierOf(r, 0)) + want := fmt.Sprint(c.format, c.vbr, c.src, c.bitrate, c.depth, c.yr, c.rate, c.tier) + if got != want { + t.Errorf("Parse(%q)\n got %s\nwant %s", c.name, got, want) + } + } +} + +func TestParse_VideoAndDiscographies(t *testing.T) { + if v := Parse("Artist.Album.2018.1080p.BluRay.x264").Video; v != "1080p" { + t.Errorf("video = %q", v) + } + if d := Parse("Artist Discography 1994-2010 FLAC").Discography; d != "discography" { + t.Errorf("discography = %q", d) + } + if d := Parse("Artist - Studio Albums 1994 - 2010 [MP3]").Discography; d != "1994–2010" { + t.Errorf("year range = %q", d) + } + if d := Parse("Artist - Album (2010) [FLAC]").Discography; d != "" { + t.Errorf("one year read as a range: %q", d) + } +} + +func judge(w Want, name string) Match { return Judge(w, Parse(name)) } + +func TestJudge_TheAlbumAndWhatIsNot(t *testing.T) { + ram := Want{Artist: "Daft Punk", Title: "Random Access Memories", Year: 2013, PrimaryType: "Album"} + for name, veto := range map[string]string{ + "Daft Punk - Random Access Memories (2013) [FLAC]": "", + "Daft_Punk-Random_Access_Memories-WEB-2013": "", + "Daft Punk Random Access Memories (Drumless Edition 2023) [Flac 24-88]": VetoVersion, + "Daft Punk - Random Access Memories (Instrumentals) [MP3]": VetoVersion, + "Daft Punk - Random Access Memories Remixes": VetoVersion, + "Daft Punk - Random Access Memories (Live) FLAC": VetoVersion, + "Daft Punk - Discovery (2001) [FLAC]": VetoTitle, + "Random Access Memories (2013) [FLAC]": VetoArtist, + "Daft Punk - Discography 1997-2013 FLAC": VetoTitle, + "Daft Punk - Random Access Memories + Discovery + Homework Collection": VetoDiscography, + "Daft Punk - Random Access Memories (2013) 1080p BluRay": VetoVideo, + "Daft Punk - Random Access Memories (Single) 2013": VetoType, + "Daft Punk - Random Access Memories (Deluxe 10th Anniversary Edition) 2023": "", + } { + if got := judge(ram, name).Veto; got != veto { + t.Errorf("%q: veto %q, want %q", name, got, veto) + } + } + if !judge(ram, "Daft Punk - Random Access Memories (2013) [FLAC]").YearAgrees { + t.Error("the year did not agree") + } +} + +func TestJudge_WordsTheWantedTitleHoldsDoNotVeto(t *testing.T) { + hole := Want{Artist: "Hole", Title: "Live Through This", PrimaryType: "Album"} + if m := judge(hole, "Hole - Live Through This (1994) [FLAC]"); m.Veto != "" { + t.Errorf("Live Through This: %s", m.Reason) + } + alive := Want{Artist: "Daft Punk", Title: "Alive 2007", PrimaryType: "Album", SecondaryTypes: []string{"Live"}} + if m := judge(alive, "Daft Punk - Alive 2007 (Live) [FLAC]"); m.Veto != "" { + t.Errorf("a wanted live album: %s", m.Reason) + } + alive.SecondaryTypes = nil + if m := judge(alive, "Daft Punk - Alive 2007 (Live) [FLAC]"); m.Veto != VetoVersion { + t.Errorf("live without the Live type: veto %q", m.Veto) + } +} + +func TestJudge_SpellingsOfNames(t *testing.T) { + for _, c := range []struct { + w Want + name string + }{ + {Want{Artist: "AC/DC", Title: "Back in Black"}, "ACDC - Back In Black (1980) FLAC"}, + {Want{Artist: "AC/DC", Title: "Back in Black"}, "AC-DC - Back in Black [MP3 320]"}, + {Want{Artist: "Sigur Rós", Title: "Ágætis byrjun"}, "Sigur Ros - Agaetis Byrjun (1999) [FLAC]"}, + {Want{Artist: "Simon & Garfunkel", Title: "Bookends"}, "Simon and Garfunkel - Bookends"}, + {Want{Artist: "The Beatles", Title: "Abbey Road"}, "Beatles - Abbey Road (Remastered 2009)"}, + {Want{Artist: "Florence + the Machine", Title: "Lungs"}, "Florence and The Machine - Lungs (2009)"}, + {Want{Artist: "Guns N' Roses", Title: "Appetite for Destruction"}, "Guns N Roses - Appetite For Destruction"}, + } { + if m := judge(c.w, c.name); m.Veto != "" { + t.Errorf("%q for %s / %s: %s", c.name, c.w.Artist, c.w.Title, m.Reason) + } + } + // One word is never matched run together: Muse is not inside Museum. + if m := judge(Want{Artist: "Muse", Title: "Absolution"}, "Museum - Absolution"); m.Veto != VetoArtist { + t.Errorf("Museum read as Muse: %q", m.Veto) + } +} + +func TestJudge_ASelfTitledAlbumNeedsMoreThanTheName(t *testing.T) { + weezer := Want{Artist: "Weezer", Title: "Weezer", Year: 1994, PrimaryType: "Album"} + if m := judge(weezer, "Weezer - Weezer (Blue Album) [FLAC]"); m.Doubt != "" || m.Veto != "" { + t.Errorf("named twice: %+v", m) + } + if m := judge(weezer, "Weezer (1994) [FLAC]"); m.Doubt != "" { + t.Errorf("the year agrees: %+v", m) + } + if m := judge(weezer, "Weezer (Green Album) FLAC"); m.Doubt == "" { + t.Error("a self-titled name with nothing else was taken as sure") + } +} + +func TestTierOf_TheLadder(t *testing.T) { + for _, c := range []struct { + r Release + cat int + want Tier + }{ + {Release{Format: FormatMP3, Bitrate: 256}, 0, TierLossyStandard}, + {Release{Format: FormatMP3, Bitrate: 128}, 0, TierLossyPoor}, + {Release{Format: FormatMP3, Bitrate: 96}, 0, TierLossyTrash}, + {Release{Format: FormatMP3, VBR: "V2"}, 0, TierLossyStandard}, + {Release{Format: FormatMP3}, 0, TierLossyStandard}, + {Release{Format: FormatAAC, Bitrate: 192}, 0, TierLossyStandard}, + {Release{Format: FormatALAC, SampleRate: 96}, 0, TierLosslessHiRes}, + {Release{Format: FormatDSD}, 0, TierLosslessHiRes}, + {Release{}, 3040, TierLossless}, + {Release{}, 3010, TierUnknown}, + } { + if got := TierOf(c.r, c.cat); got != c.want { + t.Errorf("TierOf(%+v, %d) = %s, want %s", c.r, c.cat, got, c.want) + } + } + if Rank(TierLossless) <= Rank(TierLossyHigh) || Rank(TierUnknown) >= Rank(TierLossyTrash) { + t.Error("the ladder is out of order") + } + if !NewProfile([]string{"lossless"}).Allows(TierLossless) || NewProfile([]string{"lossless"}).Allows(TierUnknown) { + t.Error("profile") + } +} + +func files(paths ...string) []File { + out := make([]File, len(paths)) + for i, p := range paths { + out[i] = File{Index: i, Path: p, Size: 1 << 20} + } + return out +} + +func numbered(n int, ext string) []string { + var out []string + for i := 1; i <= n; i++ { + out = append(out, fmt.Sprintf("Album/%02d - Track.%s", i, ext)) + } + return out +} + +func TestJudgeFiles_WhatTheTorrentHolds(t *testing.T) { + w := Want{MinTracks: 10, MaxTracks: 13} + if veto, _, _ := JudgeFiles(w, files(numbered(13, "flac")...)); veto != "" { + t.Errorf("a whole album: %s", veto) + } + if veto, reason, _ := JudgeFiles(w, files(numbered(3, "flac")...)); veto != VetoIncomplete || !strings.Contains(reason, "3 tracks") { + t.Errorf("three tracks: %s %s", veto, reason) + } + if veto, _, _ := JudgeFiles(w, files("Concert/Concert.mkv", "Concert/cover.jpg")); veto != VetoVideo { + t.Errorf("a video: %s", veto) + } + if veto, _, _ := JudgeFiles(w, files("Album.rar", "Album.r00")); veto != VetoNoAudio { + t.Errorf("an archive: %s", veto) + } + if _, _, doubt := JudgeFiles(w, files(numbered(60, "mp3")...)); doubt == "" { + t.Error("sixty tracks for a thirteen-track album raised no doubt") + } + if c := Inspect(files(append(numbered(12, "flac"), "Album/cover.jpg", "Album/album.cue")...)); c.Format != FormatFLAC || c.Audio != 12 { + t.Errorf("inspect = %+v", c) + } +} + +// bencode writes the few shapes a test torrent needs. +func bencode(v any) string { + switch x := v.(type) { + case string: + return fmt.Sprintf("%d:%s", len(x), x) + case int: + return fmt.Sprintf("i%de", x) + case []any: + var b strings.Builder + b.WriteString("l") + for _, e := range x { + b.WriteString(bencode(e)) + } + return b.String() + "e" + case [][2]any: // an ordered dictionary + var b strings.Builder + b.WriteString("d") + for _, kv := range x { + b.WriteString(bencode(kv[0]) + bencode(kv[1])) + } + return b.String() + "e" + } + panic(fmt.Sprintf("bencode %T", v)) +} + +func TestParseTorrent_FilesAndInfohash(t *testing.T) { + info := bencode([][2]any{ + {"files", []any{ + [][2]any{{"length", 300}, {"path", []any{"01 - One.flac"}}}, + [][2]any{{"attr", "p"}, {"length", 12}, {"path", []any{".pad", "12"}}}, + [][2]any{{"length", 200}, {"path", []any{"Scans", "front.jpg"}}}, + }}, + {"name", "Artist - Album"}, + {"piece length", 16384}, + {"pieces", strings.Repeat("x", 20)}, + }) + body := "d" + bencode("announce") + bencode("http://tracker/") + bencode("info") + info + "e" + meta, err := ParseTorrent([]byte(body)) + if err != nil { + t.Fatal(err) + } + sum := sha1.Sum([]byte(info)) + if meta.InfoHash != hex.EncodeToString(sum[:]) { + t.Errorf("infohash = %s", meta.InfoHash) + } + if len(meta.Files) != 2 || meta.Files[0].Path != "Artist - Album/01 - One.flac" || meta.Files[1].Path != "Artist - Album/Scans/front.jpg" { + t.Errorf("files = %+v", meta.Files) + } + if _, err := ParseTorrent([]byte("not a torrent")); err == nil { + t.Error("HTML parsed as a torrent") + } +} diff --git a/internal/release/torrent.go b/internal/release/torrent.go new file mode 100644 index 00000000..578528de --- /dev/null +++ b/internal/release/torrent.go @@ -0,0 +1,178 @@ +package release + +import ( + "bytes" + "crypto/sha1" + "encoding/hex" + "errors" + "fmt" + "strconv" + "strings" +) + +// Reading a .torrent file, ported from Fabled Librarian (#4463). A .torrent +// is the metadata: its info dictionary names every file, and its SHA-1 is the +// infohash. So a release offered as a .torrent link can be judged on what it +// holds before any of it is downloaded. Bencode is four shapes; walking them +// here costs less than a dependency. + +// TorrentMeta is what a .torrent says about itself. +type TorrentMeta struct { + // InfoHash is the v1 infohash: the SHA-1 of the info dictionary's own + // bytes, lowercase hex. + InfoHash string + Name string + Files []File +} + +// ErrNotTorrent is bytes that are not a .torrent. +var ErrNotTorrent = errors.New("not a .torrent file") + +// maxTorrentBytes bounds what is read, as Exchange bounds what it accepts. +const maxTorrentBytes = 10 << 20 + +// ParseTorrent reads a .torrent's info dictionary. +func ParseTorrent(b []byte) (TorrentMeta, error) { + if len(b) == 0 || len(b) > maxTorrentBytes || b[0] != 'd' { + return TorrentMeta{}, ErrNotTorrent + } + d := &bdecoder{b: b} + root, err := d.value(0) + if err != nil { + return TorrentMeta{}, fmt.Errorf("%w: %v", ErrNotTorrent, err) + } + top, ok := root.(map[string]any) + if !ok || d.infoEnd == 0 { + return TorrentMeta{}, fmt.Errorf("%w: no info dictionary", ErrNotTorrent) + } + info, ok := top["info"].(map[string]any) + if !ok { + return TorrentMeta{}, fmt.Errorf("%w: no info dictionary", ErrNotTorrent) + } + sum := sha1.Sum(b[d.infoStart:d.infoEnd]) + out := TorrentMeta{InfoHash: hex.EncodeToString(sum[:])} + out.Name, _ = info["name"].(string) + + files, multi := info["files"].([]any) + if !multi { + // A single-file torrent: the name IS the file. + length, _ := info["length"].(int64) + out.Files = []File{{Path: out.Name, Size: length}} + return out, nil + } + for index, f := range files { + entry, ok := f.(map[string]any) + if !ok { + continue + } + parts, _ := entry["path"].([]any) + segs := []string{out.Name} + for _, p := range parts { + if s, ok := p.(string); ok { + segs = append(segs, s) + } + } + length, _ := entry["length"].(int64) + // BEP 47 padding files carry no content. + if attr, _ := entry["attr"].(string); strings.Contains(attr, "p") { + continue + } + out.Files = append(out.Files, File{Index: index, Path: strings.Join(segs, "/"), Size: length}) + } + return out, nil +} + +// bdecoder walks bencode, and remembers where the top-level info dictionary +// sits so its raw bytes can be hashed exactly as published. +type bdecoder struct { + b []byte + i int + infoStart, infoEnd int +} + +// maxDepth: a real torrent nests four deep. Anything past this is hostile. +const maxDepth = 32 + +func (d *bdecoder) value(depth int) (any, error) { + if depth > maxDepth { + return nil, errors.New("nested too deep") + } + if d.i >= len(d.b) { + return nil, errors.New("truncated") + } + switch c := d.b[d.i]; { + case c == 'i': + end := bytes.IndexByte(d.b[d.i:], 'e') + if end < 0 { + return nil, errors.New("unterminated integer") + } + n, err := strconv.ParseInt(string(d.b[d.i+1:d.i+end]), 10, 64) + if err != nil { + return nil, err + } + d.i += end + 1 + return n, nil + case c == 'l': + d.i++ + var out []any + for d.i < len(d.b) && d.b[d.i] != 'e' { + v, err := d.value(depth + 1) + if err != nil { + return nil, err + } + out = append(out, v) + } + if d.i >= len(d.b) { + return nil, errors.New("unterminated list") + } + d.i++ + return out, nil + case c == 'd': + d.i++ + out := map[string]any{} + for d.i < len(d.b) && d.b[d.i] != 'e' { + k, err := d.str() + if err != nil { + return nil, err + } + start := d.i + v, err := d.value(depth + 1) + if err != nil { + return nil, err + } + if depth == 0 && k == "info" { + d.infoStart, d.infoEnd = start, d.i + } + // Pieces are 20 bytes a piece and never read: not kept. + if k != "pieces" { + out[k] = v + } + } + if d.i >= len(d.b) { + return nil, errors.New("unterminated dictionary") + } + d.i++ + return out, nil + case c >= '0' && c <= '9': + return d.str() + default: + return nil, fmt.Errorf("unexpected %q", c) + } +} + +func (d *bdecoder) str() (string, error) { + colon := bytes.IndexByte(d.b[d.i:], ':') + if colon < 0 { + return "", errors.New("unterminated string length") + } + n, err := strconv.Atoi(string(d.b[d.i : d.i+colon])) + if err != nil || n < 0 { + return "", errors.New("bad string length") + } + start := d.i + colon + 1 + if start+n > len(d.b) { + return "", errors.New("string runs past the end") + } + d.i = start + n + return string(d.b[start : start+n]), nil +} diff --git a/internal/wanted/search.go b/internal/wanted/search.go new file mode 100644 index 00000000..3b5efb2e --- /dev/null +++ b/internal/wanted/search.go @@ -0,0 +1,322 @@ +package wanted + +import ( + "context" + "encoding/json" + "errors" + "log/slog" + "strconv" + "time" + + "github.com/jackc/pgx/v5/pgtype" + "github.com/jackc/pgx/v5/pgxpool" + + "git.fabledsword.com/bvandeusen/minstrel/internal/acquire" + "git.fabledsword.com/bvandeusen/minstrel/internal/catalogue" + "git.fabledsword.com/bvandeusen/minstrel/internal/db/dbq" + "git.fabledsword.com/bvandeusen/minstrel/internal/indexer" + "git.fabledsword.com/bvandeusen/minstrel/internal/release" +) + +// Searching for wants on a schedule (M502 #5521), ported from Fabled +// Librarian's internal/wanted. The schedule decides which wants and when; +// internal/acquire judges what a search finds and grabs. + +// Search outcomes, matching wants_search_outcome_check. +const ( + OutcomeGrabbed = "grabbed" + OutcomeWaiting = "waiting" + OutcomeNothing = "nothing" + OutcomeError = "error" +) + +const ( + // searchTick is how often the schedule looks for wants due; the + // operator's setting is how long a want waits between searches. + searchTick = 10 * time.Minute + // searchWindow is how long one pass searches: most of a tick, so a pass + // ends before the next begins. + searchWindow = 9 * time.Minute + // duePage bounds what one pass reads. The sources' pace is the real + // limit; this caps a pass whose sources are all unpaced. + duePage = 200 + // grabTries is how many grabbable releases one search tries in turn + // when the first is refused for its files. + grabTries = 3 + // passTimeout backs up the window for a call none of the clients' + // own timeouts cover, such as a wedged database read. + passTimeout = 15 * time.Minute +) + +// ErrWantHeld: the want is already answered by the library. +var ErrWantHeld = errors.New("wanted: the library already holds this album") + +// Searcher searches for wants and grabs what they need. +type Searcher struct { + pool *pgxpool.Pool + cat *catalogue.Service + logger *slog.Logger +} + +// NewSearcher returns a Searcher reading editions through cat (nil reads +// only what the catalogue already holds). +func NewSearcher(pool *pgxpool.Pool, cat *catalogue.Service, logger *slog.Logger) *Searcher { + if logger == nil { + logger = slog.Default() + } + return &Searcher{pool: pool, cat: cat, logger: logger} +} + +// Report is one pass. +type Report struct { + Looked int `json:"looked"` + Grabbed int `json:"grabbed"` + Waiting int `json:"waiting"` + Nothing int `json:"nothing"` + Failed int `json:"failed"` + // Paced: the pass stopped because the next search would wait past its + // window for a source's pace. + Paced bool `json:"paced"` + // Off: the schedule is switched off. Unconfigured: nothing to search + // with. Both are states, not faults. + Off bool `json:"off"` + Unconfigured bool `json:"unconfigured"` + AutoGrab bool `json:"auto_grab"` + Error string `json:"error,omitempty"` +} + +// Run searches every searchTick until ctx ends. +func (s *Searcher) Run(ctx context.Context) { + ticker := time.NewTicker(searchTick) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + } + rep := s.Pass(ctx) + if rep.Looked > 0 || rep.Error != "" { + s.logger.Info("wanted: search pass", "looked", rep.Looked, "grabbed", rep.Grabbed, + "waiting", rep.Waiting, "nothing", rep.Nothing, "failed", rep.Failed, + "paced", rep.Paced, "error", rep.Error) + } + } +} + +// Pass searches for the wants due, paced by the sources, until the window +// closes, and records what it did. +func (s *Searcher) Pass(ctx context.Context) Report { + rep := s.pass(ctx) + if body, err := json.Marshal(rep); err == nil { + if err := dbq.New(s.pool).RecordSearchPass(ctx, body); err != nil && ctx.Err() == nil { + s.logger.Warn("wanted: record search pass", "err", err) + } + } + return rep +} + +func (s *Searcher) pass(ctx context.Context) Report { + var rep Report + ctx, cancel := context.WithTimeout(ctx, passTimeout) + defer cancel() + svc, settings, err := acquire.Build(ctx, s.pool) + if err != nil { + rep.Error = indexer.FirstLine(err.Error()) + return rep + } + rep.AutoGrab = settings.AutoGrab + switch { + case settings.SearchIntervalHours <= 0: + rep.Off = true + return rep + case svc.Search == nil: + rep.Unconfigured = true + return rep + } + deadline := time.Now().Add(searchWindow) + due, err := dbq.New(s.pool).ListWantsDueForSearch(ctx, dbq.ListWantsDueForSearchParams{ + Hours: settings.SearchIntervalHours, MaxRows: duePage, + }) + if err != nil { + rep.Error = indexer.FirstLine(err.Error()) + return rep + } + for _, row := range due { + if !svc.Ready(ctx, deadline) { + rep.Paced = ctx.Err() == nil + return rep + } + want := s.wantFor(ctx, row.ID, row.ArtistName, row.Title, row.PrimaryType, row.SecondaryTypes, + row.FirstReleaseDate, row.ReleaseGroupMbid) + out := s.one(ctx, svc, want, settings.AutoGrab) + rep.Looked++ + switch out.Outcome { + case OutcomeGrabbed: + rep.Grabbed++ + case OutcomeWaiting: + rep.Waiting++ + case OutcomeNothing: + rep.Nothing++ + default: + rep.Failed++ + } + } + return rep +} + +// wantFor describes a want for the matcher, with its editions' track counts +// from the catalogue (read from MusicBrainz the first time when it can be). +func (s *Searcher) wantFor(ctx context.Context, id pgtype.UUID, artist, title, primary string, secondary []string, date, group string) acquire.Want { + w := acquire.Want{ID: id, Want: release.Want{ + Artist: artist, Title: title, Year: yearOf(date), PrimaryType: primary, SecondaryTypes: secondary, + }} + q := dbq.New(s.pool) + span, err := q.EditionTrackSpan(ctx, group) + if err == nil && span.MinTracks == 0 && s.cat != nil { + if _, rerr := s.cat.Releases(ctx, group); rerr == nil { + span, err = q.EditionTrackSpan(ctx, group) + } + } + if err == nil { + w.MinTracks, w.MaxTracks = int(span.MinTracks), int(span.MaxTracks) + } + return w +} + +func yearOf(date string) int { + if len(date) < 4 { + return 0 + } + y, _ := strconv.Atoi(date[:4]) + return y +} + +// Searched is what one want's search came to. +type Searched struct { + Outcome string `json:"outcome"` + Considered int `json:"considered"` + Best string `json:"best_release"` + Verdict string `json:"best_verdict"` + Grabbed string `json:"grabbed,omitempty"` + Narrowed []acquire.Narrowing `json:"narrowed"` + Error string `json:"error,omitempty"` +} + +// one searches for a want, takes the best release when auto-grab is on, and +// records what came of it on the want. +func (s *Searcher) one(ctx context.Context, svc *acquire.Service, want acquire.Want, autoGrab bool) Searched { + found := svc.For(ctx, want) + if found.KeepError != "" { + s.logger.Warn("wanted: searched, but could not keep the candidates", "want", want.Title, "err", found.KeepError) + } + out := Searched{Considered: found.Considered, Narrowed: found.Narrowed} + if out.Narrowed == nil { + out.Narrowed = []acquire.Narrowing{} + } + if len(found.Candidates) > 0 { + out.Best, out.Verdict = found.Candidates[0].Name, string(found.Candidates[0].Verdict) + } + switch { + case found.Unconfigured: + out.Outcome, out.Error = OutcomeError, "No indexer is set up, so there is nothing to search with." + case found.Error != "": + out.Outcome, out.Error = OutcomeError, found.Error + case !found.ForAPerson(): + out.Outcome = OutcomeNothing + case len(found.Grabbable()) == 0 || !autoGrab: + out.Outcome = OutcomeWaiting + default: + out.Outcome = OutcomeWaiting + s.grabBest(ctx, svc, want, found, &out) + } + s.record(ctx, want.ID, out) + return out +} + +// grabBest takes the best grabbable release, trying the next when one is +// refused for what its files hold. +func (s *Searcher) grabBest(ctx context.Context, svc *acquire.Service, want acquire.Want, found acquire.Result, out *Searched) { + for i, c := range found.Grabbable() { + if i == grabTries { + return + } + took := svc.Grab(ctx, want, &c, found.Candidates, false, acquire.Origin{By: acquire.OriginSchedule}) + switch { + case took.Added: + out.Outcome, out.Grabbed, out.Best, out.Verdict = OutcomeGrabbed, c.Name, c.Name, string(acquire.Grab) + if took.Error != "" { + s.logger.Warn("wanted: took a release but could not record it", "want", want.Title, "err", took.Error) + } + s.logger.Info("wanted: took a release", "want", want.Title, "artist", want.Artist, + "release", c.Name, "quality", c.Tier, "save_path", took.SavePath) + return + case took.Refused: + continue + case took.Unconfigured: + out.Outcome, out.Error = OutcomeError, "No download client is set up, so nothing was taken." + return + case took.Contest != nil: + return + case took.Error != "": + out.Outcome, out.Error = OutcomeError, took.Error + return + } + } +} + +func (s *Searcher) record(ctx context.Context, id pgtype.UUID, out Searched) { + if err := dbq.New(s.pool).RecordWantSearch(ctx, dbq.RecordWantSearchParams{ + ID: id, Outcome: out.Outcome, Considered: int32(out.Considered), + BestRelease: out.Best, BestVerdict: out.Verdict, SearchError: out.Error, + }); err != nil && ctx.Err() == nil { + s.logger.Warn("wanted: record search", "err", err) + } +} + +// SearchNow searches for one want at once, as the schedule would, for a +// person pressing Search: it spends the sources' pace but never waits on it. +func (s *Searcher) SearchNow(ctx context.Context, id pgtype.UUID) (Searched, error) { + want, err := s.load(ctx, id) + if err != nil { + return Searched{}, err + } + svc, settings, err := acquire.Build(ctx, s.pool) + if err != nil { + return Searched{}, err + } + return s.one(ctx, svc, want, settings.AutoGrab), nil +} + +// Take grabs the release named by key for a person, searching again first. +func (s *Searcher) Take(ctx context.Context, id pgtype.UUID, key string, force bool, by pgtype.UUID) (acquire.Taken, error) { + want, err := s.load(ctx, id) + if err != nil { + return acquire.Taken{}, err + } + svc, _, err := acquire.Build(ctx, s.pool) + if err != nil { + return acquire.Taken{}, err + } + took := svc.Take(ctx, want, key, force, acquire.Origin{By: acquire.OriginOperator, User: by}) + if took.Added && took.Chosen != nil { + s.record(ctx, id, Searched{ + Outcome: OutcomeGrabbed, Best: took.Chosen.Name, Verdict: string(took.Chosen.Verdict), + Considered: 0, Grabbed: took.Chosen.Name, + }) + } + return took, nil +} + +// load reads an open want for a search. +func (s *Searcher) load(ctx context.Context, id pgtype.UUID) (acquire.Want, error) { + row, err := dbq.New(s.pool).GetWantForSearch(ctx, id) + if err != nil { + return acquire.Want{}, err + } + if row.FulfilledAt.Valid { + return acquire.Want{}, ErrWantHeld + } + return s.wantFor(ctx, row.ID, row.ArtistName, row.Title, row.PrimaryType, row.SecondaryTypes, + row.FirstReleaseDate, row.ReleaseGroupMbid), nil +} diff --git a/internal/wanted/search_test.go b/internal/wanted/search_test.go new file mode 100644 index 00000000..1e96274f --- /dev/null +++ b/internal/wanted/search_test.go @@ -0,0 +1,195 @@ +package wanted + +import ( + "context" + "encoding/json" + "io" + "log/slog" + "net/http" + "net/http/httptest" + "strings" + "sync/atomic" + "testing" + + "git.fabledsword.com/bvandeusen/minstrel/internal/acquire" + "git.fabledsword.com/bvandeusen/minstrel/internal/db/dbq" +) + +const ramHash = "0123456789abcdef0123456789abcdef01234567" + +const ramFeed = ` + +Daft Punk - Random Access Memories (2013) [FLAC] + magnet:?xt=urn:btih:` + ramHash + `400000000 + +Daft Punk Random Access Memories (Drumless Edition 2023) [Flac 24-88] + magnet:?xt=urn:btih:fedcba9876543210fedcba9876543210fedcba98900000000 + +` + +// searchStack is a fake indexer feed and a fake FabledExchange download +// client, configured the way the operator would in Integrations. +func searchStack(t *testing.T, ctx context.Context, store *acquire.Store) *atomic.Int32 { + t.Helper() + var added atomic.Int32 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch { + case r.URL.Path == "/feed": + if q := r.URL.Query().Get("q"); q != "Daft Punk Random Access Memories" { + t.Errorf("searched for %q", q) + } + _, _ = io.WriteString(w, ramFeed) + case r.URL.Path == "/api/downloads" && r.Method == http.MethodPost: + var body struct { + Magnet string `json:"magnet"` + SavePath string `json:"save_path"` + } + _ = json.NewDecoder(r.Body).Decode(&body) + if !strings.Contains(body.Magnet, ramHash) || body.SavePath != "/downloads/"+ramHash { + t.Errorf("added %+v", body) + } + added.Add(1) + _, _ = io.WriteString(w, "{}") + default: + http.NotFound(w, r) + } + })) + t.Cleanup(srv.Close) + + feed, err := store.Create(ctx, acquire.ProtocolTorznab) + if err != nil { + t.Fatal(err) + } + if _, err := store.Save(ctx, feed.Code, acquire.Update{ + Label: "Feed", Enabled: true, BaseURL: srv.URL + "/feed", TimeoutMs: 5000, LocalPath: "/downloads", + }); err != nil { + t.Fatal(err) + } + ex, err := store.Create(ctx, acquire.ProtocolExchange) + if err != nil { + t.Fatal(err) + } + if _, err := store.Save(ctx, ex.Code, acquire.Update{ + Label: "Exchange", Enabled: true, BaseURL: srv.URL, TimeoutMs: 5000, ServesDownloads: true, LocalPath: "/downloads", + }); err != nil { + t.Fatal(err) + } + return &added +} + +func TestSearcher_FindsAndTakesAWant_Integration(t *testing.T) { + ctx := context.Background() + pool := newPool(t) + q := dbq.New(pool) + added := searchStack(t, ctx, acquire.NewStore(pool)) + w, err := WantWith(ctx, q, WantParams{ + GroupMBID: "rg-ram", ArtistMBID: "mb-daft-punk", ArtistName: "Daft Punk", Title: "Random Access Memories", + PrimaryType: "Album", FirstReleaseDate: "2013-05-17", Reason: ReasonRequest, + }) + if err != nil { + t.Fatal(err) + } + s := NewSearcher(pool, nil, slog.New(slog.NewTextHandler(io.Discard, nil))) + + rep := s.Pass(ctx) + if rep.Looked != 1 || rep.Grabbed != 1 || !rep.AutoGrab || added.Load() != 1 { + t.Fatalf("pass = %+v, added %d", rep, added.Load()) + } + got, err := q.GetWantByReleaseGroup(ctx, "rg-ram") + if err != nil { + t.Fatal(err) + } + if got.SearchOutcome != OutcomeGrabbed || got.SearchConsidered != 2 || !strings.Contains(got.BestRelease, "(2013) [FLAC]") { + t.Errorf("want after the pass: %s %d %q", got.SearchOutcome, got.SearchConsidered, got.BestRelease) + } + cands, err := q.ListWantCandidates(ctx, w.ID) + if err != nil { + t.Fatal(err) + } + if len(cands) != 2 || cands[0].Verdict != "grab" || cands[1].Verdict != "rejected" || !strings.Contains(cands[1].Reason, "drumless") { + t.Errorf("candidates = %+v", cands) + } + g, err := q.GetLiveGrab(ctx, w.ID) + if err != nil { + t.Fatal(err) + } + if g.InfoHash != ramHash || g.Client != "exchange" || g.Origin != acquire.OriginSchedule || g.Quality != "lossless" { + t.Errorf("grab = %+v", g) + } + settings, _ := q.GetAcquisitionSettings(ctx) + if !settings.LastPassAt.Valid || !strings.Contains(string(settings.LastPass), `"grabbed":1`) { + t.Errorf("last pass = %s", settings.LastPass) + } + + // A want with a download running is not searched again. + if rep := s.Pass(ctx); rep.Looked != 0 { + t.Errorf("searched a want already downloading: %+v", rep) + } + + // Setting the drumless edition aside marks it on every kept candidate; + // allowing it again leaves it for a person until the next search. + mem := acquire.NewMemory(pool) + if err := mem.Block(ctx, dbq.BlockReleaseParams{ + InfoHash: cands[1].InfoHash, Name: cands[1].Name, Reason: acquire.BlockOperator, WantID: w.ID, + }); err != nil { + t.Fatal(err) + } + c, _ := q.GetWantCandidate(ctx, dbq.GetWantCandidateParams{WantID: w.ID, Key: cands[1].Key}) + if c.Verdict != "blocked" { + t.Errorf("blocked candidate = %s", c.Verdict) + } + if ok, err := mem.Unblock(ctx, cands[1].InfoHash); err != nil || !ok { + t.Fatalf("unblock: %v %v", ok, err) + } + c, _ = q.GetWantCandidate(ctx, dbq.GetWantCandidateParams{WantID: w.ID, Key: cands[1].Key}) + if c.Verdict != "review" { + t.Errorf("unblocked candidate = %s", c.Verdict) + } +} + +func TestSearcher_OffAndUnconfigured_Integration(t *testing.T) { + ctx := context.Background() + pool := newPool(t) + s := NewSearcher(pool, nil, slog.New(slog.NewTextHandler(io.Discard, nil))) + if rep := s.Pass(ctx); !rep.Unconfigured { + t.Errorf("no indexer: %+v", rep) + } + if _, err := dbq.New(pool).UpdateAcquisitionSettings(ctx, dbq.UpdateAcquisitionSettingsParams{ + AutoGrab: true, SearchIntervalHours: 0, AllowedQualities: []string{"lossless"}, + }); err != nil { + t.Fatal(err) + } + if rep := s.Pass(ctx); !rep.Off { + t.Errorf("interval 0: %+v", rep) + } +} + +func TestGrabs_OneLivePerWant_Integration(t *testing.T) { + ctx := context.Background() + pool := newPool(t) + q := dbq.New(pool) + w, err := WantWith(ctx, q, WantParams{GroupMBID: "rg-x", ArtistMBID: "mb-x", ArtistName: "X", Title: "Y", Reason: ReasonFollow}) + if err != nil { + t.Fatal(err) + } + mem := acquire.NewMemory(pool) + grab := func(hash string) dbq.InsertGrabParams { + return dbq.InsertGrabParams{WantID: w.ID, InfoHash: hash, Name: hash, Key: hash, Client: "qbittorrent", + SavePath: "/downloads/" + hash, Origin: acquire.OriginSchedule} + } + first, err := mem.Grabbed(ctx, grab("aaaa"), dbq.Grab{}.ID, "") + if err != nil { + t.Fatal(err) + } + if _, err := mem.Grabbed(ctx, grab("bbbb"), dbq.Grab{}.ID, ""); err == nil { + t.Error("a second live grab for one want was recorded") + } + second, err := mem.Grabbed(ctx, grab("bbbb"), first.ID, "Replaced by bbbb.") + if err != nil { + t.Fatalf("replacing: %v", err) + } + live, ok, err := mem.LiveGrab(ctx, w.ID) + if err != nil || !ok || live.ID != second.ID { + t.Errorf("live = %+v %v %v", live, ok, err) + } +} diff --git a/web/src/lib/api/queries.ts b/web/src/lib/api/queries.ts index f8c99725..8abd41d4 100644 --- a/web/src/lib/api/queries.ts +++ b/web/src/lib/api/queries.ts @@ -45,6 +45,9 @@ export const qk = { adminFollows: () => ['adminFollows'] as const, adminFollow: (artistMbid: string) => ['adminFollow', artistMbid] as const, releaseProfiles: () => ['releaseProfiles'] as const, + // Search and grab (M502 #5521). + wantCandidates: (wantId: string) => ['wantCandidates', wantId] as const, + searchSettings: () => ['searchSettings'] as const, myQuarantine: () => ['myQuarantine'] as const, adminQuarantine: () => ['adminQuarantine'] as const, adminQuarantineActions: (limit?: number) => diff --git a/web/src/lib/api/types.ts b/web/src/lib/api/types.ts index 995f350c..783a5985 100644 --- a/web/src/lib/api/types.ts +++ b/web/src/lib/api/types.ts @@ -590,6 +590,101 @@ export type Want = { wanted_at: string; held_at: string | null; image_url: string; + // The search schedule's last look, and the download running (#5521). + search?: WantSearch | null; + grab?: WantGrab | null; +}; + +export type SearchOutcome = 'grabbed' | 'waiting' | 'nothing' | 'error'; + +export type WantSearch = { + searched_at: string; + outcome: SearchOutcome; + considered: number; + best_release: string; + best_verdict: string; + error: string; +}; + +export type WantGrab = { + name: string; + grabbed_at: string; + origin: 'schedule' | 'operator'; + quality: string; +}; + +// Search and grab (M502 #5521) ---------------------------------------------- + +export type CandidateVerdict = 'grab' | 'review' | 'refused' | 'rejected' | 'blocked'; + +// One release a want's last search found, with what was decided about it. +export type Candidate = { + key: string; + name: string; + info_hash: string; + indexer: string; + size_bytes: number; + seeders: number; + files_count: number; + format: string; + quality: string; + year: number; + verdict: CandidateVerdict; + reason: string; + reasons: { kind: 'hard' | 'soft'; code: string; text: string }[]; + folded_from: string[]; + searched_at: string; + downloading: boolean; +}; + +export type Narrowing = { source: string; label: string; reason: string }; + +export type SearchResult = { + outcome: SearchOutcome; + considered: number; + best_release: string; + best_verdict: string; + grabbed?: string; + narrowed: Narrowing[]; + error?: string; +}; + +export type GrabResult = { + added: boolean; + refused: boolean; + gone: boolean; + unconfigured: boolean; + missing?: string; + kept: boolean; + replaced: boolean; + verdict?: CandidateVerdict; + reason?: string; + error?: string; +}; + +export type QualityTier = { tier: string; label: string; what: string }; + +// The last scheduled pass; empty before the first. +export type SearchPass = { + looked?: number; + grabbed?: number; + waiting?: number; + nothing?: number; + failed?: number; + paced?: boolean; + off?: boolean; + unconfigured?: boolean; + auto_grab?: boolean; + error?: string; +}; + +export type SearchSettings = { + auto_grab: boolean; + search_interval_hours: number; + allowed_qualities: string[]; + tiers: QualityTier[]; + last_pass_at: string | null; + last_pass: SearchPass; }; export type WantCounts = { wanted: number; missing: number; held: number; following: number }; diff --git a/web/src/lib/api/wanted.ts b/web/src/lib/api/wanted.ts index 39f95cec..4d201d45 100644 --- a/web/src/lib/api/wanted.ts +++ b/web/src/lib/api/wanted.ts @@ -2,11 +2,15 @@ import { createInfiniteQuery, createQuery } from '@tanstack/svelte-query'; import { api } from './client'; import { qk } from './queries'; import type { + Candidate, Follow, FollowState, ReleaseProfile, ReleaseProfileInput, ReleaseProfilesResponse, + GrabResult, + SearchResult, + SearchSettings, WantState, WantsPage } from './types'; @@ -100,3 +104,59 @@ export async function updateReleaseProfile(id: string, body: ReleaseProfileInput export async function deleteReleaseProfile(id: string): Promise { await api.del(`/api/admin/release-profiles/${id}`); } + +// Search and grab (M502 #5521): each want's last search, a person's Search, +// Grab and "never this one", and the schedule's settings. + +export async function listCandidates(wantId: string): Promise { + const res = await api.get<{ items: Candidate[] }>(`/api/admin/wanted/${wantId}/candidates`); + return res.items; +} + +export function createCandidatesQuery(wantId: string) { + return createQuery({ + queryKey: qk.wantCandidates(wantId), + queryFn: () => listCandidates(wantId), + staleTime: 30_000 + }); +} + +// Searches now, as the schedule would: takes the best release when +// auto-grab is on. Can take most of a minute. +export async function searchWant(wantId: string): Promise { + return api.post(`/api/admin/wanted/${wantId}/search`, {}); +} + +// Takes the release named by key; force takes what the verdict would not. +export async function grabCandidate(wantId: string, key: string, force: boolean): Promise { + return api.post(`/api/admin/wanted/${wantId}/grab`, { key, force }); +} + +// Never takes this release again, for any want. +export async function blockCandidate(wantId: string, key: string): Promise { + await api.post(`/api/admin/wanted/${wantId}/block`, { key }); +} + +export async function unblockRelease(infoHash: string): Promise { + await api.del(`/api/admin/release-blocks/${infoHash}`); +} + +export async function getSearchSettings(): Promise { + return api.get('/api/admin/acquisition/settings'); +} + +export function createSearchSettingsQuery() { + return createQuery({ + queryKey: qk.searchSettings(), + queryFn: getSearchSettings, + staleTime: 30_000 + }); +} + +export async function putSearchSettings(body: { + auto_grab: boolean; + search_interval_hours: number; + allowed_qualities: string[]; +}): Promise { + return api.put('/api/admin/acquisition/settings', body); +} diff --git a/web/src/lib/components/SearchSettingsCard.svelte b/web/src/lib/components/SearchSettingsCard.svelte new file mode 100644 index 00000000..fb97fdcf --- /dev/null +++ b/web/src/lib/components/SearchSettingsCard.svelte @@ -0,0 +1,162 @@ + + +
+
+

Searching and grabbing

+

+ Minstrel searches your indexers for each album on the + wanted list, judges every + release it finds, and hands the best one to your download client. Each album's releases and + the reason for every verdict are on the Wanted page. +

+
+ + {#if query.isPending} +

Loading…

+ {:else if query.isError || !form || !query.data} +

Couldn't load the search settings.

+ {:else} + + + + +
+ Qualities to take + + The best of these is taken first. A release that names no format has its file list read to + find out. + +
+ {#each query.data.tiers as t (t.tier)} + + {/each} +
+
+ +

{lastPass(query.data)}

+ +
+ +
+ {/if} +
diff --git a/web/src/lib/components/SearchSettingsCard.test.ts b/web/src/lib/components/SearchSettingsCard.test.ts new file mode 100644 index 00000000..c5b919be --- /dev/null +++ b/web/src/lib/components/SearchSettingsCard.test.ts @@ -0,0 +1,69 @@ +import { afterEach, describe, expect, test, vi } from 'vitest'; +import { fireEvent, render, screen, waitFor } from '@testing-library/svelte'; +import { mockQuery } from '../../test-utils/query'; +import type { SearchSettings } from '#lib/api/types.js'; + +vi.mock('#lib/api/wanted.js', () => ({ + createSearchSettingsQuery: vi.fn(), + putSearchSettings: vi.fn() +})); +vi.mock('#lib/stores/toast.svelte.js', () => ({ pushToast: vi.fn() })); + +import SearchSettingsCard from './SearchSettingsCard.svelte'; +import { createSearchSettingsQuery, putSearchSettings } from '#lib/api/wanted.js'; + +const settings: SearchSettings = { + auto_grab: true, + search_interval_hours: 24, + allowed_qualities: ['lossless', 'lossy_high'], + tiers: [ + { tier: 'lossless', label: 'Lossless', what: 'FLAC, ALAC' }, + { tier: 'lossy_high', label: 'High quality lossy', what: 'MP3 320 or V0' }, + { tier: 'unknown', label: 'Unknown', what: 'Nothing says what format it is' } + ], + last_pass_at: new Date(Date.now() - 5 * 60_000).toISOString(), + last_pass: { looked: 12, grabbed: 3, waiting: 2, nothing: 7, failed: 0, paced: true } +}; + +afterEach(() => vi.clearAllMocks()); + +function setup(s: SearchSettings = settings) { + vi.mocked(createSearchSettingsQuery).mockReturnValue( + mockQuery({ data: s }) as unknown as ReturnType + ); + vi.mocked(putSearchSettings).mockImplementation(async (body) => ({ ...s, ...body })); + return render(SearchSettingsCard); +} + +describe('SearchSettingsCard', () => { + test('shows the settings and what the last pass did', () => { + setup(); + expect((screen.getByLabelText(/take the best release without asking/i) as HTMLInputElement).checked).toBe(true); + expect((screen.getByLabelText(/hours between searches/i) as HTMLInputElement).value).toBe('24'); + expect((screen.getByLabelText(/^unknown/i) as HTMLInputElement).checked).toBe(false); + const last = screen.getByTestId('last-pass').textContent ?? ''; + expect(last).toMatch(/searched 12, took 3, 2 waiting for you, 7 with nothing found\./); + expect(last).toMatch(/searches per hour/); + }); + + test('saves what changed', async () => { + setup(); + const save = screen.getByRole('button', { name: /^save$/i }) as HTMLButtonElement; + expect(save.disabled).toBe(true); + await fireEvent.click(screen.getByLabelText(/take the best release without asking/i)); + await fireEvent.click(screen.getByLabelText(/^unknown/i)); + await fireEvent.click(save); + await waitFor(() => + expect(putSearchSettings).toHaveBeenCalledWith({ + auto_grab: false, + search_interval_hours: 24, + allowed_qualities: ['lossless', 'lossy_high', 'unknown'] + }) + ); + }); + + test('before the first pass it says so', () => { + setup({ ...settings, last_pass_at: null, last_pass: {} }); + expect(screen.getByTestId('last-pass').textContent).toMatch(/hasn’t run yet/); + }); +}); diff --git a/web/src/lib/components/WantCandidates.svelte b/web/src/lib/components/WantCandidates.svelte new file mode 100644 index 00000000..878eabc8 --- /dev/null +++ b/web/src/lib/components/WantCandidates.svelte @@ -0,0 +1,184 @@ + + +
+ {#if query.isPending} +

Loading what the last search found…

+ {:else if query.isError} +

Couldn't load the releases.

+ {:else if items.length === 0} +

+ Nothing kept from a search yet. Search now, or wait for the schedule. +

+ {:else} +

+ {items.length === 1 ? '1 release' : `${items.length} releases`} found {relativeTime(items[0].searched_at)}, best first. +

+
    + {#each items as c (c.key)} +
  • +
    +
    {c.name}
    +
    {meta(c)}
    +
    + + {c.downloading ? 'Downloading' : verdictLabel[c.verdict]} + + {c.reason} +
    +
    +
    + {#if c.verdict === 'blocked'} + + {:else if !c.downloading} + + {#if c.info_hash} + + {/if} + {/if} +
    +
    • + {/each} +
+ {/if} +
diff --git a/web/src/lib/components/WantCandidates.test.ts b/web/src/lib/components/WantCandidates.test.ts new file mode 100644 index 00000000..3c7fcecf --- /dev/null +++ b/web/src/lib/components/WantCandidates.test.ts @@ -0,0 +1,90 @@ +import { afterEach, describe, expect, test, vi } from 'vitest'; +import { fireEvent, render, screen, waitFor } from '@testing-library/svelte'; +import { mockQuery } from '../../test-utils/query'; +import type { Candidate } from '#lib/api/types.js'; + +vi.mock('#lib/api/wanted.js', () => ({ + createCandidatesQuery: vi.fn(), + grabCandidate: vi.fn().mockResolvedValue({ added: true, refused: false, gone: false, unconfigured: false, kept: false, replaced: false }), + blockCandidate: vi.fn().mockResolvedValue(undefined), + unblockRelease: vi.fn().mockResolvedValue(undefined) +})); +vi.mock('#lib/stores/toast.svelte.js', () => ({ pushToast: vi.fn() })); + +import WantCandidates from './WantCandidates.svelte'; +import { blockCandidate, createCandidatesQuery, grabCandidate, unblockRelease } from '#lib/api/wanted.js'; +import { pushToast } from '#lib/stores/toast.svelte.js'; + +function cand(key: string, extra: Partial = {}): Candidate { + return { + key, + name: `Daft Punk - Random Access Memories (2013) [${key}]`, + info_hash: key, + indexer: 'Knaben', + size_bytes: 450_000_000, + seeders: 12, + files_count: 13, + format: 'FLAC', + quality: 'lossless', + year: 2013, + verdict: 'grab', + reason: 'The album, FLAC.', + reasons: [], + folded_from: [], + searched_at: new Date().toISOString(), + downloading: false, + ...extra + }; +} + +afterEach(() => vi.clearAllMocks()); + +function setup(items: Candidate[]) { + vi.mocked(createCandidatesQuery).mockReturnValue( + mockQuery({ data: items }) as unknown as ReturnType + ); + return render(WantCandidates, { wantId: 'w1', title: 'Random Access Memories' }); +} + +describe('WantCandidates', () => { + test('every release shows its verdict and why', () => { + setup([ + cand('aaaa'), + cand('bbbb', { verdict: 'rejected', reason: 'This is a drumless edition (“drumless”).' }) + ]); + expect(screen.getAllByTestId('candidate-row')).toHaveLength(2); + expect(screen.getByText('Would take')).toBeTruthy(); + expect(screen.getByText('Not the album')).toBeTruthy(); + expect(screen.getByText(/drumless edition/)).toBeTruthy(); + expect(screen.getAllByText(/FLAC · 450 MB · 12 seeders · Knaben/)).toHaveLength(2); + }); + + test('a release the verdict would take is grabbed; any other is grabbed anyway', async () => { + setup([cand('aaaa'), cand('bbbb', { verdict: 'review', reason: 'Nobody is sharing it right now.' })]); + await fireEvent.click(screen.getByRole('button', { name: /^grab daft punk.*\[aaaa\]/i })); + await waitFor(() => expect(grabCandidate).toHaveBeenCalledWith('w1', 'aaaa', false)); + await waitFor(() => expect(pushToast).toHaveBeenCalledWith('Taken for Random Access Memories.', 'info')); + await fireEvent.click(screen.getByRole('button', { name: /^grab anyway daft punk.*\[bbbb\]/i })); + await waitFor(() => expect(grabCandidate).toHaveBeenCalledWith('w1', 'bbbb', true)); + }); + + test('a release can be set aside, and allowed again', async () => { + setup([cand('aaaa'), cand('bbbb', { verdict: 'blocked', reason: 'You set this release aside.' })]); + await fireEvent.click(screen.getByRole('button', { name: /never take .*\[aaaa\]/i })); + await waitFor(() => expect(blockCandidate).toHaveBeenCalledWith('w1', 'aaaa')); + await fireEvent.click(screen.getByRole('button', { name: /allow .*\[bbbb\] again/i })); + await waitFor(() => expect(unblockRelease).toHaveBeenCalledWith('bbbb')); + expect(screen.queryByRole('button', { name: /grab.*\[bbbb\]/i })).toBeNull(); + }); + + test('the release downloading says so and offers nothing', () => { + setup([cand('aaaa', { downloading: true })]); + expect(screen.getByText('Downloading')).toBeTruthy(); + expect(screen.queryByRole('button', { name: /grab/i })).toBeNull(); + }); + + test('before any search, it says how to get one', () => { + setup([]); + expect(screen.getByText(/nothing kept from a search yet/i)).toBeTruthy(); + }); +}); diff --git a/web/src/lib/styles/error-copy.json b/web/src/lib/styles/error-copy.json index 68a365d9..90abec17 100644 --- a/web/src/lib/styles/error-copy.json +++ b/web/src/lib/styles/error-copy.json @@ -67,5 +67,11 @@ "profile_is_default": "The default profile can't be deleted. Make another one the default first.", "want_not_found": "That's no longer on the wanted list. Refresh the page.", "follow_not_found": "That artist isn't followed any more. Refresh the page.", - "invalid_profile": "That isn't a release-type profile." + "invalid_profile": "That isn't a release-type profile.", + "bad_interval": "Hours between searches must be between 0 and 720.", + "bad_quality": "Choose at least one quality to take.", + "candidate_not_found": "That release is no longer in the list. Search again.", + "no_infohash": "That release can't be set aside: its indexer gave no infohash to know it by.", + "want_held": "The library already holds that album.", + "block_not_found": "That release isn't set aside any more." } diff --git a/web/src/routes/admin/integrations/+page.svelte b/web/src/routes/admin/integrations/+page.svelte index fa9be790..94542a0d 100644 --- a/web/src/routes/admin/integrations/+page.svelte +++ b/web/src/routes/admin/integrations/+page.svelte @@ -30,6 +30,7 @@ import AcoustIDCard from '#lib/components/AcoustIDCard.svelte'; import AcquisitionCard from '#lib/components/AcquisitionCard.svelte'; import ReleaseProfilesCard from '#lib/components/ReleaseProfilesCard.svelte'; + import SearchSettingsCard from '#lib/components/SearchSettingsCard.svelte'; import NetworkSettingsCard from '#lib/components/NetworkSettingsCard.svelte'; import PublicAddressCard from '#lib/components/PublicAddressCard.svelte'; import NotificationEmailCard from '#lib/components/NotificationEmailCard.svelte'; @@ -424,6 +425,8 @@ + + diff --git a/web/src/routes/admin/integrations/integrations.test.ts b/web/src/routes/admin/integrations/integrations.test.ts index e5449fd7..26b475d7 100644 --- a/web/src/routes/admin/integrations/integrations.test.ts +++ b/web/src/routes/admin/integrations/integrations.test.ts @@ -60,7 +60,22 @@ vi.mock('#lib/api/wanted.js', async () => { ), createReleaseProfile: vi.fn(), updateReleaseProfile: vi.fn(), - deleteReleaseProfile: vi.fn() + deleteReleaseProfile: vi.fn(), + // SearchSettingsCard, settled the same way. + createSearchSettingsQuery: vi.fn(() => + mockQuery({ + data: { + auto_grab: true, + search_interval_hours: 24, + // Not "Lossless": a Lidarr quality profile on this page is named that. + allowed_qualities: ['lossy_high'], + tiers: [{ tier: 'lossy_high', label: 'High quality lossy', what: 'MP3 320' }], + last_pass_at: null, + last_pass: {} + } + }) + ), + putSearchSettings: vi.fn() }; }); diff --git a/web/src/routes/admin/wanted/+page.svelte b/web/src/routes/admin/wanted/+page.svelte index 1fed0c87..d791c2d5 100644 --- a/web/src/routes/admin/wanted/+page.svelte +++ b/web/src/routes/admin/wanted/+page.svelte @@ -1,7 +1,7 @@ e.key === 'Escape' && (menuOpen = false)} /> diff --git a/web/src/lib/components/Shell.test.ts b/web/src/lib/components/Shell.test.ts index f7b658df..92a63aab 100644 --- a/web/src/lib/components/Shell.test.ts +++ b/web/src/lib/components/Shell.test.ts @@ -67,6 +67,21 @@ describe('Shell', () => { expect(screen.getByRole('link', { name: 'Discover' })).toHaveAttribute('href', '/discover'); }); + test('admins get Wanted after Discover; others do not', () => { + const primary = () => + screen + .getAllByRole('link') + .filter((el) => el.closest('nav[aria-label="Primary"]') !== null) + .map((el) => el.textContent?.trim()); + const { unmount } = render(Shell); + expect(primary()).not.toContain('Wanted'); + unmount(); + userState.current = { id: '1', username: 'alice', is_admin: true }; + render(Shell); + expect(primary()).toEqual(['Home', 'Library', 'Discover', 'Wanted']); + expect(screen.getByRole('link', { name: 'Wanted' })).toHaveAttribute('href', '/wanted'); + }); + test('main nav omits Artists/Albums/Liked/History/Playlists/Search/Settings/Admin', () => { userState.current = { id: '1', username: 'alice', is_admin: true }; render(Shell); diff --git a/web/src/lib/components/WantedTabs.svelte b/web/src/lib/components/WantedTabs.svelte new file mode 100644 index 00000000..3a9bf1a4 --- /dev/null +++ b/web/src/lib/components/WantedTabs.svelte @@ -0,0 +1,54 @@ + + + diff --git a/web/src/lib/components/wanted/BlockList.svelte b/web/src/lib/components/wanted/BlockList.svelte new file mode 100644 index 00000000..cd41332a --- /dev/null +++ b/web/src/lib/components/wanted/BlockList.svelte @@ -0,0 +1,88 @@ + + +{#if blocks.isPending} +

Reading the blocked releases…

+{:else if blocks.isError && !blocks.isFetchNextPageError} +

Couldn't load the blocked releases.

+{:else if items.length === 0} +
+ +

No releases are blocked.

+

+ A release you set aside, or one an import found wasn't the album, lands here so it's never + taken again. +

+
+{:else} +
    + {#each items as b (b.info_hash)} +
  • +
    +
    {b.name || b.info_hash}
    + {#if b.want_title} +
    For {b.want_artist} – {b.want_title}
    + {/if} +
    {by(b)} {relativeTime(b.blocked_at)}
    + {#if b.detail} +

    {b.detail}

    + {/if} +
    + +
    • + {/each} +
+ + blocks.fetchNextPage()} + /> +{/if} diff --git a/web/src/lib/components/wanted/BlockList.test.ts b/web/src/lib/components/wanted/BlockList.test.ts new file mode 100644 index 00000000..bd3574c3 --- /dev/null +++ b/web/src/lib/components/wanted/BlockList.test.ts @@ -0,0 +1,50 @@ +import { afterEach, describe, expect, test, vi } from 'vitest'; +import { fireEvent, render, screen, waitFor } from '@testing-library/svelte'; +import { mockInfiniteQuery } from '../../../test-utils/query'; +import type { ReleaseBlock } from '#lib/api/types.js'; + +vi.mock('#lib/api/wanted.js', () => ({ + createReleaseBlocksQuery: vi.fn(), + unblockRelease: vi.fn().mockResolvedValue(undefined) +})); +vi.mock('#lib/stores/toast.svelte.js', () => ({ pushToast: vi.fn() })); + +import BlockList from './BlockList.svelte'; +import { createReleaseBlocksQuery, unblockRelease } from '#lib/api/wanted.js'; + +const block: ReleaseBlock = { + info_hash: 'abc123', + name: 'Daft Punk - Random Access Memories (Drumless Edition)', + reason: 'import', + detail: 'The album tag says another version.', + blocked_at: new Date(Date.now() - 3_600_000).toISOString(), + want_title: 'Random Access Memories', + want_artist: 'Daft Punk' +}; + +afterEach(() => vi.clearAllMocks()); + +function setup(items: ReleaseBlock[]) { + vi.mocked(createReleaseBlocksQuery).mockReturnValue( + mockInfiniteQuery({ pages: [{ items, total: items.length, limit: 50, offset: 0 }] }) as unknown as ReturnType< + typeof createReleaseBlocksQuery + > + ); + return render(BlockList); +} + +describe('BlockList', () => { + test('a blocked release says why and can be allowed again', async () => { + setup([block]); + expect(screen.getByText(/refused on import/i)).toBeTruthy(); + expect(screen.getByText(/another version/)).toBeTruthy(); + expect(screen.getByText(/for daft punk – random access memories/i)).toBeTruthy(); + await fireEvent.click(screen.getByRole('button', { name: /allow .* again/i })); + await waitFor(() => expect(unblockRelease).toHaveBeenCalledWith('abc123')); + }); + + test('with nothing blocked it says so', () => { + setup([]); + expect(screen.getByText(/no releases are blocked/i)).toBeTruthy(); + }); +}); diff --git a/web/src/lib/components/wanted/DownloadList.svelte b/web/src/lib/components/wanted/DownloadList.svelte new file mode 100644 index 00000000..183babbb --- /dev/null +++ b/web/src/lib/components/wanted/DownloadList.svelte @@ -0,0 +1,192 @@ + + +{#if downloads.isPending} +

Reading the downloads…

+{:else if downloads.isError && !downloads.isFetchNextPageError} +

Couldn't load the downloads.

+{:else if items.length === 0} +
+ +

Nothing has been downloaded yet.

+

+ A release the search takes, or one you take from a want's Releases, shows here while it + downloads and once it's imported. +

+
+{:else} +
    + {#each items as d (d.id)} +
  • +
    +
    +
    + {d.want_artist ? `${d.want_artist} – ${d.want_title}` : d.name} +
    +
    + {d.name}{#if d.quality} · {d.quality}{/if} +
    +
    + + {label(d)} + +
    + + {#if !d.ended_at && (d.import_state === 'downloading' || d.import_state === '')} +
    +
    +
    + {/if} + + {#if d.import_reason} +

    + {d.import_reason} +

    + {/if} + {#if d.evidence} +

    {counted(d.evidence)}

    + {/if} + {#if filed(d)} +

    {filed(d)}

    + {/if} + {#if d.rehomed_from} +

    Taken for {d.rehomed_from}; it turned out to be this album.

    + {/if} +

    {when(d)}

    + + {#if needsYou(d)} +
    + {#if d.import_state === 'review'} + + {:else} + + {/if} + +
    + {/if} +
    • + {/each} +
+ + downloads.fetchNextPage()} + /> +{/if} diff --git a/web/src/lib/components/wanted/DownloadList.test.ts b/web/src/lib/components/wanted/DownloadList.test.ts new file mode 100644 index 00000000..812353c2 --- /dev/null +++ b/web/src/lib/components/wanted/DownloadList.test.ts @@ -0,0 +1,109 @@ +import { afterEach, describe, expect, test, vi } from 'vitest'; +import { fireEvent, render, screen, waitFor } from '@testing-library/svelte'; +import { mockInfiniteQuery } from '../../../test-utils/query'; +import type { Download } from '#lib/api/types.js'; + +vi.mock('#lib/api/wanted.js', () => ({ + createDownloadsQuery: vi.fn(), + importAnyway: vi.fn().mockResolvedValue(undefined), + retryImport: vi.fn().mockResolvedValue(undefined), + refuseDownload: vi.fn().mockResolvedValue(undefined) +})); +vi.mock('#lib/stores/toast.svelte.js', () => ({ pushToast: vi.fn() })); + +import DownloadList from './DownloadList.svelte'; +import { createDownloadsQuery, importAnyway, refuseDownload, retryImport } from '#lib/api/wanted.js'; + +const ago = (ms: number) => new Date(Date.now() - ms).toISOString(); + +function download(id: string, extra: Partial = {}): Download { + return { + id, + want_id: 'w1', + want_title: 'Random Access Memories', + want_artist: 'Daft Punk', + name: 'Daft Punk - Random Access Memories (2013) [FLAC]', + indexer: 'Prowlarr', + client: 'qbittorrent', + quality: 'lossless', + origin: 'schedule', + grabbed_at: ago(3_600_000), + ended_at: null, + end_reason: '', + progress: 0, + import_state: 'downloading', + import_reason: '', + imported_at: null, + imported_files: 0, + destination: '', + files: [], + skipped: 0, + ...extra + }; +} + +afterEach(() => vi.clearAllMocks()); + +function setup(items: Download[]) { + vi.mocked(createDownloadsQuery).mockReturnValue( + mockInfiniteQuery({ pages: [{ items, total: items.length, limit: 50, offset: 0 }] }) as unknown as ReturnType< + typeof createDownloadsQuery + > + ); + return render(DownloadList); +} + +describe('DownloadList', () => { + test('a running download shows how far it has got', () => { + setup([download('d1', { progress: 0.375 })]); + expect(screen.getByTestId('download-state').textContent).toMatch(/downloading/i); + expect(screen.getByRole('progressbar', { name: /downloaded/i }).getAttribute('aria-valuenow')).toBe('37'); + expect(screen.queryByRole('button')).toBeNull(); + }); + + test('a doubtful download says why and can be imported anyway or refused', async () => { + setup([ + download('d1', { + import_state: 'review', + import_reason: 'Only the lengths agree. A person decides.', + evidence: { files: 13, tracks: 13, identified: 0, recordings: 0, lengths: 13, titles: 0, fingerprinted: false } + }) + ]); + expect(screen.getByTestId('download-state').textContent).toMatch(/needs you/i); + expect(screen.getByTestId('download-reason').textContent).toMatch(/a person decides/i); + expect(screen.getByText(/13 files against 13 tracks · no AcoustID · 13 lengths agree/)).toBeTruthy(); + await fireEvent.click(screen.getByRole('button', { name: /import anyway/i })); + await waitFor(() => expect(importAnyway).toHaveBeenCalledWith('d1')); + await fireEvent.click(screen.getByRole('button', { name: /refuse/i })); + await waitFor(() => expect(refuseDownload).toHaveBeenCalledWith('d1')); + }); + + test('a failed import can be tried again', async () => { + setup([download('d1', { import_state: 'failed', import_reason: 'The library folder is not writable.' })]); + expect(screen.queryByRole('button', { name: /import anyway/i })).toBeNull(); + await fireEvent.click(screen.getByRole('button', { name: /try again/i })); + await waitFor(() => expect(retryImport).toHaveBeenCalledWith('d1')); + }); + + test('an imported download says what was filed where', () => { + setup([ + download('d1', { + ended_at: ago(60_000), + end_reason: 'imported', + import_state: 'imported', + imported_at: ago(60_000), + imported_files: 12, + skipped: 1, + destination: '/music/Daft Punk/Random Access Memories (2013)' + }) + ]); + expect(screen.getByTestId('download-state').textContent).toMatch(/imported/i); + expect(screen.getByText(/12 tracks filed, 1 already held into \/music\/Daft Punk/)).toBeTruthy(); + expect(screen.queryByRole('button')).toBeNull(); + }); + + test('with nothing downloaded it says what lands here', () => { + setup([]); + expect(screen.getByText(/nothing has been downloaded yet/i)).toBeTruthy(); + }); +}); diff --git a/web/src/lib/components/wanted/FollowList.svelte b/web/src/lib/components/wanted/FollowList.svelte new file mode 100644 index 00000000..bd1e3fbd --- /dev/null +++ b/web/src/lib/components/wanted/FollowList.svelte @@ -0,0 +1,117 @@ + + +{#if follows.isPending} +

Reading who's followed…

+{:else if follows.isError} +

Couldn't load the followed artists.

+{:else if (follows.data ?? []).length === 0} +

No artists are followed. Follow one from its artist page.

+{:else} +
    + {#each follows.data ?? [] as f (f.artist_mbid)} +
  • + +
    +
    + {#if f.artist_id} + {f.artist_name} + {:else} + {f.artist_name} + {/if} +
    +
    + {f.open_wants === 1 ? '1 album wanted' : `${f.open_wants} albums wanted`} · + {f.checked_at ? `checked ${relativeTime(f.checked_at)}` : 'not checked yet'} +
    +
    + + +
    • + {/each} +
+{/if} diff --git a/web/src/lib/components/wanted/FollowList.test.ts b/web/src/lib/components/wanted/FollowList.test.ts new file mode 100644 index 00000000..7ae23c05 --- /dev/null +++ b/web/src/lib/components/wanted/FollowList.test.ts @@ -0,0 +1,73 @@ +import { afterEach, describe, expect, test, vi } from 'vitest'; +import { fireEvent, render, screen, waitFor } from '@testing-library/svelte'; +import { mockQuery } from '../../../test-utils/query'; +import type { Follow, ReleaseProfilesResponse } from '#lib/api/types.js'; + +vi.mock('#lib/api/wanted.js', () => ({ + createFollowsQuery: vi.fn(), + createReleaseProfilesQuery: vi.fn(), + putFollow: vi.fn().mockResolvedValue({ following: true }), + deleteFollow: vi.fn().mockResolvedValue(undefined) +})); +vi.mock('#lib/stores/toast.svelte.js', () => ({ pushToast: vi.fn() })); + +import FollowList from './FollowList.svelte'; +import { createFollowsQuery, createReleaseProfilesQuery, deleteFollow, putFollow } from '#lib/api/wanted.js'; + +const ago = (ms: number) => new Date(Date.now() - ms).toISOString(); + +const follow: Follow = { + artist_mbid: 'blur', + artist_name: 'Blur', + artist_id: 'art-1', + profile_id: 'p-std', + profile_name: 'Standard', + followed_at: ago(10 * 86_400_000), + checked_at: ago(3 * 3_600_000), + open_wants: 2, + image_url: '' +}; + +const profiles: ReleaseProfilesResponse = { + profiles: [ + { id: 'p-std', name: 'Standard', primary_types: ['Album'], secondary_types: [], is_default: true, follow_count: 1 }, + { id: 'p-live', name: 'With live', primary_types: ['Album'], secondary_types: ['Live'], is_default: false, follow_count: 0 } + ], + primary_types: ['Album'], + secondary_types: ['Live'] +}; + +afterEach(() => vi.clearAllMocks()); + +function setup(follows: Follow[] = [follow]) { + vi.mocked(createFollowsQuery).mockReturnValue( + mockQuery({ data: follows }) as unknown as ReturnType + ); + vi.mocked(createReleaseProfilesQuery).mockReturnValue( + mockQuery({ data: profiles }) as unknown as ReturnType + ); + return render(FollowList); +} + +describe('FollowList', () => { + test('lists artists; the profile can change and the artist be unfollowed', async () => { + setup(); + expect(screen.getAllByTestId('follow-row')).toHaveLength(1); + expect(screen.getByText(/2 albums wanted/)).toBeTruthy(); + + const select = screen.getByLabelText(/profile for blur/i) as HTMLSelectElement; + expect(select.value).toBe('p-std'); + await fireEvent.change(select, { target: { value: 'p-live' } }); + await waitFor(() => + expect(putFollow).toHaveBeenCalledWith('blur', { artist_name: 'Blur', profile_id: 'p-live' }) + ); + + await fireEvent.click(screen.getByRole('button', { name: /unfollow blur/i })); + await waitFor(() => expect(deleteFollow).toHaveBeenCalledWith('blur')); + }); + + test('with no follows it says where to follow from', () => { + setup([]); + expect(screen.getByText(/follow one from its artist page/i)).toBeTruthy(); + }); +}); diff --git a/web/src/lib/components/wanted/WantList.svelte b/web/src/lib/components/wanted/WantList.svelte new file mode 100644 index 00000000..d13e390f --- /dev/null +++ b/web/src/lib/components/wanted/WantList.svelte @@ -0,0 +1,263 @@ + + +{#if wants.isPending} +

Reading the wanted list…

+{:else if wants.isError && !wants.isFetchNextPageError} +

Couldn't load the wanted list.

+{:else if items.length === 0} +
+ +

+ {#if state === 'held'}Nothing on the list has been held yet. + {:else if state === 'missing'}No missing albums are wanted back. + {:else}Nothing is wanted.{/if} +

+

+ {#if state === 'missing'} + An album whose files stay missing past the grace window on the Missing files page lands here. + {:else} + A request you approve, a new release from a followed artist, or an album whose files went + missing lands here. + {/if} +

+
+{:else} +
    + {#each items as w (w.id)} +
  • +
    + +
    +
    + {#if w.album_id && w.held_at} + {w.title} + {:else} + {w.title} + {/if} +
    +
    + {#if w.artist_id} + {w.artist_name} + {:else} + {w.artist_name} + {/if} + {#if kind(w)} · {kind(w)}{/if} +
    +
    + {why(w)} · {w.held_at ? `held ${relativeTime(w.held_at)}` : `wanted ${relativeTime(w.wanted_at)}`} +
    + {#if !w.held_at} +
    + {status(w)} + {#if attention(w)} + Decide on Activity + {/if} +
    + {:else if arrived(w)} +
    {arrived(w)}
    + {/if} +
    + {#if !w.held_at} +
    + + + +
    + {/if} +
    + {#if open === w.id && !w.held_at} +
    + +
    + {/if} +
    • + {/each} +
+ + wants.fetchNextPage()} + /> +{/if} diff --git a/web/src/routes/admin/wanted/wanted.test.ts b/web/src/lib/components/wanted/WantList.test.ts similarity index 58% rename from web/src/routes/admin/wanted/wanted.test.ts rename to web/src/lib/components/wanted/WantList.test.ts index 410505ae..47e6ae09 100644 --- a/web/src/routes/admin/wanted/wanted.test.ts +++ b/web/src/lib/components/wanted/WantList.test.ts @@ -1,15 +1,11 @@ import { afterEach, describe, expect, test, vi } from 'vitest'; import { fireEvent, render, screen, waitFor } from '@testing-library/svelte'; import { mockInfiniteQuery, mockQuery } from '../../../test-utils/query'; -import type { Follow, ReleaseProfilesResponse, Want, WantsPage } from '#lib/api/types.js'; +import type { Want, WantGrab, WantsPage } from '#lib/api/types.js'; vi.mock('#lib/api/wanted.js', () => ({ createWantsQuery: vi.fn(), - createFollowsQuery: vi.fn(), - createReleaseProfilesQuery: vi.fn(), removeWant: vi.fn().mockResolvedValue(undefined), - putFollow: vi.fn().mockResolvedValue({ following: true }), - deleteFollow: vi.fn().mockResolvedValue(undefined), searchWant: vi.fn().mockResolvedValue({ outcome: 'nothing', considered: 0, best_release: '', best_verdict: '', narrowed: [] }), @@ -20,18 +16,10 @@ vi.mock('#lib/api/wanted.js', () => ({ })); vi.mock('#lib/stores/toast.svelte.js', () => ({ pushToast: vi.fn() })); -import AdminWantedPage from './+page.svelte'; +import WantList from './WantList.svelte'; import { pushToast } from '#lib/stores/toast.svelte.js'; -import { - createCandidatesQuery, - createFollowsQuery, - createReleaseProfilesQuery, - createWantsQuery, - deleteFollow, - putFollow, - removeWant, - searchWant -} from '#lib/api/wanted.js'; +import { createCandidatesQuery, createWantsQuery, removeWant, searchWant } from '#lib/api/wanted.js'; +import type { WantState } from '#lib/api/types.js'; const DAY = 24 * 3_600_000; const ago = (ms: number) => new Date(Date.now() - ms).toISOString(); @@ -61,54 +49,48 @@ function page(items: Want[]): WantsPage { total: items.length, limit: 50, offset: 0, - counts: { wanted: 2, missing: 1, held: 4, following: 1 } + counts: { wanted: 2, missing: 1, held: 4, following: 1, activity: 1, attention: 0, blocked: 0 } }; } -const follow: Follow = { - artist_mbid: 'blur', - artist_name: 'Blur', - artist_id: 'art-1', - profile_id: 'p-std', - profile_name: 'Standard', - followed_at: ago(10 * DAY), - checked_at: ago(3 * 3_600_000), - open_wants: 2, - image_url: '' -}; - -const profiles: ReleaseProfilesResponse = { - profiles: [ - { id: 'p-std', name: 'Standard', primary_types: ['Album'], secondary_types: [], is_default: true, follow_count: 1 }, - { id: 'p-live', name: 'With live', primary_types: ['Album'], secondary_types: ['Live'], is_default: false, follow_count: 0 } - ], - primary_types: ['Album'], - secondary_types: ['Live'] -}; +function grab(extra: Partial = {}): WantGrab { + return { + id: 'g1', + name: 'Blur - Blur (1997) [FLAC]', + grabbed_at: ago(60_000), + origin: 'schedule', + quality: 'lossless', + live: true, + end_reason: '', + progress: 0, + import_state: '', + import_reason: '', + imported_at: null, + imported_files: 0, + ...extra + }; +} afterEach(() => vi.clearAllMocks()); -function setup(items: Want[] = [want('w1'), want('w2', { reason: 'request', requested_by: 'sam', title: 'Parklife' })]) { +function setup( + items: Want[] = [want('w1'), want('w2', { reason: 'request', requested_by: 'sam', title: 'Parklife' })], + state: WantState = 'wanted' +) { vi.mocked(createWantsQuery).mockReturnValue( mockInfiniteQuery({ pages: [page(items)] }) as unknown as ReturnType ); - vi.mocked(createFollowsQuery).mockReturnValue( - mockQuery({ data: [follow] }) as unknown as ReturnType - ); - vi.mocked(createReleaseProfilesQuery).mockReturnValue( - mockQuery({ data: profiles }) as unknown as ReturnType - ); vi.mocked(createCandidatesQuery).mockReturnValue( mockQuery({ data: [] }) as unknown as ReturnType ); - return render(AdminWantedPage); + return render(WantList, { state }); } -describe('/admin/wanted', () => { - test('each tab shows its count', () => { - setup(); - expect(screen.getByRole('tab', { name: /wanted/i }).textContent).toMatch(/2/); - expect(screen.getByRole('tab', { name: /held/i }).textContent).toMatch(/4/); +describe('WantList', () => { + test('asks for its own state', () => { + setup([], 'missing'); + expect(createWantsQuery).toHaveBeenCalledWith('missing'); + expect(screen.getByText(/no missing albums are wanted back/i)).toBeTruthy(); }); test('a want says why it is wanted', () => { @@ -118,12 +100,6 @@ describe('/admin/wanted', () => { expect(why[1]).toMatch(/requested by sam/i); }); - test('switching tab asks for that state', async () => { - setup(); - await fireEvent.click(screen.getByRole('tab', { name: /missing files/i })); - expect(createWantsQuery).toHaveBeenCalledWith('missing'); - }); - test('remove takes a want off the list', async () => { setup(); await fireEvent.click(screen.getByRole('button', { name: /stop wanting parklife/i })); @@ -136,22 +112,6 @@ describe('/admin/wanted', () => { expect(screen.getByRole('link', { name: /the ballad of darren/i }).getAttribute('href')).toBe('/albums/al-1'); }); - test('following lists artists; the profile can change and the artist be unfollowed', async () => { - setup(); - await fireEvent.click(screen.getByRole('tab', { name: /following/i })); - expect(screen.getAllByTestId('follow-row')).toHaveLength(1); - - const select = screen.getByLabelText(/profile for blur/i) as HTMLSelectElement; - expect(select.value).toBe('p-std'); - await fireEvent.change(select, { target: { value: 'p-live' } }); - await waitFor(() => - expect(putFollow).toHaveBeenCalledWith('blur', { artist_name: 'Blur', profile_id: 'p-live' }) - ); - - await fireEvent.click(screen.getByRole('button', { name: /unfollow blur/i })); - await waitFor(() => expect(deleteFollow).toHaveBeenCalledWith('blur')); - }); - test('an empty list explains what lands there', () => { setup([]); expect(screen.getByText(/nothing is wanted/i)).toBeTruthy(); @@ -164,15 +124,49 @@ describe('/admin/wanted', () => { title: 'Parklife', search: { searched_at: ago(3_600_000), outcome: 'nothing', considered: 4, best_release: 'x', best_verdict: 'rejected', error: '' } }), - want('w3', { - title: 'Blur', - grab: { name: 'Blur - Blur (1997) [FLAC]', grabbed_at: ago(60_000), origin: 'schedule', quality: 'lossless' } - }) + want('w3', { title: 'Blur', grab: grab() }), + want('w4', { title: '13', grab: grab({ import_state: 'downloading', progress: 0.42 }) }) ]); const lines = screen.getAllByTestId('want-search').map((e) => e.textContent ?? ''); expect(lines[0]).toMatch(/not searched yet/i); expect(lines[1]).toMatch(/4 found, none of them this album/i); expect(lines[2]).toMatch(/downloading blur - blur \(1997\) \[flac\], taken by the schedule/i); + expect(lines[3]).toMatch(/downloading 42%/i); + }); + + test('a download waiting on a person says why and points to Activity', () => { + setup([ + want('w1', { grab: grab({ import_state: 'review', import_reason: 'Only the lengths agree. A person decides.' }) }) + ]); + const line = screen.getByTestId('want-search'); + expect(line.textContent).toMatch(/needs you: only the lengths agree/i); + expect(screen.getByRole('link', { name: /decide on activity/i }).getAttribute('href')).toBe('/wanted/activity'); + // A live download can't be searched for again. + expect((screen.getByRole('button', { name: /search for the ballad of darren now/i }) as HTMLButtonElement).disabled).toBe(true); + }); + + test('a refused download is named beside the search that follows it', () => { + setup([ + want('w1', { + search: { searched_at: ago(3_600_000), outcome: 'nothing', considered: 0, best_release: '', best_verdict: '', error: '' }, + grab: grab({ live: false, end_reason: 'refused', import_state: 'refused', import_reason: 'Another album.' }) + }) + ]); + expect(screen.getByTestId('want-search').textContent).toMatch(/last download refused: another album/i); + }); + + test('a held album says how it arrived', () => { + setup( + [ + want('w1', { + held_at: ago(DAY), + album_id: 'al-1', + grab: grab({ live: false, end_reason: 'imported', import_state: 'imported', imported_at: ago(DAY), imported_files: 11 }) + }) + ], + 'held' + ); + expect(screen.getByTestId('want-arrived').textContent).toMatch(/imported 11 tracks .* from blur - blur/i); }); test('Search runs one now and says what came of it', async () => { diff --git a/web/src/lib/styles/error-copy.json b/web/src/lib/styles/error-copy.json index 90abec17..e9a072c6 100644 --- a/web/src/lib/styles/error-copy.json +++ b/web/src/lib/styles/error-copy.json @@ -26,6 +26,9 @@ "bad_body": "Couldn't read the request.", "bad_kind": "Invalid request type.", "bad_paging": "Invalid page size or offset.", + "invalid_paging": "Invalid page size or offset.", + "not_decidable": "That download isn't waiting on a decision any more. Refresh the list.", + "bad_import_root": "New music has to be filed into one of the library's folders.", "bad_reason": "Invalid quarantine reason.", "mbid_required": "An MBID is required for this lookup.", "system_playlist_readonly": "System playlists can't be edited directly.", diff --git a/web/src/routes/admin/integrations/+page.svelte b/web/src/routes/admin/integrations/+page.svelte index 94542a0d..3095e4cf 100644 --- a/web/src/routes/admin/integrations/+page.svelte +++ b/web/src/routes/admin/integrations/+page.svelte @@ -29,7 +29,6 @@ import Modal from '#lib/components/Modal.svelte'; import AcoustIDCard from '#lib/components/AcoustIDCard.svelte'; import AcquisitionCard from '#lib/components/AcquisitionCard.svelte'; - import ReleaseProfilesCard from '#lib/components/ReleaseProfilesCard.svelte'; import SearchSettingsCard from '#lib/components/SearchSettingsCard.svelte'; import NetworkSettingsCard from '#lib/components/NetworkSettingsCard.svelte'; import PublicAddressCard from '#lib/components/PublicAddressCard.svelte'; @@ -427,8 +426,6 @@ - -
diff --git a/web/src/routes/admin/integrations/integrations.test.ts b/web/src/routes/admin/integrations/integrations.test.ts index 26b475d7..5bd63952 100644 --- a/web/src/routes/admin/integrations/integrations.test.ts +++ b/web/src/routes/admin/integrations/integrations.test.ts @@ -43,25 +43,10 @@ vi.mock('#lib/api/admin.js', () => ({ addAcquisitionService: vi.fn() })); -// ReleaseProfilesCard reads its own query; settled, so it adds no "Loading…". +// SearchSettingsCard reads its own query; settled, so it adds no "Loading…". vi.mock('#lib/api/wanted.js', async () => { const { mockQuery } = await import('../../../test-utils/query'); return { - createReleaseProfilesQuery: vi.fn(() => - mockQuery({ - data: { - profiles: [ - { id: 'p-std', name: 'Standard', primary_types: ['Album', 'EP', 'Single'], secondary_types: [], is_default: true, follow_count: 3 } - ], - primary_types: ['Album', 'EP', 'Single'], - secondary_types: ['Live'] - } - }) - ), - createReleaseProfile: vi.fn(), - updateReleaseProfile: vi.fn(), - deleteReleaseProfile: vi.fn(), - // SearchSettingsCard, settled the same way. createSearchSettingsQuery: vi.fn(() => mockQuery({ data: { @@ -71,7 +56,9 @@ vi.mock('#lib/api/wanted.js', async () => { allowed_qualities: ['lossy_high'], tiers: [{ tier: 'lossy_high', label: 'High quality lossy', what: 'MP3 320' }], last_pass_at: null, - last_pass: {} + last_pass: {}, + import_root: '', + scan_paths: ['/music'] } }) ), diff --git a/web/src/routes/admin/missing-files/+page.svelte b/web/src/routes/admin/missing-files/+page.svelte index f64636f5..fb89e25d 100644 --- a/web/src/routes/admin/missing-files/+page.svelte +++ b/web/src/routes/admin/missing-files/+page.svelte @@ -107,7 +107,7 @@ {#if r.held_at} Held again {relativeTime(r.held_at)}. The rows here clear on the next scan. {:else if r.wanted_at} - On the wanted list since {relativeTime(r.wanted_at)}. {/if} diff --git a/web/src/routes/admin/missing-files/missing-files.test.ts b/web/src/routes/admin/missing-files/missing-files.test.ts index e8adf9b7..cb804653 100644 --- a/web/src/routes/admin/missing-files/missing-files.test.ts +++ b/web/src/routes/admin/missing-files/missing-files.test.ts @@ -140,7 +140,7 @@ describe('admin missing files', () => { expect( within(screen.getByTestId('reacquisition-state')).getByRole('link', { name: /wanted list/i }).getAttribute('href') ).toBe( - '/admin/wanted?tab=missing' + '/wanted/missing' ); }); diff --git a/web/src/routes/admin/requests/+page.svelte b/web/src/routes/admin/requests/+page.svelte index 06395cd9..b2952c24 100644 --- a/web/src/routes/admin/requests/+page.svelte +++ b/web/src/routes/admin/requests/+page.svelte @@ -146,7 +146,7 @@

Requests

Approve or set aside what users have asked for. An approved album or track goes on the - wanted list; an approved + wanted list; an approved artist is followed, wanting what they've released and what they release next.

diff --git a/web/src/routes/admin/wanted/+page.svelte b/web/src/routes/admin/wanted/+page.svelte deleted file mode 100644 index d791c2d5..00000000 --- a/web/src/routes/admin/wanted/+page.svelte +++ /dev/null @@ -1,363 +0,0 @@ - - -{pageTitle('Admin · Wanted')} - -
-
-

Wanted

-

- Albums Minstrel is to get: ones users requested, new releases from the artists it follows, - and albums whose files went missing. Minstrel searches your indexers for each one on a - schedule and hands the best release to your download client; searching and quality are set - under Integrations. - An album leaves the list once the library holds it. -

-
- - - - {#if tab === 'following'} -

- Each artist is followed on a release-type profile, which says which kinds of release are - wanted. Profiles are edited under - Integrations. -

- {#if follows.isPending} -

Reading who's followed…

- {:else if follows.isError} -

Couldn't load the followed artists.

- {:else if (follows.data ?? []).length === 0} -

- No artists are followed. Follow one from its artist page. -

- {:else} -
    - {#each follows.data ?? [] as f (f.artist_mbid)} -
  • - -
    -
    - {#if f.artist_id} - {f.artist_name} - {:else} - {f.artist_name} - {/if} -
    -
    - {f.open_wants === 1 ? '1 album wanted' : `${f.open_wants} albums wanted`} · - {f.checked_at ? `checked ${relativeTime(f.checked_at)}` : 'not checked yet'} -
    -
    - - -
    • - {/each} -
- {/if} - {:else if wants.isPending} -

Reading the wanted list…

- {:else if wants.isError && !wants.isFetchNextPageError} -

Couldn't load the wanted list.

- {:else if items.length === 0} -
- -

- {#if tab === 'held'}Nothing on the list has been held yet. - {:else if tab === 'missing'}No missing albums are wanted back. - {:else}Nothing is wanted.{/if} -

-

- {#if tab === 'missing'} - An album whose files stay missing past the grace window on the Missing files page lands here. - {:else} - A request you approve, a new release from a followed artist, or an album whose files went - missing lands here. - {/if} -

-
- {:else} -
    - {#each items as w (w.id)} -
  • -
    - -
    -
    - {#if w.album_id && w.held_at} - {w.title} - {:else} - {w.title} - {/if} -
    -
    - {#if w.artist_id} - {w.artist_name} - {:else} - {w.artist_name} - {/if} - {#if kind(w)} · {kind(w)}{/if} -
    -
    - {why(w)} · {w.held_at ? `held ${relativeTime(w.held_at)}` : `wanted ${relativeTime(w.wanted_at)}`} -
    - {#if !w.held_at} -
    {status(w)}
    - {/if} -
    - {#if !w.held_at} -
    - - - -
    - {/if} -
    - {#if open === w.id && !w.held_at} -
    - -
    - {/if} -
    • - {/each} -
- - wants.fetchNextPage()} - /> - {/if} -
diff --git a/web/src/routes/wanted/+layout.svelte b/web/src/routes/wanted/+layout.svelte new file mode 100644 index 00000000..0e2116bf --- /dev/null +++ b/web/src/routes/wanted/+layout.svelte @@ -0,0 +1,48 @@ + + +
+
+

Wanted

+

+ Albums Minstrel is getting: ones users requested, new releases from the artists it follows, + and albums whose files went missing. It searches your indexers on a schedule, checks each + download against MusicBrainz, and files what's right into the library. Searching and quality + are set under Integrations. +

+ {#if verdict} +

+ {verdict} + {#if counts?.attention} + See Activity + {/if} +

+ {/if} +
+ + + + {@render children?.()} +
diff --git a/web/src/routes/wanted/+layout.ts b/web/src/routes/wanted/+layout.ts new file mode 100644 index 00000000..f22d5109 --- /dev/null +++ b/web/src/routes/wanted/+layout.ts @@ -0,0 +1,13 @@ +import { redirect } from '@sveltejs/kit'; +import { user } from '#lib/auth/store.svelte.js'; +import type { LayoutLoad } from './$types'; + +// Hard route gate: the Wanted view is an admin's (M502). Runs before the +// layout renders; auth is bootstrapped by the root +layout.ts, so user.value is +// either a settled User object or null. +export const load: LayoutLoad = () => { + if (!user.value || !user.value.is_admin) { + throw redirect(302, '/'); + } + return {}; +}; diff --git a/web/src/routes/wanted/+page.svelte b/web/src/routes/wanted/+page.svelte new file mode 100644 index 00000000..96b2637e --- /dev/null +++ b/web/src/routes/wanted/+page.svelte @@ -0,0 +1,8 @@ + + +{pageTitle('Wanted')} + + diff --git a/web/src/routes/wanted/activity/+page.svelte b/web/src/routes/wanted/activity/+page.svelte new file mode 100644 index 00000000..c98588ee --- /dev/null +++ b/web/src/routes/wanted/activity/+page.svelte @@ -0,0 +1,8 @@ + + +{pageTitle('Wanted · Activity')} + + diff --git a/web/src/routes/wanted/blocked/+page.svelte b/web/src/routes/wanted/blocked/+page.svelte new file mode 100644 index 00000000..3c4ea0d0 --- /dev/null +++ b/web/src/routes/wanted/blocked/+page.svelte @@ -0,0 +1,8 @@ + + +{pageTitle('Wanted · Blocked')} + + diff --git a/web/src/routes/wanted/following/+page.svelte b/web/src/routes/wanted/following/+page.svelte new file mode 100644 index 00000000..7303fe95 --- /dev/null +++ b/web/src/routes/wanted/following/+page.svelte @@ -0,0 +1,12 @@ + + +{pageTitle('Wanted · Following')} + +
+ + +
diff --git a/web/src/routes/wanted/held/+page.svelte b/web/src/routes/wanted/held/+page.svelte new file mode 100644 index 00000000..dd44902b --- /dev/null +++ b/web/src/routes/wanted/held/+page.svelte @@ -0,0 +1,8 @@ + + +{pageTitle('Wanted · Held')} + + diff --git a/web/src/routes/wanted/missing/+page.svelte b/web/src/routes/wanted/missing/+page.svelte new file mode 100644 index 00000000..f2b7e6df --- /dev/null +++ b/web/src/routes/wanted/missing/+page.svelte @@ -0,0 +1,8 @@ + + +{pageTitle('Wanted · Missing files')} + + diff --git a/web/src/routes/wanted/wanted.test.ts b/web/src/routes/wanted/wanted.test.ts new file mode 100644 index 00000000..9787f91b --- /dev/null +++ b/web/src/routes/wanted/wanted.test.ts @@ -0,0 +1,61 @@ +import { afterEach, describe, expect, test, vi } from 'vitest'; +import { render, screen } from '@testing-library/svelte'; +import { createRawSnippet } from 'svelte'; +import { mockInfiniteQuery } from '../../test-utils/query'; +import { pageUrlModule } from '#test-utils/mocks/appState.js'; +import type { WantCounts, WantsPage } from '#lib/api/types.js'; + +const state = vi.hoisted(() => ({ pageUrl: new URL('http://localhost/wanted') })); +vi.mock('$app/state', () => pageUrlModule(state)); +vi.mock('#lib/api/wanted.js', () => ({ createWantsQuery: vi.fn() })); + +import WantedLayout from './+layout.svelte'; +import { createWantsQuery } from '#lib/api/wanted.js'; + +const counts: WantCounts = { wanted: 5, missing: 1, held: 9, following: 2, activity: 3, attention: 0, blocked: 4 }; +const children = createRawSnippet(() => ({ render: () => `

body

` })); + +afterEach(() => vi.clearAllMocks()); + +function setup(c: WantCounts = counts, path = '/wanted') { + state.pageUrl = new URL(`http://localhost${path}`); + const page: WantsPage = { items: [], total: 0, limit: 50, offset: 0, counts: c }; + vi.mocked(createWantsQuery).mockReturnValue( + mockInfiniteQuery({ pages: [page] }) as unknown as ReturnType + ); + // eslint-disable-next-line @typescript-eslint/no-explicit-any + return render(WantedLayout, { children } as any); +} + +describe('/wanted layout', () => { + test('tabs are links carrying their counts, the current one marked', () => { + setup(counts, '/wanted/activity'); + const nav = screen.getByRole('navigation', { name: /wanted sections/i }); + const links = Array.from(nav.querySelectorAll('a')).map((a) => [a.getAttribute('href'), a.textContent?.replace(/\s+/g, ' ').trim()]); + expect(links).toEqual([ + ['/wanted', 'Wanted 5'], + ['/wanted/activity', 'Activity 3'], + ['/wanted/missing', 'Missing files 1'], + ['/wanted/held', 'Held 9'], + ['/wanted/following', 'Following 2'], + ['/wanted/blocked', 'Blocked 4'] + ]); + expect(screen.getByRole('link', { name: /activity/i })).toHaveAttribute('aria-current', 'page'); + expect(screen.getByRole('link', { name: /^wanted/i })).not.toHaveAttribute('aria-current'); + expect(screen.getByTestId('tab-body')).toBeTruthy(); + }); + + test('when nothing needs a person the verdict says so', () => { + setup(); + expect(screen.getByTestId('wanted-verdict').textContent).toMatch(/5 albums wanted, 3 downloading\. Nothing needs you\./); + expect(screen.queryByTestId('tab-attention')).toBeNull(); + }); + + test('downloads waiting on a person lead the page and mark Activity', () => { + setup({ ...counts, attention: 2 }); + const verdict = screen.getByTestId('wanted-verdict'); + expect(verdict.textContent).toMatch(/2 downloads need you/); + expect(screen.getByRole('link', { name: /see activity/i }).getAttribute('href')).toBe('/wanted/activity'); + expect(screen.getByTestId('tab-attention').textContent).toBe('2'); + }); +}); -- 2.54.0 From f268b4df096a20444bcfd1eacd8ae732e904cc33 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Sat, 10 Oct 2026 10:44:21 -0400 Subject: [PATCH 014/117] fix: WantList's prop is list, not state, which Svelte reads as the $state store; the length test's 4-minute case fits within 3% (M502 #5522) Co-Authored-By: Claude Opus 5.5 --- internal/importer/verify_test.go | 4 +++- web/src/lib/components/wanted/WantList.svelte | 10 +++++----- web/src/lib/components/wanted/WantList.test.ts | 2 +- web/src/routes/wanted/+page.svelte | 2 +- web/src/routes/wanted/held/+page.svelte | 2 +- web/src/routes/wanted/missing/+page.svelte | 2 +- 6 files changed, 12 insertions(+), 10 deletions(-) diff --git a/internal/importer/verify_test.go b/internal/importer/verify_test.go index 6bf8425c..2a25d39f 100644 --- a/internal/importer/verify_test.go +++ b/internal/importer/verify_test.go @@ -166,7 +166,9 @@ func TestLengthFits(t *testing.T) { file, track int ok bool }{ - {240000, 244000, true}, {240000, 246000, false}, {600000, 615000, true}, {600000, 620000, false}, + // Short tracks get the 5s floor; longer ones 3% of the track. + {120000, 125000, true}, {120000, 126000, false}, + {240000, 246000, true}, {240000, 248000, false}, {600000, 615000, true}, {600000, 620000, false}, {0, 240000, false}, {240000, 0, true}, } { if got := lengthFits(c.file, c.track); got != c.ok { diff --git a/web/src/lib/components/wanted/WantList.svelte b/web/src/lib/components/wanted/WantList.svelte index d13e390f..210b1c41 100644 --- a/web/src/lib/components/wanted/WantList.svelte +++ b/web/src/lib/components/wanted/WantList.svelte @@ -15,10 +15,10 @@ // what its last search found and how its download is getting on; Releases // lists every candidate with its verdict, and Search runs one now. - let { state }: { state: WantState } = $props(); + let { list }: { list: WantState } = $props(); const client = useQueryClient(); - const wantsStore = $derived(createWantsQuery(state)); + const wantsStore = $derived(createWantsQuery(list)); const wants = $derived($wantsStore); const items = $derived((wants.data?.pages ?? []).flatMap((p) => p.items) as Want[]); @@ -156,12 +156,12 @@

- {#if state === 'held'}Nothing on the list has been held yet. - {:else if state === 'missing'}No missing albums are wanted back. + {#if list === 'held'}Nothing on the list has been held yet. + {:else if list === 'missing'}No missing albums are wanted back. {:else}Nothing is wanted.{/if}

- {#if state === 'missing'} + {#if list === 'missing'} An album whose files stay missing past the grace window on the Missing files page lands here. {:else} A request you approve, a new release from a followed artist, or an album whose files went diff --git a/web/src/lib/components/wanted/WantList.test.ts b/web/src/lib/components/wanted/WantList.test.ts index 47e6ae09..7ca7a13d 100644 --- a/web/src/lib/components/wanted/WantList.test.ts +++ b/web/src/lib/components/wanted/WantList.test.ts @@ -83,7 +83,7 @@ function setup( vi.mocked(createCandidatesQuery).mockReturnValue( mockQuery({ data: [] }) as unknown as ReturnType ); - return render(WantList, { state }); + return render(WantList, { list: state }); } describe('WantList', () => { diff --git a/web/src/routes/wanted/+page.svelte b/web/src/routes/wanted/+page.svelte index 96b2637e..9d6f7d36 100644 --- a/web/src/routes/wanted/+page.svelte +++ b/web/src/routes/wanted/+page.svelte @@ -5,4 +5,4 @@ {pageTitle('Wanted')} - + diff --git a/web/src/routes/wanted/held/+page.svelte b/web/src/routes/wanted/held/+page.svelte index dd44902b..652d8b20 100644 --- a/web/src/routes/wanted/held/+page.svelte +++ b/web/src/routes/wanted/held/+page.svelte @@ -5,4 +5,4 @@ {pageTitle('Wanted · Held')} - + diff --git a/web/src/routes/wanted/missing/+page.svelte b/web/src/routes/wanted/missing/+page.svelte index f2b7e6df..35163ed2 100644 --- a/web/src/routes/wanted/missing/+page.svelte +++ b/web/src/routes/wanted/missing/+page.svelte @@ -5,4 +5,4 @@ {pageTitle('Wanted · Missing files')} - + -- 2.54.0 From d9e00d8bc2ebc3f394e596ea57f808db52673c4f Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Sat, 10 Oct 2026 10:53:43 -0400 Subject: [PATCH 015/117] fix: the integrations test's library folder isn't /music, which a Lidarr root folder there is named (M502 #5522) Co-Authored-By: Claude Opus 5.5 --- web/src/routes/admin/integrations/integrations.test.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/web/src/routes/admin/integrations/integrations.test.ts b/web/src/routes/admin/integrations/integrations.test.ts index 5bd63952..5b0f3ee8 100644 --- a/web/src/routes/admin/integrations/integrations.test.ts +++ b/web/src/routes/admin/integrations/integrations.test.ts @@ -58,7 +58,8 @@ vi.mock('#lib/api/wanted.js', async () => { last_pass_at: null, last_pass: {}, import_root: '', - scan_paths: ['/music'] + // Not "/music": a Lidarr root folder on this page is named that. + scan_paths: ['/library'] } }) ), -- 2.54.0 From 8245f4406e667bec103a8cbd10ba67d47e62b9f8 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Sat, 10 Oct 2026 15:27:22 -0400 Subject: [PATCH 016/117] =?UTF-8?q?feat:=20download=20lifecycle=20?= =?UTF-8?q?=E2=80=94=20stalled=20and=20failed=20downloads=20are=20given=20?= =?UTF-8?q?up=20and=20searched=20again;=20imported=20ones=20seed,=20then?= =?UTF-8?q?=20go=20(M502=20#5523)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Ported from Fabled Librarian's System 104. A new lifecycle poller, every ten minutes: - a download the client fails is given up at once; - one nobody shares, or still fetching its file list, past the stall setting (6 h) is asked for again three times, then given up; - giving up removes the download with its files, holds the release back for a week (release_blocks.expires_at), ends the grab and searches the album again, with no limit per album; - a running download the client stops listing for half an hour ends and is searched again, not held back; an empty list is not taken as gone; - an imported download seeds to ratio 2.0 or 14 days, then is removed; - a download the client would not remove is asked again every pass. Every condition and act is one open row per (client, download, condition) in download_interventions. Activity shows what is wrong with a download and how often it was retried, why one ended, its seeding, and a banner for a client that does not answer. A running download can be given up by hand (set aside for good). The seed ratio, seed days and stall hours are on the search settings card. The download client lookup moves to acquire.DownloadClientFor, shared by the importer and the lifecycle. Co-Authored-By: Claude Opus 5.5 --- cmd/minstrel/main.go | 6 + internal/acquire/memory.go | 16 +- internal/acquire/settings.go | 16 + internal/api/admin_import.go | 96 ++- internal/api/admin_search.go | 18 +- internal/api/admin_wanted.go | 4 +- internal/api/api.go | 6 + internal/db/dbq/imports.sql.go | 116 ++- internal/db/dbq/lifecycle.sql.go | 389 +++++++++++ internal/db/dbq/models.go | 24 + internal/db/dbq/search.sql.go | 49 +- internal/db/dbq/wanted.sql.go | 7 +- .../0084_download_lifecycle.down.sql | 23 + .../migrations/0084_download_lifecycle.up.sql | 90 +++ internal/db/queries/imports.sql | 18 +- internal/db/queries/lifecycle.sql | 96 +++ internal/db/queries/search.sql | 20 +- internal/db/queries/wanted.sql | 5 +- internal/dbtest/reset.go | 2 + internal/importer/importer.go | 22 +- internal/lifecycle/lifecycle.go | 659 ++++++++++++++++++ internal/lifecycle/lifecycle_test.go | 441 ++++++++++++ web/src/lib/api/types.ts | 37 +- web/src/lib/api/wanted.ts | 9 + .../lib/components/SearchSettingsCard.svelte | 87 ++- .../lib/components/SearchSettingsCard.test.ts | 24 +- .../lib/components/wanted/BlockList.svelte | 12 +- .../lib/components/wanted/BlockList.test.ts | 7 + .../lib/components/wanted/DownloadList.svelte | 81 ++- .../components/wanted/DownloadList.test.ts | 63 +- web/src/lib/components/wanted/WantList.svelte | 5 +- .../lib/components/wanted/WantList.test.ts | 11 + web/src/lib/styles/error-copy.json | 4 +- .../admin/integrations/integrations.test.ts | 5 +- 34 files changed, 2362 insertions(+), 106 deletions(-) create mode 100644 internal/db/dbq/lifecycle.sql.go create mode 100644 internal/db/migrations/0084_download_lifecycle.down.sql create mode 100644 internal/db/migrations/0084_download_lifecycle.up.sql create mode 100644 internal/db/queries/lifecycle.sql create mode 100644 internal/lifecycle/lifecycle.go create mode 100644 internal/lifecycle/lifecycle_test.go diff --git a/cmd/minstrel/main.go b/cmd/minstrel/main.go index 028f6a82..b8add60c 100644 --- a/cmd/minstrel/main.go +++ b/cmd/minstrel/main.go @@ -24,6 +24,7 @@ import ( "git.fabledsword.com/bvandeusen/minstrel/internal/gc" "git.fabledsword.com/bvandeusen/minstrel/internal/importer" "git.fabledsword.com/bvandeusen/minstrel/internal/library" + "git.fabledsword.com/bvandeusen/minstrel/internal/lifecycle" "git.fabledsword.com/bvandeusen/minstrel/internal/logging" "git.fabledsword.com/bvandeusen/minstrel/internal/mailer" "git.fabledsword.com/bvandeusen/minstrel/internal/notifications" @@ -341,6 +342,11 @@ func run() error { } go importer.New(pool, scanner, cat, printer, cfg.Library.ScanPaths, logger.With("component", "import")).Run(ctx) + // Download lifecycle (M502 #5523): every ten minutes, a download that + // failed or that nobody shares is retried, then given up and its album + // searched again; an imported one seeds, then is removed. + go lifecycle.New(pool, logger.With("component", "lifecycle")).Run(ctx) + // Duplicate resolver (M498, M502): classifies the sweep's groups and, with // the operator's auto-resolve on, merges the copies that are safe to merge. // Started after SetNotifier so its summaries reach admins. See diff --git a/internal/acquire/memory.go b/internal/acquire/memory.go index bf8ab8f7..9f408de8 100644 --- a/internal/acquire/memory.go +++ b/internal/acquire/memory.go @@ -21,6 +21,9 @@ const ( BlockFiles = "files" // BlockImport: the import found its files were not the album (#5522). BlockImport = "import" + // BlockStalled: given up on because it would not finish (#5523); held + // back until the block's expires_at. + BlockStalled = "stalled" ) // Grab origins, matching grabs_origin_check. @@ -92,11 +95,22 @@ func (m *Memory) blocked(ctx context.Context) map[string]string { // BlockSentence is what a blocked candidate says. func BlockSentence(reason, detail string) string { - if reason == BlockFiles { + switch reason { + case BlockFiles: if detail != "" { return "Set aside for what its files hold: " + detail } return "Set aside for what its files hold." + case BlockImport: + if detail != "" { + return "Refused on import: " + detail + } + return "Refused on import: its files were not the album." + case BlockStalled: + if detail != "" { + return "Given up on for now: " + detail + } + return "Given up on for now: it would not finish." } return "You set this release aside." } diff --git a/internal/acquire/settings.go b/internal/acquire/settings.go index a7a9c9ea..01d4b92a 100644 --- a/internal/acquire/settings.go +++ b/internal/acquire/settings.go @@ -12,8 +12,11 @@ package acquire import ( "context" + "errors" + "fmt" "time" + "github.com/jackc/pgx/v5" "github.com/jackc/pgx/v5/pgxpool" "git.fabledsword.com/bvandeusen/minstrel/internal/db/dbq" @@ -154,6 +157,19 @@ func DownloadClient(row dbq.AcquisitionService) (downloader.Driver, error) { return downloader.Mapped(d, downloader.PathMap{Local: row.LocalPath, Remote: row.RemotePath}), nil } +// DownloadClientFor builds the download client a grab went to, by the code +// of its acquisition_services row. +func DownloadClientFor(ctx context.Context, pool *pgxpool.Pool, code string) (downloader.Driver, error) { + row, err := dbq.New(pool).GetAcquisitionService(ctx, code) + if err != nil { + if errors.Is(err, pgx.ErrNoRows) { + return nil, fmt.Errorf("the download client %q is no longer set up", code) + } + return nil, err + } + return DownloadClient(row) +} + // Sources builds every enabled search source. func Sources(ctx context.Context, pool *pgxpool.Pool) ([]Source, error) { rows, err := dbq.New(pool).ListUsableAcquisitionServices(ctx, JobSearch) diff --git a/internal/api/admin_import.go b/internal/api/admin_import.go index 71675013..3535ab84 100644 --- a/internal/api/admin_import.go +++ b/internal/api/admin_import.go @@ -10,6 +10,7 @@ import ( "git.fabledsword.com/bvandeusen/minstrel/internal/apierror" "git.fabledsword.com/bvandeusen/minstrel/internal/db/dbq" "git.fabledsword.com/bvandeusen/minstrel/internal/importer" + "git.fabledsword.com/bvandeusen/minstrel/internal/lifecycle" ) // Downloads and their imports (M502 #5522): the Wanted view's Activity tab, @@ -40,6 +41,24 @@ type downloadView struct { Files []string `json:"files"` Skipped int `json:"skipped"` RehomedFrom string `json:"rehomed_from,omitempty"` + // EndDetail says why an ended download ended (#5523). + EndDetail string `json:"end_detail"` + // Seeding: imported and still sharing; Ratio its share ratio as last + // read; SeedReleasedAt when it was removed once it had shared enough. + Seeding bool `json:"seeding"` + Ratio float32 `json:"ratio"` + SeedReleasedAt *string `json:"seed_released_at"` + // Condition is what is wrong with it now, if anything. + Condition *conditionView `json:"condition"` +} + +// conditionView is an open row of the intervention ledger: what was seen, +// since when, and how many times it was retried. +type conditionView struct { + Trigger string `json:"trigger"` + Detail string `json:"detail"` + Retries int32 `json:"retries"` + Since string `json:"since"` } func downloadFrom(g dbq.ListGrabsRow) downloadView { @@ -48,6 +67,18 @@ func downloadFrom(g dbq.ListGrabsRow) downloadView { Origin: g.Origin, GrabbedAt: formatTimestamp(g.GrabbedAt), EndReason: g.EndReason, Progress: g.Progress, ImportState: g.ImportState, ImportReason: g.ImportReason, ImportedFiles: g.ImportedFiles, Destination: g.Destination, Files: []string{}, + EndDetail: g.EndDetail, Seeding: g.Seeding, Ratio: g.Ratio, + } + if g.SeedReleasedAt.Valid { + at := formatTimestamp(g.SeedReleasedAt) + v.SeedReleasedAt = &at + } + if g.Condition != nil { + v.Condition = &conditionView{Trigger: *g.Condition, Detail: deref(g.ConditionDetail), + Since: formatTimestamp(g.ConditionSince)} + if g.ConditionRetries != nil { + v.Condition.Retries = *g.ConditionRetries + } } if g.WantID.Valid { id := uuidToString(g.WantID) @@ -82,6 +113,14 @@ type downloadsPage struct { Items []downloadView `json:"items"` Limit int `json:"limit"` Offset int `json:"offset"` + // ClientsDown are the download clients not answering now (#5523). + ClientsDown []clientDownView `json:"clients_down"` +} + +type clientDownView struct { + Client string `json:"client"` + Detail string `json:"detail"` + Since string `json:"since"` } // handleListDownloads implements GET /api/admin/downloads?offset=: running @@ -97,13 +136,47 @@ func (h *handlers) handleListDownloads(w http.ResponseWriter, r *http.Request) { writeErrWithLog(w, h.logger, "admin downloads: list", apierror.Internal(err)) return } - out := downloadsPage{Items: make([]downloadView, 0, len(rows)), Limit: limit, Offset: offset} + down, err := dbq.New(h.pool).ListClientsDown(r.Context()) + if err != nil { + writeErrWithLog(w, h.logger, "admin downloads: clients down", apierror.Internal(err)) + return + } + out := downloadsPage{Items: make([]downloadView, 0, len(rows)), Limit: limit, Offset: offset, + ClientsDown: make([]clientDownView, 0, len(down))} for _, g := range rows { out.Items = append(out.Items, downloadFrom(g)) } + for _, c := range down { + out.ClientsDown = append(out.ClientsDown, clientDownView{Client: c.Client, Detail: c.Detail, + Since: formatTimestamp(c.FirstSeen)}) + } writeJSON(w, http.StatusOK, out) } +// handleAbandonDownload implements POST /api/admin/downloads/{id}/abandon: a +// running download is given up, its release set aside, the download removed +// and the album searched again. +func (h *handlers) handleAbandonDownload(w http.ResponseWriter, r *http.Request) { + admin, ok := requireUser(w, r) + if !ok { + return + } + id, ok := requireURLUUID(w, r, "id") + if !ok { + return + } + if _, err := h.lifecycle.Abandon(r.Context(), id, admin.ID); err != nil { + switch { + case errors.Is(err, pgx.ErrNoRows), errors.Is(err, lifecycle.ErrNotAbandonable): + writeErr(w, apierror.Conflict("not_running", "that download is no longer running")) + default: + writeErrWithLog(w, h.logger, "admin downloads: abandon", apierror.Internal(err)) + } + return + } + w.WriteHeader(http.StatusNoContent) +} + // handleImportAnyway implements POST /api/admin/downloads/{id}/import: a // download the check held for a person (or one that failed) is imported on // the next pass whatever the check says. @@ -160,13 +233,16 @@ func (h *handlers) writeImportErr(w http.ResponseWriter, what string, err error) } type releaseBlockView struct { - InfoHash string `json:"info_hash"` - Name string `json:"name"` - Reason string `json:"reason"` - Detail string `json:"detail"` - BlockedAt string `json:"blocked_at"` - WantTitle string `json:"want_title"` - WantArtist string `json:"want_artist"` + InfoHash string `json:"info_hash"` + Name string `json:"name"` + Reason string `json:"reason"` + Detail string `json:"detail"` + BlockedAt string `json:"blocked_at"` + // ExpiresAt is when a release given up on may be taken again; null is + // for good (#5523). + ExpiresAt *string `json:"expires_at"` + WantTitle string `json:"want_title"` + WantArtist string `json:"want_artist"` } type releaseBlocksPage struct { @@ -193,6 +269,10 @@ func (h *handlers) handleListReleaseBlocks(w http.ResponseWriter, r *http.Reques for _, b := range rows { v := releaseBlockView{InfoHash: b.InfoHash, Name: b.Name, Reason: b.Reason, Detail: b.Detail, BlockedAt: formatTimestamp(b.BlockedAt)} + if b.ExpiresAt.Valid { + at := formatTimestamp(b.ExpiresAt) + v.ExpiresAt = &at + } if b.WantTitle != nil { v.WantTitle = *b.WantTitle } diff --git a/internal/api/admin_search.go b/internal/api/admin_search.go index 84a17101..35540523 100644 --- a/internal/api/admin_search.go +++ b/internal/api/admin_search.go @@ -37,6 +37,12 @@ type searchSettingsView struct { // is the first of ScanPaths, the library's folders. ImportRoot string `json:"import_root"` ScanPaths []string `json:"scan_paths"` + // The download lifecycle (#5523): seed an imported download to this + // ratio or for this many days; give up on one nobody shares after this + // many hours. 0 is: remove once imported, no time limit, never. + SeedRatio float32 `json:"seed_ratio"` + SeedDays int32 `json:"seed_days"` + StallHours int32 `json:"stall_hours"` } func searchSettings(s dbq.AcquisitionSetting, roots []string) searchSettingsView { @@ -44,6 +50,7 @@ func searchSettings(s dbq.AcquisitionSetting, roots []string) searchSettingsView AutoGrab: s.AutoGrab, SearchIntervalHours: s.SearchIntervalHours, AllowedQualities: s.AllowedQualities, Tiers: release.Tiers, LastPass: s.LastPass, ImportRoot: s.ImportRoot, ScanPaths: roots, + SeedRatio: s.SeedRatio, SeedDays: s.SeedDays, StallHours: s.StallHours, } if v.ScanPaths == nil { v.ScanPaths = []string{} @@ -76,6 +83,9 @@ type putSearchSettingsBody struct { SearchIntervalHours int32 `json:"search_interval_hours"` AllowedQualities []string `json:"allowed_qualities"` ImportRoot string `json:"import_root"` + SeedRatio float32 `json:"seed_ratio"` + SeedDays int32 `json:"seed_days"` + StallHours int32 `json:"stall_hours"` } // handlePutSearchSettings implements PUT /api/admin/acquisition/settings. @@ -102,9 +112,15 @@ func (h *handlers) handlePutSearchSettings(w http.ResponseWriter, r *http.Reques writeErr(w, apierror.BadRequest("bad_import_root", "file new music into one of the library's folders")) return } + if body.SeedRatio < 0 || body.SeedRatio > 100 || body.SeedDays < 0 || body.SeedDays > 365 || + body.StallHours < 0 || body.StallHours > 720 { + writeErr(w, apierror.BadRequest("bad_lifecycle", + "seed ratio must be 0 to 100, seed days 0 to 365 and stall hours 0 to 720")) + return + } s, err := dbq.New(h.pool).UpdateAcquisitionSettings(r.Context(), dbq.UpdateAcquisitionSettingsParams{ AutoGrab: body.AutoGrab, SearchIntervalHours: body.SearchIntervalHours, AllowedQualities: body.AllowedQualities, - ImportRoot: body.ImportRoot, + ImportRoot: body.ImportRoot, SeedRatio: body.SeedRatio, SeedDays: body.SeedDays, StallHours: body.StallHours, }) if err != nil { writeErrWithLog(w, h.logger, "admin search settings: put", apierror.Internal(err)) diff --git a/internal/api/admin_wanted.go b/internal/api/admin_wanted.go index d5cc061f..8dec9100 100644 --- a/internal/api/admin_wanted.go +++ b/internal/api/admin_wanted.go @@ -62,6 +62,7 @@ type wantGrabView struct { Progress float32 `json:"progress"` ImportState string `json:"import_state"` ImportReason string `json:"import_reason"` + EndDetail string `json:"end_detail"` ImportedAt *string `json:"imported_at"` ImportedFiles int32 `json:"imported_files"` } @@ -164,7 +165,8 @@ func (h *handlers) handleListWants(w http.ResponseWriter, r *http.Request) { v.Grab = &wantGrabView{ID: uuidToString(row.GrabID), Name: *row.GrabName, GrabbedAt: formatTimestamp(row.GrabbedAt), Live: !row.GrabEndedAt.Valid, Origin: deref(row.GrabOrigin), Quality: deref(row.GrabQuality), EndReason: deref(row.GrabEndReason), - ImportState: deref(row.GrabImportState), ImportReason: deref(row.GrabImportReason)} + ImportState: deref(row.GrabImportState), ImportReason: deref(row.GrabImportReason), + EndDetail: deref(row.GrabEndDetail)} if row.GrabProgress != nil { v.Grab.Progress = *row.GrabProgress } diff --git a/internal/api/api.go b/internal/api/api.go index fc4d6133..1b7a0225 100644 --- a/internal/api/api.go +++ b/internal/api/api.go @@ -23,6 +23,7 @@ import ( "git.fabledsword.com/bvandeusen/minstrel/internal/library" "git.fabledsword.com/bvandeusen/minstrel/internal/lidarrconfig" "git.fabledsword.com/bvandeusen/minstrel/internal/lidarrquarantine" + "git.fabledsword.com/bvandeusen/minstrel/internal/lifecycle" "git.fabledsword.com/bvandeusen/minstrel/internal/mailer" "git.fabledsword.com/bvandeusen/minstrel/internal/netsettings" "git.fabledsword.com/bvandeusen/minstrel/internal/notifications" @@ -80,6 +81,7 @@ func Mount(r chi.Router, pool *pgxpool.Pool, logger *slog.Logger, events *playev wanted: wanted.New(pool, cat, logger), searcher: wanted.NewSearcher(pool, cat, logger), importer: importer.New(pool, scanner, cat, nil, scannerRoots(scanner), logger), + lifecycle: lifecycle.New(pool, logger), catalogueSearches: newSearchCache(), catalogueArt: catalogue.NewArt(dataDir, artSources(coverSettings)), leveled: newLeveledRenderer(dataDir, loudSettings, logger), @@ -264,6 +266,7 @@ func Mount(r chi.Router, pool *pgxpool.Pool, logger *slog.Logger, events *playev admin.Post("/downloads/{id}/import", h.handleImportAnyway) admin.Post("/downloads/{id}/retry", h.handleRetryImport) admin.Post("/downloads/{id}/refuse", h.handleRefuseDownload) + admin.Post("/downloads/{id}/abandon", h.handleAbandonDownload) admin.Get("/follows", h.handleListFollows) admin.Get("/follows/{mbid}", h.handleGetFollow) admin.Put("/follows/{mbid}", h.handlePutFollow) @@ -437,6 +440,9 @@ type handlers struct { // importer refuses a download at a person's word (#5522). The import // loop itself runs in main; this one only acts on a request. importer *importer.Importer + // lifecycle gives up on a download at a person's word (#5523); its loop + // runs in main. + lifecycle *lifecycle.Service // catalogueSearches reuses recent MusicBrainz answers; nil asks every // time, which is what handler tests get. catalogueSearches *searchCache diff --git a/internal/db/dbq/imports.sql.go b/internal/db/dbq/imports.sql.go index df96ed4d..1d1bb0c6 100644 --- a/internal/db/dbq/imports.sql.go +++ b/internal/db/dbq/imports.sql.go @@ -16,7 +16,7 @@ UPDATE grabs SET import_state = 'importing', import_attempts = import_attempts + 1, import_updated_at = now(), progress = 1 WHERE id = $1 AND ended_at IS NULL AND import_state IN ('', 'downloading', 'waiting') -RETURNING id, want_id, info_hash, name, key, indexer, source, client, save_path, quality, origin, forced, grabbed_by, grabbed_at, ended_at, end_reason, end_detail, progress, import_state, import_reason, import_detail, import_forced, import_attempts, import_updated_at, imported_at, imported_files, destination +RETURNING id, want_id, info_hash, name, key, indexer, source, client, save_path, quality, origin, forced, grabbed_by, grabbed_at, ended_at, end_reason, end_detail, progress, import_state, import_reason, import_detail, import_forced, import_attempts, import_updated_at, imported_at, imported_files, destination, seeding, ratio, seed_released_at ` // Takes a finished download for import. The update is the lock: a second @@ -52,6 +52,9 @@ func (q *Queries) ClaimGrabImport(ctx context.Context, id pgtype.UUID) (Grab, er &i.ImportedAt, &i.ImportedFiles, &i.Destination, + &i.Seeding, + &i.Ratio, + &i.SeedReleasedAt, ) return i, err } @@ -60,7 +63,7 @@ const forceGrabImport = `-- name: ForceGrabImport :one UPDATE grabs SET import_state = '', import_forced = true, import_reason = '', import_updated_at = now() WHERE id = $1 AND ended_at IS NULL AND import_state IN ('review', 'failed') -RETURNING id, want_id, info_hash, name, key, indexer, source, client, save_path, quality, origin, forced, grabbed_by, grabbed_at, ended_at, end_reason, end_detail, progress, import_state, import_reason, import_detail, import_forced, import_attempts, import_updated_at, imported_at, imported_files, destination +RETURNING id, want_id, info_hash, name, key, indexer, source, client, save_path, quality, origin, forced, grabbed_by, grabbed_at, ended_at, end_reason, end_detail, progress, import_state, import_reason, import_detail, import_forced, import_attempts, import_updated_at, imported_at, imported_files, destination, seeding, ratio, seed_released_at ` // "Import anyway": the next pass imports the download whatever the check said. @@ -95,12 +98,15 @@ func (q *Queries) ForceGrabImport(ctx context.Context, id pgtype.UUID) (Grab, er &i.ImportedAt, &i.ImportedFiles, &i.Destination, + &i.Seeding, + &i.Ratio, + &i.SeedReleasedAt, ) return i, err } const getGrab = `-- name: GetGrab :one -SELECT id, want_id, info_hash, name, key, indexer, source, client, save_path, quality, origin, forced, grabbed_by, grabbed_at, ended_at, end_reason, end_detail, progress, import_state, import_reason, import_detail, import_forced, import_attempts, import_updated_at, imported_at, imported_files, destination FROM grabs WHERE id = $1 +SELECT id, want_id, info_hash, name, key, indexer, source, client, save_path, quality, origin, forced, grabbed_by, grabbed_at, ended_at, end_reason, end_detail, progress, import_state, import_reason, import_detail, import_forced, import_attempts, import_updated_at, imported_at, imported_files, destination, seeding, ratio, seed_released_at FROM grabs WHERE id = $1 ` func (q *Queries) GetGrab(ctx context.Context, id pgtype.UUID) (Grab, error) { @@ -134,6 +140,9 @@ func (q *Queries) GetGrab(ctx context.Context, id pgtype.UUID) (Grab, error) { &i.ImportedAt, &i.ImportedFiles, &i.Destination, + &i.Seeding, + &i.Ratio, + &i.SeedReleasedAt, ) return i, err } @@ -202,9 +211,16 @@ const listGrabs = `-- name: ListGrabs :many SELECT g.id, g.want_id, g.name, g.indexer, g.client, g.quality, g.origin, g.forced, g.grabbed_at, g.ended_at, g.end_reason, g.end_detail, g.progress, g.import_state, g.import_reason, g.import_detail, g.import_updated_at, g.imported_at, g.imported_files, - g.destination, w.title AS want_title, w.artist_name AS want_artist + g.destination, g.seeding, g.ratio, g.seed_released_at, + w.title AS want_title, w.artist_name AS want_artist, + di.trigger AS condition, di.detail AS condition_detail, di.remediations AS condition_retries, + di.first_seen AS condition_since FROM grabs g LEFT JOIN wants w ON w.id = g.want_id + LEFT JOIN download_interventions di + ON di.id = (SELECT x.id FROM download_interventions x + WHERE x.grab_id = g.id AND x.outcome = 'open' + ORDER BY x.last_seen DESC LIMIT 1) ORDER BY (g.ended_at IS NULL AND g.import_state IN ('review', 'failed')) DESC, g.ended_at IS NULL DESC, g.grabbed_at DESC, g.id LIMIT $2 OFFSET $1 @@ -216,32 +232,40 @@ type ListGrabsParams struct { } type ListGrabsRow struct { - ID pgtype.UUID - WantID pgtype.UUID - Name string - Indexer string - Client string - Quality string - Origin string - Forced bool - GrabbedAt pgtype.Timestamptz - EndedAt pgtype.Timestamptz - EndReason string - EndDetail string - Progress float32 - ImportState string - ImportReason string - ImportDetail []byte - ImportUpdatedAt pgtype.Timestamptz - ImportedAt pgtype.Timestamptz - ImportedFiles int32 - Destination string - WantTitle *string - WantArtist *string + ID pgtype.UUID + WantID pgtype.UUID + Name string + Indexer string + Client string + Quality string + Origin string + Forced bool + GrabbedAt pgtype.Timestamptz + EndedAt pgtype.Timestamptz + EndReason string + EndDetail string + Progress float32 + ImportState string + ImportReason string + ImportDetail []byte + ImportUpdatedAt pgtype.Timestamptz + ImportedAt pgtype.Timestamptz + ImportedFiles int32 + Destination string + Seeding bool + Ratio float32 + SeedReleasedAt pgtype.Timestamptz + WantTitle *string + WantArtist *string + Condition *string + ConditionDetail *string + ConditionRetries *int32 + ConditionSince pgtype.Timestamptz } // The Wanted view's Activity tab: downloads waiting on a person first, then -// the rest still running, then the ended ones newest first. +// the rest still running, then the ended ones newest first. Each carries +// what is wrong with it now, if anything (#5523). func (q *Queries) ListGrabs(ctx context.Context, arg ListGrabsParams) ([]ListGrabsRow, error) { rows, err := q.db.Query(ctx, listGrabs, arg.Skip, arg.MaxRows) if err != nil { @@ -272,8 +296,15 @@ func (q *Queries) ListGrabs(ctx context.Context, arg ListGrabsParams) ([]ListGra &i.ImportedAt, &i.ImportedFiles, &i.Destination, + &i.Seeding, + &i.Ratio, + &i.SeedReleasedAt, &i.WantTitle, &i.WantArtist, + &i.Condition, + &i.ConditionDetail, + &i.ConditionRetries, + &i.ConditionSince, ); err != nil { return nil, err } @@ -287,7 +318,7 @@ func (q *Queries) ListGrabs(ctx context.Context, arg ListGrabsParams) ([]ListGra const listLiveGrabs = `-- name: ListLiveGrabs :many -SELECT g.id, g.want_id, g.info_hash, g.name, g.key, g.indexer, g.source, g.client, g.save_path, g.quality, g.origin, g.forced, g.grabbed_by, g.grabbed_at, g.ended_at, g.end_reason, g.end_detail, g.progress, g.import_state, g.import_reason, g.import_detail, g.import_forced, g.import_attempts, g.import_updated_at, g.imported_at, g.imported_files, g.destination, w.release_group_mbid, w.artist_mbid, w.artist_name, w.title AS want_title, +SELECT g.id, g.want_id, g.info_hash, g.name, g.key, g.indexer, g.source, g.client, g.save_path, g.quality, g.origin, g.forced, g.grabbed_by, g.grabbed_at, g.ended_at, g.end_reason, g.end_detail, g.progress, g.import_state, g.import_reason, g.import_detail, g.import_forced, g.import_attempts, g.import_updated_at, g.imported_at, g.imported_files, g.destination, g.seeding, g.ratio, g.seed_released_at, w.release_group_mbid, w.artist_mbid, w.artist_name, w.title AS want_title, w.reason AS want_reason, w.album_id AS want_album_id FROM grabs g LEFT JOIN wants w ON w.id = g.want_id @@ -323,6 +354,9 @@ type ListLiveGrabsRow struct { ImportedAt pgtype.Timestamptz ImportedFiles int32 Destination string + Seeding bool + Ratio float32 + SeedReleasedAt pgtype.Timestamptz ReleaseGroupMbid *string ArtistMbid *string ArtistName *string @@ -370,6 +404,9 @@ func (q *Queries) ListLiveGrabs(ctx context.Context) ([]ListLiveGrabsRow, error) &i.ImportedAt, &i.ImportedFiles, &i.Destination, + &i.Seeding, + &i.Ratio, + &i.SeedReleasedAt, &i.ReleaseGroupMbid, &i.ArtistMbid, &i.ArtistName, @@ -388,10 +425,11 @@ func (q *Queries) ListLiveGrabs(ctx context.Context) ([]ListLiveGrabsRow, error) } const listReleaseBlocksPage = `-- name: ListReleaseBlocksPage :many -SELECT b.info_hash, b.name, b.reason, b.detail, b.blocked_at, w.title AS want_title, +SELECT b.info_hash, b.name, b.reason, b.detail, b.blocked_at, b.expires_at, w.title AS want_title, w.artist_name AS want_artist FROM release_blocks b LEFT JOIN wants w ON w.id = b.want_id + WHERE b.expires_at IS NULL OR b.expires_at > now() ORDER BY b.blocked_at DESC, b.info_hash LIMIT $2 OFFSET $1 ` @@ -407,6 +445,7 @@ type ListReleaseBlocksPageRow struct { Reason string Detail string BlockedAt pgtype.Timestamptz + ExpiresAt pgtype.Timestamptz WantTitle *string WantArtist *string } @@ -426,6 +465,7 @@ func (q *Queries) ListReleaseBlocksPage(ctx context.Context, arg ListReleaseBloc &i.Reason, &i.Detail, &i.BlockedAt, + &i.ExpiresAt, &i.WantTitle, &i.WantArtist, ); err != nil { @@ -522,13 +562,16 @@ UPDATE grabs import_updated_at = now(), import_forced = false, imported_at = CASE WHEN $1::text = 'imported' THEN now() ELSE imported_at END, + -- The files were copied, so an imported download goes on sharing + -- until the lifecycle removes it (#5523). + seeding = $1::text = 'imported', imported_files = $4, destination = $5, ended_at = CASE WHEN $6::text <> '' THEN now() ELSE ended_at END, end_reason = $6, end_detail = CASE WHEN $6::text <> '' THEN $2::text ELSE end_detail END WHERE id = $7 -RETURNING id, want_id, info_hash, name, key, indexer, source, client, save_path, quality, origin, forced, grabbed_by, grabbed_at, ended_at, end_reason, end_detail, progress, import_state, import_reason, import_detail, import_forced, import_attempts, import_updated_at, imported_at, imported_files, destination +RETURNING id, want_id, info_hash, name, key, indexer, source, client, save_path, quality, origin, forced, grabbed_by, grabbed_at, ended_at, end_reason, end_detail, progress, import_state, import_reason, import_detail, import_forced, import_attempts, import_updated_at, imported_at, imported_files, destination, seeding, ratio, seed_released_at ` type RecordImportParams struct { @@ -582,6 +625,9 @@ func (q *Queries) RecordImport(ctx context.Context, arg RecordImportParams) (Gra &i.ImportedAt, &i.ImportedFiles, &i.Destination, + &i.Seeding, + &i.Ratio, + &i.SeedReleasedAt, ) return i, err } @@ -591,7 +637,7 @@ UPDATE grabs g SET want_id = $1 WHERE g.id = $2 AND g.ended_at IS NULL AND NOT EXISTS (SELECT 1 FROM grabs o WHERE o.want_id = $1 AND o.ended_at IS NULL) -RETURNING id, want_id, info_hash, name, key, indexer, source, client, save_path, quality, origin, forced, grabbed_by, grabbed_at, ended_at, end_reason, end_detail, progress, import_state, import_reason, import_detail, import_forced, import_attempts, import_updated_at, imported_at, imported_files, destination +RETURNING id, want_id, info_hash, name, key, indexer, source, client, save_path, quality, origin, forced, grabbed_by, grabbed_at, ended_at, end_reason, end_detail, progress, import_state, import_reason, import_detail, import_forced, import_attempts, import_updated_at, imported_at, imported_files, destination, seeding, ratio, seed_released_at ` type RehomeGrabParams struct { @@ -632,6 +678,9 @@ func (q *Queries) RehomeGrab(ctx context.Context, arg RehomeGrabParams) (Grab, e &i.ImportedAt, &i.ImportedFiles, &i.Destination, + &i.Seeding, + &i.Ratio, + &i.SeedReleasedAt, ) return i, err } @@ -640,7 +689,7 @@ const retryGrabImport = `-- name: RetryGrabImport :one UPDATE grabs SET import_state = '', import_reason = '', import_updated_at = now() WHERE id = $1 AND ended_at IS NULL AND import_state = 'failed' -RETURNING id, want_id, info_hash, name, key, indexer, source, client, save_path, quality, origin, forced, grabbed_by, grabbed_at, ended_at, end_reason, end_detail, progress, import_state, import_reason, import_detail, import_forced, import_attempts, import_updated_at, imported_at, imported_files, destination +RETURNING id, want_id, info_hash, name, key, indexer, source, client, save_path, quality, origin, forced, grabbed_by, grabbed_at, ended_at, end_reason, end_detail, progress, import_state, import_reason, import_detail, import_forced, import_attempts, import_updated_at, imported_at, imported_files, destination, seeding, ratio, seed_released_at ` func (q *Queries) RetryGrabImport(ctx context.Context, id pgtype.UUID) (Grab, error) { @@ -674,6 +723,9 @@ func (q *Queries) RetryGrabImport(ctx context.Context, id pgtype.UUID) (Grab, er &i.ImportedAt, &i.ImportedFiles, &i.Destination, + &i.Seeding, + &i.Ratio, + &i.SeedReleasedAt, ) return i, err } diff --git a/internal/db/dbq/lifecycle.sql.go b/internal/db/dbq/lifecycle.sql.go new file mode 100644 index 00000000..7e02d817 --- /dev/null +++ b/internal/db/dbq/lifecycle.sql.go @@ -0,0 +1,389 @@ +// Code generated by sqlc. DO NOT EDIT. +// versions: +// sqlc v1.31.1 +// source: lifecycle.sql + +package dbq + +import ( + "context" + + "github.com/jackc/pgx/v5/pgtype" +) + +const countRemediation = `-- name: CountRemediation :one +UPDATE download_interventions + SET remediations = remediations + 1, + action = $1, + last_seen = now(), + outcome = CASE WHEN $1::text IN ('given_up', 'removed') THEN 'resolved' ELSE outcome END, + settled_at = CASE WHEN $1::text IN ('given_up', 'removed') THEN now() ELSE settled_at END + WHERE client = $2 AND info_hash = $3 + AND trigger = $4 AND outcome = 'open' +RETURNING remediations +` + +type CountRemediationParams struct { + Action string + Client string + InfoHash string + Trigger string +} + +// Minstrel acted on an open condition. A retry climbs the ladder; giving up +// or removing ends the condition, since what it was about is gone. Written +// before the act, so a retry that happened is never uncounted. +func (q *Queries) CountRemediation(ctx context.Context, arg CountRemediationParams) (int32, error) { + row := q.db.QueryRow(ctx, countRemediation, + arg.Action, + arg.Client, + arg.InfoHash, + arg.Trigger, + ) + var remediations int32 + err := row.Scan(&remediations) + return remediations, err +} + +const deleteExpiredBlocks = `-- name: DeleteExpiredBlocks :execrows +DELETE FROM release_blocks WHERE expires_at IS NOT NULL AND expires_at <= now() +` + +// A release given up on is held back for a while; after that the search may +// take it again. +func (q *Queries) DeleteExpiredBlocks(ctx context.Context) (int64, error) { + result, err := q.db.Exec(ctx, deleteExpiredBlocks) + if err != nil { + return 0, err + } + return result.RowsAffected(), nil +} + +const giveUpGrab = `-- name: GiveUpGrab :one +UPDATE grabs + SET ended_at = now(), end_reason = $1, end_detail = $2, + import_updated_at = now() + WHERE id = $3 AND ended_at IS NULL AND import_state = ANY($4::text[]) +RETURNING id, want_id, info_hash, name, key, indexer, source, client, save_path, quality, origin, forced, grabbed_by, grabbed_at, ended_at, end_reason, end_detail, progress, import_state, import_reason, import_detail, import_forced, import_attempts, import_updated_at, imported_at, imported_files, destination, seeding, ratio, seed_released_at +` + +type GiveUpGrabParams struct { + EndReason string + EndDetail string + ID pgtype.UUID + States []string +} + +// Ends a running grab, given up on. Only from the states named: the update +// is the lock against an import that claimed the download meanwhile. +func (q *Queries) GiveUpGrab(ctx context.Context, arg GiveUpGrabParams) (Grab, error) { + row := q.db.QueryRow(ctx, giveUpGrab, + arg.EndReason, + arg.EndDetail, + arg.ID, + arg.States, + ) + var i Grab + err := row.Scan( + &i.ID, + &i.WantID, + &i.InfoHash, + &i.Name, + &i.Key, + &i.Indexer, + &i.Source, + &i.Client, + &i.SavePath, + &i.Quality, + &i.Origin, + &i.Forced, + &i.GrabbedBy, + &i.GrabbedAt, + &i.EndedAt, + &i.EndReason, + &i.EndDetail, + &i.Progress, + &i.ImportState, + &i.ImportReason, + &i.ImportDetail, + &i.ImportForced, + &i.ImportAttempts, + &i.ImportUpdatedAt, + &i.ImportedAt, + &i.ImportedFiles, + &i.Destination, + &i.Seeding, + &i.Ratio, + &i.SeedReleasedAt, + ) + return i, err +} + +const listClientsDown = `-- name: ListClientsDown :many +SELECT client, detail, first_seen FROM download_interventions + WHERE trigger = 'client_unreachable' AND outcome = 'open' + ORDER BY client +` + +type ListClientsDownRow struct { + Client string + Detail string + FirstSeen pgtype.Timestamptz +} + +// Download clients not answering now, for the Activity tab. +func (q *Queries) ListClientsDown(ctx context.Context) ([]ListClientsDownRow, error) { + rows, err := q.db.Query(ctx, listClientsDown) + if err != nil { + return nil, err + } + defer rows.Close() + var items []ListClientsDownRow + for rows.Next() { + var i ListClientsDownRow + if err := rows.Scan(&i.Client, &i.Detail, &i.FirstSeen); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const listLeftovers = `-- name: ListLeftovers :many +SELECT id, client, info_hash, grab_id, name, trigger, action, outcome, detail, observations, remediations, first_seen, last_seen, settled_at FROM download_interventions + WHERE trigger = 'not_removed' AND outcome = 'open' AND info_hash <> '' + ORDER BY first_seen +` + +// Downloads given up on that the client would not remove; each pass asks +// again. +func (q *Queries) ListLeftovers(ctx context.Context) ([]DownloadIntervention, error) { + rows, err := q.db.Query(ctx, listLeftovers) + if err != nil { + return nil, err + } + defer rows.Close() + var items []DownloadIntervention + for rows.Next() { + var i DownloadIntervention + if err := rows.Scan( + &i.ID, + &i.Client, + &i.InfoHash, + &i.GrabID, + &i.Name, + &i.Trigger, + &i.Action, + &i.Outcome, + &i.Detail, + &i.Observations, + &i.Remediations, + &i.FirstSeen, + &i.LastSeen, + &i.SettledAt, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const listSeedingGrabs = `-- name: ListSeedingGrabs :many +SELECT id, client, info_hash, name, imported_at FROM grabs + WHERE seeding AND seed_released_at IS NULL + ORDER BY imported_at, id +` + +type ListSeedingGrabsRow struct { + ID pgtype.UUID + Client string + InfoHash string + Name string + ImportedAt pgtype.Timestamptz +} + +// Imported downloads still sharing, oldest import first. +func (q *Queries) ListSeedingGrabs(ctx context.Context) ([]ListSeedingGrabsRow, error) { + rows, err := q.db.Query(ctx, listSeedingGrabs) + if err != nil { + return nil, err + } + defer rows.Close() + var items []ListSeedingGrabsRow + for rows.Next() { + var i ListSeedingGrabsRow + if err := rows.Scan( + &i.ID, + &i.Client, + &i.InfoHash, + &i.Name, + &i.ImportedAt, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const liveGrabOnHash = `-- name: LiveGrabOnHash :one +SELECT EXISTS (SELECT 1 FROM grabs WHERE info_hash = $1 AND ended_at IS NULL)::bool +` + +func (q *Queries) LiveGrabOnHash(ctx context.Context, infoHash string) (bool, error) { + row := q.db.QueryRow(ctx, liveGrabOnHash, infoHash) + var column_1 bool + err := row.Scan(&column_1) + return column_1, err +} + +const raiseIntervention = `-- name: RaiseIntervention :one + +INSERT INTO download_interventions (client, info_hash, grab_id, name, trigger, detail) +VALUES ($1, $2, $3, $4, + $5, $6) +ON CONFLICT (client, info_hash, trigger) WHERE outcome = 'open' DO UPDATE SET + observations = download_interventions.observations + 1, + last_seen = now(), + detail = EXCLUDED.detail, + grab_id = COALESCE(EXCLUDED.grab_id, download_interventions.grab_id), + name = CASE WHEN EXCLUDED.name <> '' THEN EXCLUDED.name ELSE download_interventions.name END +RETURNING id, client, info_hash, grab_id, name, trigger, action, outcome, detail, observations, remediations, first_seen, last_seen, settled_at +` + +type RaiseInterventionParams struct { + Client string + InfoHash string + GrabID pgtype.UUID + Name string + Trigger string + Detail string +} + +// The download lifecycle (M502 #5523): the intervention ledger, giving a +// download up, and seeding after import. +// A condition seen, or seen again: a second pass lands on the open row, so +// a download stuck for a week is one row that says how long. The name only +// improves, and the detail is the latest word. +func (q *Queries) RaiseIntervention(ctx context.Context, arg RaiseInterventionParams) (DownloadIntervention, error) { + row := q.db.QueryRow(ctx, raiseIntervention, + arg.Client, + arg.InfoHash, + arg.GrabID, + arg.Name, + arg.Trigger, + arg.Detail, + ) + var i DownloadIntervention + err := row.Scan( + &i.ID, + &i.Client, + &i.InfoHash, + &i.GrabID, + &i.Name, + &i.Trigger, + &i.Action, + &i.Outcome, + &i.Detail, + &i.Observations, + &i.Remediations, + &i.FirstSeen, + &i.LastSeen, + &i.SettledAt, + ) + return i, err +} + +const releaseSeed = `-- name: ReleaseSeed :exec +UPDATE grabs SET seed_released_at = now() WHERE id = $1 AND seed_released_at IS NULL +` + +// The seeding download was removed, or found gone. +func (q *Queries) ReleaseSeed(ctx context.Context, id pgtype.UUID) error { + _, err := q.db.Exec(ctx, releaseSeed, id) + return err +} + +const setGrabRatio = `-- name: SetGrabRatio :exec +UPDATE grabs SET ratio = $1 WHERE id = $2 +` + +type SetGrabRatioParams struct { + Ratio float32 + ID pgtype.UUID +} + +func (q *Queries) SetGrabRatio(ctx context.Context, arg SetGrabRatioParams) error { + _, err := q.db.Exec(ctx, setGrabRatio, arg.Ratio, arg.ID) + return err +} + +const settleEndedGrabs = `-- name: SettleEndedGrabs :execrows +UPDATE download_interventions di + SET outcome = 'resolved', settled_at = now(), last_seen = now() + FROM grabs g + WHERE g.id = di.grab_id AND g.ended_at IS NOT NULL + AND di.outcome = 'open' AND di.trigger <> 'not_removed' +` + +// What was wrong with a download whose grab has ended (imported, refused, +// given up) is over, except a download left in the client to remove. +func (q *Queries) SettleEndedGrabs(ctx context.Context) (int64, error) { + result, err := q.db.Exec(ctx, settleEndedGrabs) + if err != nil { + return 0, err + } + return result.RowsAffected(), nil +} + +const settleIntervention = `-- name: SettleIntervention :execrows +UPDATE download_interventions + SET outcome = 'resolved', settled_at = now(), last_seen = now() + WHERE client = $1 AND info_hash = $2 + AND trigger = $3 AND outcome = 'open' +` + +type SettleInterventionParams struct { + Client string + InfoHash string + Trigger string +} + +// The condition cleared. +func (q *Queries) SettleIntervention(ctx context.Context, arg SettleInterventionParams) (int64, error) { + result, err := q.db.Exec(ctx, settleIntervention, arg.Client, arg.InfoHash, arg.Trigger) + if err != nil { + return 0, err + } + return result.RowsAffected(), nil +} + +const settleInterventionsFor = `-- name: SettleInterventionsFor :execrows +UPDATE download_interventions + SET outcome = 'resolved', settled_at = now(), last_seen = now() + WHERE client = $1 AND info_hash = $2 AND outcome = 'open' +` + +type SettleInterventionsForParams struct { + Client string + InfoHash string +} + +// Everything open about one download is over: it is gone from the client. +func (q *Queries) SettleInterventionsFor(ctx context.Context, arg SettleInterventionsForParams) (int64, error) { + result, err := q.db.Exec(ctx, settleInterventionsFor, arg.Client, arg.InfoHash) + if err != nil { + return 0, err + } + return result.RowsAffected(), nil +} diff --git a/internal/db/dbq/models.go b/internal/db/dbq/models.go index c42a4152..d656ee35 100644 --- a/internal/db/dbq/models.go +++ b/internal/db/dbq/models.go @@ -226,6 +226,9 @@ type AcquisitionSetting struct { LastPass []byte UpdatedAt pgtype.Timestamptz ImportRoot string + SeedRatio float32 + SeedDays int32 + StallHours int32 } type Album struct { @@ -355,6 +358,23 @@ type DiscoverTuning struct { UpdatedAt pgtype.Timestamptz } +type DownloadIntervention struct { + ID pgtype.UUID + Client string + InfoHash string + GrabID pgtype.UUID + Name string + Trigger string + Action string + Outcome string + Detail string + Observations int32 + Remediations int32 + FirstSeen pgtype.Timestamptz + LastSeen pgtype.Timestamptz + SettledAt pgtype.Timestamptz +} + type DuplicateGroup struct { ID pgtype.UUID MemberKey string @@ -459,6 +479,9 @@ type Grab struct { ImportedAt pgtype.Timestamptz ImportedFiles int32 Destination string + Seeding bool + Ratio float32 + SeedReleasedAt pgtype.Timestamptz } type LibraryChange struct { @@ -701,6 +724,7 @@ type ReleaseBlock struct { WantID pgtype.UUID BlockedBy pgtype.UUID BlockedAt pgtype.Timestamptz + ExpiresAt pgtype.Timestamptz } type ReleaseTypeProfile struct { diff --git a/internal/db/dbq/search.sql.go b/internal/db/dbq/search.sql.go index 9ecde58f..aa30dc94 100644 --- a/internal/db/dbq/search.sql.go +++ b/internal/db/dbq/search.sql.go @@ -12,11 +12,13 @@ import ( ) const blockRelease = `-- name: BlockRelease :exec -INSERT INTO release_blocks (info_hash, name, reason, detail, want_id, blocked_by) +INSERT INTO release_blocks (info_hash, name, reason, detail, want_id, blocked_by, expires_at) VALUES ($1, $2, $3, $4, - $5, $6) + $5, $6, $7) ON CONFLICT (info_hash) DO UPDATE - SET reason = EXCLUDED.reason, detail = EXCLUDED.detail, blocked_at = now() + SET reason = EXCLUDED.reason, detail = EXCLUDED.detail, blocked_at = now(), + expires_at = EXCLUDED.expires_at, blocked_by = EXCLUDED.blocked_by + WHERE release_blocks.expires_at IS NOT NULL OR EXCLUDED.expires_at IS NULL ` type BlockReleaseParams struct { @@ -26,8 +28,12 @@ type BlockReleaseParams struct { Detail string WantID pgtype.UUID BlockedBy pgtype.UUID + ExpiresAt pgtype.Timestamptz } +// A release set aside. expires_at NULL is for good. A block for good is never +// shortened by a timed one: only a timed block, or another for good, is +// replaced. func (q *Queries) BlockRelease(ctx context.Context, arg BlockReleaseParams) error { _, err := q.db.Exec(ctx, blockRelease, arg.InfoHash, @@ -36,6 +42,7 @@ func (q *Queries) BlockRelease(ctx context.Context, arg BlockReleaseParams) erro arg.Detail, arg.WantID, arg.BlockedBy, + arg.ExpiresAt, ) return err } @@ -89,7 +96,7 @@ func (q *Queries) EndGrab(ctx context.Context, arg EndGrabParams) error { const getAcquisitionSettings = `-- name: GetAcquisitionSettings :one -SELECT id, auto_grab, search_interval_hours, allowed_qualities, last_pass_at, last_pass, updated_at, import_root FROM acquisition_settings WHERE id = true +SELECT id, auto_grab, search_interval_hours, allowed_qualities, last_pass_at, last_pass, updated_at, import_root, seed_ratio, seed_days, stall_hours FROM acquisition_settings WHERE id = true ` // Searching for wants and grabbing releases (M502 #5521). @@ -105,12 +112,15 @@ func (q *Queries) GetAcquisitionSettings(ctx context.Context) (AcquisitionSettin &i.LastPass, &i.UpdatedAt, &i.ImportRoot, + &i.SeedRatio, + &i.SeedDays, + &i.StallHours, ) return i, err } const getLiveGrab = `-- name: GetLiveGrab :one -SELECT id, want_id, info_hash, name, key, indexer, source, client, save_path, quality, origin, forced, grabbed_by, grabbed_at, ended_at, end_reason, end_detail, progress, import_state, import_reason, import_detail, import_forced, import_attempts, import_updated_at, imported_at, imported_files, destination FROM grabs WHERE want_id = $1 AND ended_at IS NULL +SELECT id, want_id, info_hash, name, key, indexer, source, client, save_path, quality, origin, forced, grabbed_by, grabbed_at, ended_at, end_reason, end_detail, progress, import_state, import_reason, import_detail, import_forced, import_attempts, import_updated_at, imported_at, imported_files, destination, seeding, ratio, seed_released_at FROM grabs WHERE want_id = $1 AND ended_at IS NULL ` func (q *Queries) GetLiveGrab(ctx context.Context, wantID pgtype.UUID) (Grab, error) { @@ -144,6 +154,9 @@ func (q *Queries) GetLiveGrab(ctx context.Context, wantID pgtype.UUID) (Grab, er &i.ImportedAt, &i.ImportedFiles, &i.Destination, + &i.Seeding, + &i.Ratio, + &i.SeedReleasedAt, ) return i, err } @@ -230,7 +243,7 @@ INSERT INTO grabs (want_id, info_hash, name, key, indexer, source, client, save_ VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12) -RETURNING id, want_id, info_hash, name, key, indexer, source, client, save_path, quality, origin, forced, grabbed_by, grabbed_at, ended_at, end_reason, end_detail, progress, import_state, import_reason, import_detail, import_forced, import_attempts, import_updated_at, imported_at, imported_files, destination +RETURNING id, want_id, info_hash, name, key, indexer, source, client, save_path, quality, origin, forced, grabbed_by, grabbed_at, ended_at, end_reason, end_detail, progress, import_state, import_reason, import_detail, import_forced, import_attempts, import_updated_at, imported_at, imported_files, destination, seeding, ratio, seed_released_at ` type InsertGrabParams struct { @@ -292,6 +305,9 @@ func (q *Queries) InsertGrab(ctx context.Context, arg InsertGrabParams) (Grab, e &i.ImportedAt, &i.ImportedFiles, &i.Destination, + &i.Seeding, + &i.Ratio, + &i.SeedReleasedAt, ) return i, err } @@ -350,9 +366,13 @@ func (q *Queries) InsertWantCandidate(ctx context.Context, arg InsertWantCandida } const listReleaseBlocks = `-- name: ListReleaseBlocks :many -SELECT info_hash, name, reason, detail, want_id, blocked_by, blocked_at FROM release_blocks ORDER BY blocked_at DESC +SELECT info_hash, name, reason, detail, want_id, blocked_by, blocked_at, expires_at FROM release_blocks + WHERE expires_at IS NULL OR expires_at > now() + ORDER BY blocked_at DESC ` +// Every release set aside and still held back: a stalled one lifts by itself +// at expires_at (#5523). func (q *Queries) ListReleaseBlocks(ctx context.Context) ([]ReleaseBlock, error) { rows, err := q.db.Query(ctx, listReleaseBlocks) if err != nil { @@ -370,6 +390,7 @@ func (q *Queries) ListReleaseBlocks(ctx context.Context) ([]ReleaseBlock, error) &i.WantID, &i.BlockedBy, &i.BlockedAt, + &i.ExpiresAt, ); err != nil { return nil, err } @@ -582,9 +603,12 @@ UPDATE acquisition_settings search_interval_hours = $2, allowed_qualities = $3, import_root = $4, + seed_ratio = $5, + seed_days = $6, + stall_hours = $7, updated_at = now() WHERE id = true -RETURNING id, auto_grab, search_interval_hours, allowed_qualities, last_pass_at, last_pass, updated_at, import_root +RETURNING id, auto_grab, search_interval_hours, allowed_qualities, last_pass_at, last_pass, updated_at, import_root, seed_ratio, seed_days, stall_hours ` type UpdateAcquisitionSettingsParams struct { @@ -592,6 +616,9 @@ type UpdateAcquisitionSettingsParams struct { SearchIntervalHours int32 AllowedQualities []string ImportRoot string + SeedRatio float32 + SeedDays int32 + StallHours int32 } func (q *Queries) UpdateAcquisitionSettings(ctx context.Context, arg UpdateAcquisitionSettingsParams) (AcquisitionSetting, error) { @@ -600,6 +627,9 @@ func (q *Queries) UpdateAcquisitionSettings(ctx context.Context, arg UpdateAcqui arg.SearchIntervalHours, arg.AllowedQualities, arg.ImportRoot, + arg.SeedRatio, + arg.SeedDays, + arg.StallHours, ) var i AcquisitionSetting err := row.Scan( @@ -611,6 +641,9 @@ func (q *Queries) UpdateAcquisitionSettings(ctx context.Context, arg UpdateAcqui &i.LastPass, &i.UpdatedAt, &i.ImportRoot, + &i.SeedRatio, + &i.SeedDays, + &i.StallHours, ) return i, err } diff --git a/internal/db/dbq/wanted.sql.go b/internal/db/dbq/wanted.sql.go index 3f467f48..7e7df749 100644 --- a/internal/db/dbq/wanted.sql.go +++ b/internal/db/dbq/wanted.sql.go @@ -30,7 +30,8 @@ SELECT COUNT(*) FILTER (WHERE fulfilled_at IS NULL AND reason <> 'missing')::big (SELECT COUNT(*) FROM grabs WHERE ended_at IS NULL)::bigint AS activity, (SELECT COUNT(*) FROM grabs WHERE ended_at IS NULL AND import_state IN ('review', 'failed'))::bigint AS attention, - (SELECT COUNT(*) FROM release_blocks)::bigint AS blocked + (SELECT COUNT(*) FROM release_blocks b + WHERE b.expires_at IS NULL OR b.expires_at > now())::bigint AS blocked FROM wants ` @@ -537,7 +538,7 @@ SELECT w.id, w.release_group_mbid, w.artist_mbid, w.artist_name, w.title, w.reas gr.id AS grab_id, gr.name AS grab_name, gr.grabbed_at, gr.origin AS grab_origin, gr.quality AS grab_quality, gr.ended_at AS grab_ended_at, gr.end_reason AS grab_end_reason, gr.progress AS grab_progress, gr.import_state AS grab_import_state, - gr.import_reason AS grab_import_reason, gr.imported_at AS grab_imported_at, + gr.import_reason AS grab_import_reason, gr.end_detail AS grab_end_detail, gr.imported_at AS grab_imported_at, gr.imported_files AS grab_imported_files FROM wants w LEFT JOIN users u ON u.id = w.created_by @@ -590,6 +591,7 @@ type ListWantsRow struct { GrabProgress *float32 GrabImportState *string GrabImportReason *string + GrabEndDetail *string GrabImportedAt pgtype.Timestamptz GrabImportedFiles *int32 } @@ -636,6 +638,7 @@ func (q *Queries) ListWants(ctx context.Context, arg ListWantsParams) ([]ListWan &i.GrabProgress, &i.GrabImportState, &i.GrabImportReason, + &i.GrabEndDetail, &i.GrabImportedAt, &i.GrabImportedFiles, ); err != nil { diff --git a/internal/db/migrations/0084_download_lifecycle.down.sql b/internal/db/migrations/0084_download_lifecycle.down.sql new file mode 100644 index 00000000..9e00a4d5 --- /dev/null +++ b/internal/db/migrations/0084_download_lifecycle.down.sql @@ -0,0 +1,23 @@ +ALTER TABLE acquisition_settings + DROP COLUMN seed_ratio, + DROP COLUMN seed_days, + DROP COLUMN stall_hours; + +ALTER TABLE release_blocks DROP COLUMN expires_at; +DELETE FROM release_blocks WHERE reason = 'stalled'; +ALTER TABLE release_blocks DROP CONSTRAINT release_blocks_reason_check; +ALTER TABLE release_blocks ADD CONSTRAINT release_blocks_reason_check + CHECK (reason IN ('operator', 'files', 'import')); + +DROP INDEX grabs_seeding_idx; +ALTER TABLE grabs + DROP COLUMN seeding, + DROP COLUMN ratio, + DROP COLUMN seed_released_at; + +UPDATE grabs SET end_reason = 'superseded' WHERE end_reason IN ('stalled', 'failed', 'abandoned', 'gone'); +ALTER TABLE grabs DROP CONSTRAINT grabs_end_reason_check; +ALTER TABLE grabs ADD CONSTRAINT grabs_end_reason_check + CHECK (end_reason IN ('', 'superseded', 'imported', 'refused', 'held')); + +DROP TABLE download_interventions; diff --git a/internal/db/migrations/0084_download_lifecycle.up.sql b/internal/db/migrations/0084_download_lifecycle.up.sql new file mode 100644 index 00000000..067bf26f --- /dev/null +++ b/internal/db/migrations/0084_download_lifecycle.up.sql @@ -0,0 +1,90 @@ +-- Download lifecycle (M502 #5523), ported from Fabled Librarian's System 104: +-- a download that stalls or fails is given up and its album searched again; +-- an imported download seeds, then is removed. +-- +-- download_interventions what was seen wrong with a download or a client, +-- what Minstrel did about it, and whether it +-- cleared. One open row per (client, download, +-- condition), updated in place. +-- grabs (columns) a grab also ends by being given up (stalled, +-- failed), set aside by a person, or leaving the +-- client; an imported one records its seeding. +-- release_blocks a release given up on is held back for a while, +-- not for ever: expires_at. +-- acquisition_settings how long to seed, and how long a download may sit +-- with nobody sharing it. +CREATE TABLE download_interventions ( + id uuid PRIMARY KEY DEFAULT gen_random_uuid(), + -- The acquisition_services code of the download client. + client text NOT NULL, + -- The torrent, or '' for a condition of the client itself. + info_hash text NOT NULL DEFAULT '', + grab_id uuid REFERENCES grabs (id) ON DELETE SET NULL, + name text NOT NULL DEFAULT '', + -- no_peers nobody is sharing an unfinished download. + -- no_metadata still fetching the torrent's file list. + -- client_error the client gave up on it, in its own words. + -- client_unreachable the client does not answer, or its engine is down. + -- gone the client no longer lists a running download. + -- not_removed given up on, and the client would not remove it. + trigger text NOT NULL CONSTRAINT download_interventions_trigger_check CHECK (trigger IN ( + 'no_peers', 'no_metadata', 'client_error', 'client_unreachable', 'gone', 'not_removed')), + -- escalated seen and shown; nothing done yet. + -- retried asked the trackers again and re-read the files. + -- given_up the download was removed and its album searched again. + -- removed a download left behind was removed. + action text NOT NULL DEFAULT 'escalated' + CONSTRAINT download_interventions_action_check CHECK (action IN ('escalated', 'retried', 'given_up', 'removed')), + outcome text NOT NULL DEFAULT 'open' + CONSTRAINT download_interventions_outcome_check CHECK (outcome IN ('open', 'resolved')), + detail text NOT NULL DEFAULT '', + -- How many passes have seen it, and how many times Minstrel acted on it. + -- The second is the retry ladder's rung, kept here so a restart does not + -- hand a stuck download fresh retries. + observations integer NOT NULL DEFAULT 1, + remediations integer NOT NULL DEFAULT 0, + first_seen timestamptz NOT NULL DEFAULT now(), + last_seen timestamptz NOT NULL DEFAULT now(), + settled_at timestamptz, + CONSTRAINT download_interventions_settled_together CHECK ((outcome = 'open') = (settled_at IS NULL)) +); + +-- One open row per condition: a pass that sees it again lands on the same row. +CREATE UNIQUE INDEX download_interventions_one_open + ON download_interventions (client, info_hash, trigger) WHERE outcome = 'open'; +CREATE INDEX download_interventions_grab_idx ON download_interventions (grab_id) WHERE grab_id IS NOT NULL; + +ALTER TABLE grabs DROP CONSTRAINT grabs_end_reason_check; +ALTER TABLE grabs ADD CONSTRAINT grabs_end_reason_check + CHECK (end_reason IN ('', 'superseded', 'imported', 'refused', 'held', 'stalled', 'failed', 'abandoned', 'gone')); + +-- An imported download goes on sharing (the files were copied) until its +-- ratio reaches the setting or its time is up; seed_released_at is when it +-- was removed, or found gone. +ALTER TABLE grabs + ADD COLUMN seeding boolean NOT NULL DEFAULT false, + ADD COLUMN ratio real NOT NULL DEFAULT 0, + ADD COLUMN seed_released_at timestamptz; + +CREATE INDEX grabs_seeding_idx ON grabs (imported_at) WHERE seeding AND seed_released_at IS NULL; + +-- stalled: given up on because it would not finish; held back until +-- expires_at. NULL is for good. +ALTER TABLE release_blocks DROP CONSTRAINT release_blocks_reason_check; +ALTER TABLE release_blocks ADD CONSTRAINT release_blocks_reason_check + CHECK (reason IN ('operator', 'files', 'import', 'stalled')); +ALTER TABLE release_blocks ADD COLUMN expires_at timestamptz; + +-- FL's rulings: seed to ratio 2.0 or for 14 days at most; a download nobody +-- has shared for 6 hours is retried, then given up. +-- seed_ratio 0 the download is removed once imported. +-- seed_days 0 no time limit: only the ratio ends it. +-- stall_hours 0 never give up on a download for want of peers (a failed +-- one is still given up). +ALTER TABLE acquisition_settings + ADD COLUMN seed_ratio real NOT NULL DEFAULT 2.0 + CONSTRAINT acquisition_settings_seed_ratio_check CHECK (seed_ratio BETWEEN 0 AND 100), + ADD COLUMN seed_days integer NOT NULL DEFAULT 14 + CONSTRAINT acquisition_settings_seed_days_check CHECK (seed_days BETWEEN 0 AND 365), + ADD COLUMN stall_hours integer NOT NULL DEFAULT 6 + CONSTRAINT acquisition_settings_stall_hours_check CHECK (stall_hours BETWEEN 0 AND 720); diff --git a/internal/db/queries/imports.sql b/internal/db/queries/imports.sql index 28bf0ef0..ffc67643 100644 --- a/internal/db/queries/imports.sql +++ b/internal/db/queries/imports.sql @@ -51,6 +51,9 @@ UPDATE grabs import_updated_at = now(), import_forced = false, imported_at = CASE WHEN sqlc.arg(import_state)::text = 'imported' THEN now() ELSE imported_at END, + -- The files were copied, so an imported download goes on sharing + -- until the lifecycle removes it (#5523). + seeding = sqlc.arg(import_state)::text = 'imported', imported_files = sqlc.arg(imported_files), destination = sqlc.arg(destination), ended_at = CASE WHEN sqlc.arg(end_reason)::text <> '' THEN now() ELSE ended_at END, @@ -110,21 +113,30 @@ SELECT DISTINCT COALESCE(t.disc_number, 1)::int AS disc, t.track_number::int AS -- name: ListGrabs :many -- The Wanted view's Activity tab: downloads waiting on a person first, then --- the rest still running, then the ended ones newest first. +-- the rest still running, then the ended ones newest first. Each carries +-- what is wrong with it now, if anything (#5523). SELECT g.id, g.want_id, g.name, g.indexer, g.client, g.quality, g.origin, g.forced, g.grabbed_at, g.ended_at, g.end_reason, g.end_detail, g.progress, g.import_state, g.import_reason, g.import_detail, g.import_updated_at, g.imported_at, g.imported_files, - g.destination, w.title AS want_title, w.artist_name AS want_artist + g.destination, g.seeding, g.ratio, g.seed_released_at, + w.title AS want_title, w.artist_name AS want_artist, + di.trigger AS condition, di.detail AS condition_detail, di.remediations AS condition_retries, + di.first_seen AS condition_since FROM grabs g LEFT JOIN wants w ON w.id = g.want_id + LEFT JOIN download_interventions di + ON di.id = (SELECT x.id FROM download_interventions x + WHERE x.grab_id = g.id AND x.outcome = 'open' + ORDER BY x.last_seen DESC LIMIT 1) ORDER BY (g.ended_at IS NULL AND g.import_state IN ('review', 'failed')) DESC, g.ended_at IS NULL DESC, g.grabbed_at DESC, g.id LIMIT sqlc.arg(max_rows) OFFSET sqlc.arg(skip); -- name: ListReleaseBlocksPage :many -SELECT b.info_hash, b.name, b.reason, b.detail, b.blocked_at, w.title AS want_title, +SELECT b.info_hash, b.name, b.reason, b.detail, b.blocked_at, b.expires_at, w.title AS want_title, w.artist_name AS want_artist FROM release_blocks b LEFT JOIN wants w ON w.id = b.want_id + WHERE b.expires_at IS NULL OR b.expires_at > now() ORDER BY b.blocked_at DESC, b.info_hash LIMIT sqlc.arg(max_rows) OFFSET sqlc.arg(skip); diff --git a/internal/db/queries/lifecycle.sql b/internal/db/queries/lifecycle.sql new file mode 100644 index 00000000..ca634f2a --- /dev/null +++ b/internal/db/queries/lifecycle.sql @@ -0,0 +1,96 @@ +-- The download lifecycle (M502 #5523): the intervention ledger, giving a +-- download up, and seeding after import. + +-- name: RaiseIntervention :one +-- A condition seen, or seen again: a second pass lands on the open row, so +-- a download stuck for a week is one row that says how long. The name only +-- improves, and the detail is the latest word. +INSERT INTO download_interventions (client, info_hash, grab_id, name, trigger, detail) +VALUES (sqlc.arg(client), sqlc.arg(info_hash), sqlc.narg(grab_id), sqlc.arg(name), + sqlc.arg(trigger), sqlc.arg(detail)) +ON CONFLICT (client, info_hash, trigger) WHERE outcome = 'open' DO UPDATE SET + observations = download_interventions.observations + 1, + last_seen = now(), + detail = EXCLUDED.detail, + grab_id = COALESCE(EXCLUDED.grab_id, download_interventions.grab_id), + name = CASE WHEN EXCLUDED.name <> '' THEN EXCLUDED.name ELSE download_interventions.name END +RETURNING *; + +-- name: SettleIntervention :execrows +-- The condition cleared. +UPDATE download_interventions + SET outcome = 'resolved', settled_at = now(), last_seen = now() + WHERE client = sqlc.arg(client) AND info_hash = sqlc.arg(info_hash) + AND trigger = sqlc.arg(trigger) AND outcome = 'open'; + +-- name: SettleInterventionsFor :execrows +-- Everything open about one download is over: it is gone from the client. +UPDATE download_interventions + SET outcome = 'resolved', settled_at = now(), last_seen = now() + WHERE client = sqlc.arg(client) AND info_hash = sqlc.arg(info_hash) AND outcome = 'open'; + +-- name: SettleEndedGrabs :execrows +-- What was wrong with a download whose grab has ended (imported, refused, +-- given up) is over, except a download left in the client to remove. +UPDATE download_interventions di + SET outcome = 'resolved', settled_at = now(), last_seen = now() + FROM grabs g + WHERE g.id = di.grab_id AND g.ended_at IS NOT NULL + AND di.outcome = 'open' AND di.trigger <> 'not_removed'; + +-- name: CountRemediation :one +-- Minstrel acted on an open condition. A retry climbs the ladder; giving up +-- or removing ends the condition, since what it was about is gone. Written +-- before the act, so a retry that happened is never uncounted. +UPDATE download_interventions + SET remediations = remediations + 1, + action = sqlc.arg(action), + last_seen = now(), + outcome = CASE WHEN sqlc.arg(action)::text IN ('given_up', 'removed') THEN 'resolved' ELSE outcome END, + settled_at = CASE WHEN sqlc.arg(action)::text IN ('given_up', 'removed') THEN now() ELSE settled_at END + WHERE client = sqlc.arg(client) AND info_hash = sqlc.arg(info_hash) + AND trigger = sqlc.arg(trigger) AND outcome = 'open' +RETURNING remediations; + +-- name: ListLeftovers :many +-- Downloads given up on that the client would not remove; each pass asks +-- again. +SELECT * FROM download_interventions + WHERE trigger = 'not_removed' AND outcome = 'open' AND info_hash <> '' + ORDER BY first_seen; + +-- name: ListClientsDown :many +-- Download clients not answering now, for the Activity tab. +SELECT client, detail, first_seen FROM download_interventions + WHERE trigger = 'client_unreachable' AND outcome = 'open' + ORDER BY client; + +-- name: LiveGrabOnHash :one +SELECT EXISTS (SELECT 1 FROM grabs WHERE info_hash = sqlc.arg(info_hash) AND ended_at IS NULL)::bool; + +-- name: GiveUpGrab :one +-- Ends a running grab, given up on. Only from the states named: the update +-- is the lock against an import that claimed the download meanwhile. +UPDATE grabs + SET ended_at = now(), end_reason = sqlc.arg(end_reason), end_detail = sqlc.arg(end_detail), + import_updated_at = now() + WHERE id = sqlc.arg(id) AND ended_at IS NULL AND import_state = ANY(sqlc.arg(states)::text[]) +RETURNING *; + +-- name: ListSeedingGrabs :many +-- Imported downloads still sharing, oldest import first. +SELECT id, client, info_hash, name, imported_at FROM grabs + WHERE seeding AND seed_released_at IS NULL + ORDER BY imported_at, id; + +-- name: SetGrabRatio :exec +UPDATE grabs SET ratio = sqlc.arg(ratio) WHERE id = sqlc.arg(id); + +-- name: ReleaseSeed :exec +-- The seeding download was removed, or found gone. +UPDATE grabs SET seed_released_at = now() WHERE id = $1 AND seed_released_at IS NULL; + +-- name: DeleteExpiredBlocks :execrows +-- A release given up on is held back for a while; after that the search may +-- take it again. +DELETE FROM release_blocks WHERE expires_at IS NOT NULL AND expires_at <= now(); diff --git a/internal/db/queries/search.sql b/internal/db/queries/search.sql index 9dc88be8..30677414 100644 --- a/internal/db/queries/search.sql +++ b/internal/db/queries/search.sql @@ -9,6 +9,9 @@ UPDATE acquisition_settings search_interval_hours = sqlc.arg(search_interval_hours), allowed_qualities = sqlc.arg(allowed_qualities), import_root = sqlc.arg(import_root), + seed_ratio = sqlc.arg(seed_ratio), + seed_days = sqlc.arg(seed_days), + stall_hours = sqlc.arg(stall_hours), updated_at = now() WHERE id = true RETURNING *; @@ -109,14 +112,23 @@ UPDATE grabs WHERE id = sqlc.arg(id) AND ended_at IS NULL; -- name: ListReleaseBlocks :many -SELECT * FROM release_blocks ORDER BY blocked_at DESC; +-- Every release set aside and still held back: a stalled one lifts by itself +-- at expires_at (#5523). +SELECT * FROM release_blocks + WHERE expires_at IS NULL OR expires_at > now() + ORDER BY blocked_at DESC; -- name: BlockRelease :exec -INSERT INTO release_blocks (info_hash, name, reason, detail, want_id, blocked_by) +-- A release set aside. expires_at NULL is for good. A block for good is never +-- shortened by a timed one: only a timed block, or another for good, is +-- replaced. +INSERT INTO release_blocks (info_hash, name, reason, detail, want_id, blocked_by, expires_at) VALUES (sqlc.arg(info_hash), sqlc.arg(name), sqlc.arg(reason), sqlc.arg(detail), - sqlc.narg(want_id), sqlc.narg(blocked_by)) + sqlc.narg(want_id), sqlc.narg(blocked_by), sqlc.narg(expires_at)) ON CONFLICT (info_hash) DO UPDATE - SET reason = EXCLUDED.reason, detail = EXCLUDED.detail, blocked_at = now(); + SET reason = EXCLUDED.reason, detail = EXCLUDED.detail, blocked_at = now(), + expires_at = EXCLUDED.expires_at, blocked_by = EXCLUDED.blocked_by + WHERE release_blocks.expires_at IS NOT NULL OR EXCLUDED.expires_at IS NULL; -- name: UnblockRelease :execrows DELETE FROM release_blocks WHERE info_hash = $1; diff --git a/internal/db/queries/wanted.sql b/internal/db/queries/wanted.sql index d52d47a6..5da52d33 100644 --- a/internal/db/queries/wanted.sql +++ b/internal/db/queries/wanted.sql @@ -194,7 +194,7 @@ SELECT w.id, w.release_group_mbid, w.artist_mbid, w.artist_name, w.title, w.reas gr.id AS grab_id, gr.name AS grab_name, gr.grabbed_at, gr.origin AS grab_origin, gr.quality AS grab_quality, gr.ended_at AS grab_ended_at, gr.end_reason AS grab_end_reason, gr.progress AS grab_progress, gr.import_state AS grab_import_state, - gr.import_reason AS grab_import_reason, gr.imported_at AS grab_imported_at, + gr.import_reason AS grab_import_reason, gr.end_detail AS grab_end_detail, gr.imported_at AS grab_imported_at, gr.imported_files AS grab_imported_files FROM wants w LEFT JOIN users u ON u.id = w.created_by @@ -219,5 +219,6 @@ SELECT COUNT(*) FILTER (WHERE fulfilled_at IS NULL AND reason <> 'missing')::big (SELECT COUNT(*) FROM grabs WHERE ended_at IS NULL)::bigint AS activity, (SELECT COUNT(*) FROM grabs WHERE ended_at IS NULL AND import_state IN ('review', 'failed'))::bigint AS attention, - (SELECT COUNT(*) FROM release_blocks)::bigint AS blocked + (SELECT COUNT(*) FROM release_blocks b + WHERE b.expires_at IS NULL OR b.expires_at > now())::bigint AS blocked FROM wants; diff --git a/internal/dbtest/reset.go b/internal/dbtest/reset.go index 8b138e24..5da0e417 100644 --- a/internal/dbtest/reset.go +++ b/internal/dbtest/reset.go @@ -109,6 +109,7 @@ var dataTables = []string{ "mb_release_groups", // M502 #5519 "mb_artists", // M502 #5519 "requests", // M502 #5520 (was lidarr_requests) + "download_interventions", // M502 #5523 "release_blocks", // M502 #5521 "grabs", // M502 #5521 "want_candidates", // M502 #5521 @@ -187,6 +188,7 @@ func ResetDB(t *testing.T, pool *pgxpool.Pool) { if _, err := pool.Exec(ctx, ` UPDATE acquisition_settings SET auto_grab = DEFAULT, search_interval_hours = DEFAULT, allowed_qualities = DEFAULT, import_root = DEFAULT, + seed_ratio = DEFAULT, seed_days = DEFAULT, stall_hours = DEFAULT, last_pass_at = NULL, last_pass = DEFAULT, updated_at = DEFAULT`, ); err != nil { t.Fatalf("dbtest.ResetDB reset acquisition settings: %v", err) diff --git a/internal/importer/importer.go b/internal/importer/importer.go index 5811f03d..1c070fcd 100644 --- a/internal/importer/importer.go +++ b/internal/importer/importer.go @@ -13,7 +13,6 @@ import ( "sync" "time" - "github.com/jackc/pgx/v5" "github.com/jackc/pgx/v5/pgtype" "github.com/jackc/pgx/v5/pgxpool" @@ -67,8 +66,8 @@ type Importer struct { // Roots are the library's folders (the scan paths). New music is filed // under the one the settings name, or the first. Roots []string - // ClientFor builds the download client a grab went to. Defaults to the - // acquisition_services row of that code. + // ClientFor builds the download client a grab went to. Defaults to + // acquire.DownloadClientFor. ClientFor func(ctx context.Context, code string) (downloader.Driver, error) logger *slog.Logger wake chan struct{} @@ -83,19 +82,10 @@ func New(pool *pgxpool.Pool, scanner Scanner, cat Catalogue, printer Printer, ro } im := &Importer{pool: pool, Scanner: scanner, Catalogue: cat, Printer: printer, Roots: roots, logger: logger, wake: make(chan struct{}, 1)} - im.ClientFor = im.clientFor - return im -} - -func (im *Importer) clientFor(ctx context.Context, code string) (downloader.Driver, error) { - row, err := dbq.New(im.pool).GetAcquisitionService(ctx, code) - if err != nil { - if errors.Is(err, pgx.ErrNoRows) { - return nil, fmt.Errorf("the download client %q is no longer set up", code) - } - return nil, err + im.ClientFor = func(ctx context.Context, code string) (downloader.Driver, error) { + return acquire.DownloadClientFor(ctx, pool, code) } - return acquire.DownloadClient(row) + return im } // Wake asks for a pass now: after a grab, or a person's decision. @@ -201,7 +191,7 @@ func (im *Importer) passClient(ctx context.Context, q *dbq.Queries, code string, } t, ok := byHash[indexer.NormalizeHash(g.InfoHash)] if !ok { - continue // gone from the client: the download lifecycle's (#5523) + continue // gone from the client: the lifecycle settles it (#5523) } if !g.WantID.Valid { wait(g, "Its album is no longer wanted.") diff --git a/internal/lifecycle/lifecycle.go b/internal/lifecycle/lifecycle.go new file mode 100644 index 00000000..79ad280a --- /dev/null +++ b/internal/lifecycle/lifecycle.go @@ -0,0 +1,659 @@ +// Package lifecycle looks after a download from the grab to its removal +// (M502 #5523), ported from Fabled Librarian's internal/health and its seed +// release (System 104). +// +// - A download the client has failed is given up at once. +// - One nobody has shared, or still fetching its file list, after the +// stall setting is retried: the trackers asked again and the files +// re-read, three times a pass apart. Then it is given up. +// - Giving up removes the download with its files, holds the release back +// for a week, ends the grab and searches for the album again. There is +// no limit per album: what stops it churning is the hold on each release. +// - A running download the client no longer lists, for half an hour, ends +// and its album is searched again. The release is not held back: nothing +// was wrong with it. +// - An imported download seeds (its files were copied) until its share +// ratio reaches the setting or its days are up, then is removed. +// +// Every condition and every act goes in the intervention ledger, one open +// row per (client, download, condition), so the Activity tab can say what +// is wrong with a download and what was done. +package lifecycle + +import ( + "context" + "errors" + "fmt" + "log/slog" + "strings" + "sync" + "time" + + "github.com/jackc/pgx/v5/pgtype" + "github.com/jackc/pgx/v5/pgxpool" + + "git.fabledsword.com/bvandeusen/minstrel/internal/acquire" + "git.fabledsword.com/bvandeusen/minstrel/internal/db/dbq" + "git.fabledsword.com/bvandeusen/minstrel/internal/downloader" + "git.fabledsword.com/bvandeusen/minstrel/internal/indexer" +) + +// Conditions, matching download_interventions_trigger_check. +const ( + NoPeers = "no_peers" + NoMetadata = "no_metadata" + ClientError = "client_error" + ClientUnreachable = "client_unreachable" + Gone = "gone" + NotRemoved = "not_removed" +) + +// Acts, matching download_interventions_action_check. +const ( + actRetried = "retried" + actGivenUp = "given_up" + actRemoved = "removed" +) + +// End reasons a grab is given with, matching grabs_end_reason_check. +const ( + EndStalled = "stalled" + EndFailed = "failed" + EndAbandoned = "abandoned" + EndGone = "gone" +) + +const ( + // interval: three retries a pass apart is half an hour of asking before + // a stalled download is given up. + interval = 10 * time.Minute + passTimeout = 5 * time.Minute + // Retries is how many times one stuck download is nudged first. + Retries = 3 + // StallBlock is how long a release given up on is held back. + StallBlock = 7 * 24 * time.Hour + // goneAfter: a client restarting can answer without a download for a + // while; half an hour missing is gone. + goneAfter = 30 * time.Minute +) + +// running are the import states of a grab still downloading: the lifecycle +// acts only on these, so an import under way is never pulled from under. +var running = []string{"", "downloading", "waiting"} + +// Settings are the limits, from acquisition_settings. +type Settings struct { + // SeedRatio is the share ratio an imported download seeds to; 0 removes + // it once imported. + SeedRatio float64 + // SeedTime is the longest it seeds; 0 is no limit. + SeedTime time.Duration + // Patience is how long a download may sit with nobody sharing it before + // it is retried, then given up; 0 never gives up for that. + Patience time.Duration +} + +// Defaults are FL's rulings. +var Defaults = Settings{SeedRatio: 2, SeedTime: 14 * 24 * time.Hour, Patience: 6 * time.Hour} + +// Service runs the lifecycle passes. +type Service struct { + pool *pgxpool.Pool + // ClientFor builds a download client by its code. Defaults to + // acquire.DownloadClientFor. + ClientFor func(ctx context.Context, code string) (downloader.Driver, error) + // Now is the clock; nil is time.Now. + Now func() time.Time + logger *slog.Logger + mu sync.Mutex // one pass at a time +} + +// New returns a Service. +func New(pool *pgxpool.Pool, logger *slog.Logger) *Service { + if logger == nil { + logger = slog.Default() + } + return &Service{pool: pool, logger: logger, + ClientFor: func(ctx context.Context, code string) (downloader.Driver, error) { + return acquire.DownloadClientFor(ctx, pool, code) + }} +} + +func (s *Service) now() time.Time { + if s.Now != nil { + return s.Now() + } + return time.Now() +} + +// Run passes a minute after start, then every ten minutes, until ctx ends. +// Each pass is bounded, so a client that never answers costs one pass. +func (s *Service) Run(ctx context.Context) { + first := time.NewTimer(time.Minute) + defer first.Stop() + select { + case <-ctx.Done(): + return + case <-first.C: + } + t := time.NewTicker(interval) + defer t.Stop() + for { + rep := s.Pass(ctx) + if rep.Retried+rep.GivenUp+rep.Gone+rep.Released+rep.Removed > 0 || rep.Error != "" { + s.logger.Info("lifecycle: pass", "retried", rep.Retried, "given_up", rep.GivenUp, "gone", rep.Gone, + "released", rep.Released, "removed", rep.Removed, "raised", rep.Raised, "settled", rep.Settled, + "error", rep.Error) + } + select { + case <-ctx.Done(): + return + case <-t.C: + } + } +} + +// Report is one pass. +type Report struct { + // Raised and Settled count ledger writes; the rest count acts. + Raised, Settled int + Retried, GivenUp, Gone, Released, Removed int + Error string +} + +// Pass looks at every running download, every seeding one and every one +// left behind, once. +func (s *Service) Pass(ctx context.Context) Report { + s.mu.Lock() + defer s.mu.Unlock() + ctx, cancel := context.WithTimeout(ctx, passTimeout) + defer cancel() + var rep Report + q := dbq.New(s.pool) + set := s.settings(ctx, q) + + if _, err := q.DeleteExpiredBlocks(ctx); err != nil { + s.logger.Warn("lifecycle: lift expired holds", "err", err) + } + if n, err := q.SettleEndedGrabs(ctx); err == nil { + rep.Settled += int(n) + } + live, err := q.ListLiveGrabs(ctx) + if err != nil { + rep.Error = err.Error() + return rep + } + seeds, err := q.ListSeedingGrabs(ctx) + if err != nil { + rep.Error = err.Error() + return rep + } + left, err := q.ListLeftovers(ctx) + if err != nil { + rep.Error = err.Error() + return rep + } + + work := map[string]*clientWork{} + var order []string + of := func(code string) *clientWork { + if w, ok := work[code]; ok { + return w + } + order = append(order, code) + work[code] = &clientWork{} + return work[code] + } + for _, g := range live { + if g.InfoHash != "" { + of(g.Client).live = append(of(g.Client).live, g) + } + } + for _, g := range seeds { + if g.InfoHash != "" { + of(g.Client).seeds = append(of(g.Client).seeds, g) + } + } + for _, l := range left { + of(l.Client).left = append(of(l.Client).left, l) + } + for _, code := range order { + if ctx.Err() != nil { + break + } + s.passClient(ctx, q, code, work[code], set, &rep) + } + return rep +} + +type clientWork struct { + live []dbq.ListLiveGrabsRow + seeds []dbq.ListSeedingGrabsRow + left []dbq.DownloadIntervention +} + +func (s *Service) settings(ctx context.Context, q *dbq.Queries) Settings { + row, err := q.GetAcquisitionSettings(ctx) + if err != nil { + // A blip keeps the defaults rather than stopping the pass. + return Defaults + } + return Settings{ + SeedRatio: float64(row.SeedRatio), + SeedTime: time.Duration(row.SeedDays) * 24 * time.Hour, + Patience: time.Duration(row.StallHours) * time.Hour, + } +} + +func (s *Service) passClient(ctx context.Context, q *dbq.Queries, code string, w *clientWork, set Settings, rep *Report) { + d, err := s.ClientFor(ctx, code) + if err != nil { + s.raise(ctx, q, code, "", pgtype.UUID{}, "", ClientUnreachable, firstLine(err.Error()), rep) + return + } + var hashes []string + for _, g := range w.live { + hashes = append(hashes, g.InfoHash) + } + for _, g := range w.seeds { + hashes = append(hashes, g.InfoHash) + } + for _, l := range w.left { + hashes = append(hashes, l.InfoHash) + } + ts, err := d.Torrents(ctx, hashes...) + if err != nil { + // Nothing about any one download is observed through a client that + // cannot answer. + s.raise(ctx, q, code, "", pgtype.UUID{}, "", ClientUnreachable, firstLine(err.Error()), rep) + return + } + for _, t := range ts { + if t.EngineDown != "" { + s.raise(ctx, q, code, "", pgtype.UUID{}, "", ClientUnreachable, t.EngineDown, rep) + return + } + } + s.settle(ctx, q, code, "", ClientUnreachable, rep) + + byHash := make(map[string]downloader.Torrent, len(ts)) + for _, t := range ts { + byHash[indexer.NormalizeHash(t.Hash)] = t + } + // ABSENCE IS ONLY A MEASUREMENT FROM A CLIENT THAT LISTS SOMETHING. An + // engine switched off, or a store that would not read, answers with an + // empty list; then nothing is concluded from what is missing. + listed := len(ts) > 0 + if !listed { + all, err := d.Torrents(ctx) + listed = err == nil && len(all) > 0 + } + + for _, l := range w.left { + t, ok := byHash[indexer.NormalizeHash(l.InfoHash)] + s.removeAgain(ctx, q, d, l, t, ok, listed, rep) + } + for _, g := range w.live { + t, ok := byHash[indexer.NormalizeHash(g.InfoHash)] + s.judge(ctx, q, d, g, t, ok, listed, set, rep) + } + for _, g := range w.seeds { + t, ok := byHash[indexer.NormalizeHash(g.InfoHash)] + s.seed(ctx, q, d, g, t, ok, set, rep) + } +} + +// judge decides what, if anything, is wrong with one running download, and +// answers each condition either way: raised when true, settled when not. +func (s *Service) judge(ctx context.Context, q *dbq.Queries, d downloader.Driver, g dbq.ListLiveGrabsRow, + t downloader.Torrent, ok, listed bool, set Settings, rep *Report) { + if !isRunning(g.ImportState) { + return // the importer's, or a person's + } + if !ok { + if !listed { + return + } + row, raised := s.raise(ctx, q, g.Client, g.InfoHash, g.ID, g.Name, Gone, + "The download client no longer lists it.", rep) + if raised && s.now().Sub(row.FirstSeen.Time) >= goneAfter { + s.endGone(ctx, q, g, rep) + } + return + } + s.settle(ctx, q, g.Client, g.InfoHash, Gone, rep) + + if t.Done() { + // Finished: whatever was wrong while it downloaded is over, and the + // import is the importer's. + for _, c := range []string{ClientError, NoMetadata, NoPeers} { + s.settle(ctx, q, g.Client, g.InfoHash, c, rep) + } + return + } + + // The client's own verdict, given up on at once: it has already made + // the decision a retry would ask about (FL #4749). + if t.Failed() { + why := t.FailReason + if why == "" { + why = fmt.Sprintf("the download client reports it %q", t.State) + } + if _, raised := s.raise(ctx, q, g.Client, g.InfoHash, g.ID, g.Name, ClientError, why, rep); raised { + s.giveUp(ctx, q, d, g, ClientError, EndFailed, why, rep) + } + return + } + s.settle(ctx, q, g.Client, g.InfoHash, ClientError, rep) + + // Below here the stall setting applies, measured from the grab. With it + // off nothing is raised and what was raised clears. + age := s.now().Sub(g.GrabbedAt.Time) + patient := set.Patience > 0 && age > set.Patience + + if patient && t.Fetching() { + row, raised := s.raise(ctx, q, g.Client, g.InfoHash, g.ID, g.Name, NoMetadata, + fmt.Sprintf("Still fetching its file list after %s.", span(age)), rep) + if raised { + s.ladder(ctx, q, d, g, NoMetadata, row.Remediations, age, rep) + } + } else { + s.settle(ctx, q, g.Client, g.InfoHash, NoMetadata, rep) + } + // Not while fetching: a torrent with no file list has no swarm to be + // alone in. + if patient && !t.Fetching() && t.Alone() { + row, raised := s.raise(ctx, q, g.Client, g.InfoHash, g.ID, g.Name, NoPeers, + fmt.Sprintf("Nobody has shared it in %s; %.0f%% downloaded.", span(age), t.Progress*100), rep) + if raised { + s.ladder(ctx, q, d, g, NoPeers, row.Remediations, age, rep) + } + } else { + s.settle(ctx, q, g.Client, g.InfoHash, NoPeers, rep) + } +} + +// ladder retries a stuck download until the retries are spent, then gives +// it up. The rung is the ledger row's, so a restart hands out no more. +func (s *Service) ladder(ctx context.Context, q *dbq.Queries, d downloader.Driver, g dbq.ListLiveGrabsRow, + trigger string, done int32, age time.Duration, rep *Report) { + if int(done) < Retries { + // Counted before it happens: a retry that happened and was not + // counted is a bound that never binds. + n, err := q.CountRemediation(ctx, dbq.CountRemediationParams{ + Client: g.Client, InfoHash: g.InfoHash, Trigger: trigger, Action: actRetried}) + if err != nil { + return + } + // Exchange schedules its own announces and has no recheck: for it a + // retry is the time between passes, and the ladder still climbs. + if err := d.Reannounce(ctx, g.InfoHash); err != nil && !errors.Is(err, downloader.ErrUnsupported) { + s.logger.Warn("lifecycle: reannounce", "grab", g.Name, "err", err) + } + if err := d.Recheck(ctx, g.InfoHash); err != nil && !errors.Is(err, downloader.ErrUnsupported) { + s.logger.Warn("lifecycle: recheck", "grab", g.Name, "err", err) + } + s.logger.Info("lifecycle: retried a stalled download", "grab", g.Name, "trigger", trigger, "attempt", n) + rep.Retried++ + return + } + why := fmt.Sprintf("taken %s ago and never finished: %s", span(age), reasonFor(trigger)) + s.giveUp(ctx, q, d, g, trigger, EndStalled, why, rep) +} + +// giveUp ends the grab, holds the release back for StallBlock, removes the +// download with its files and searches for the album again. +func (s *Service) giveUp(ctx context.Context, q *dbq.Queries, d downloader.Driver, g dbq.ListLiveGrabsRow, + trigger, end, why string, rep *Report) { + until := s.now().Add(StallBlock) + detail := capital(why) + fmt.Sprintf(". Held back until %s; the album is searched for again.", until.Format("2 Jan")) + // The update is the lock: an import that claimed the download meanwhile + // keeps it. + if _, err := q.GiveUpGrab(ctx, dbq.GiveUpGrabParams{ID: g.ID, EndReason: end, EndDetail: detail, States: running}); err != nil { + return + } + // Held back before it is removed: if the pass dies between the two, the + // release is still not taken again. + if err := acquire.NewMemory(s.pool).Block(ctx, dbq.BlockReleaseParams{ + InfoHash: g.InfoHash, Name: g.Name, Reason: acquire.BlockStalled, Detail: capital(why) + ".", + WantID: g.WantID, ExpiresAt: pgtype.Timestamptz{Time: until, Valid: true}, + }); err != nil { + s.logger.Warn("lifecycle: hold the release back", "grab", g.Name, "err", err) + } + removed := s.remove(ctx, q, d, g.Client, g.InfoHash, g.ID, g.Name, rep) + if g.WantID.Valid { + if err := q.SearchWantAgain(ctx, g.WantID); err != nil { + s.logger.Warn("lifecycle: search again", "grab", g.Name, "err", err) + } + } + if _, err := q.CountRemediation(ctx, dbq.CountRemediationParams{ + Client: g.Client, InfoHash: g.InfoHash, Trigger: trigger, Action: actGivenUp}); err != nil { + s.logger.Warn("lifecycle: stamp the ledger", "grab", g.Name, "err", err) + } + if removed { + if n, err := q.SettleInterventionsFor(ctx, dbq.SettleInterventionsForParams{ + Client: g.Client, InfoHash: g.InfoHash}); err == nil { + rep.Settled += int(n) + } + } + s.logger.Info("lifecycle: gave up on a download", "grab", g.Name, "trigger", trigger, "why", why) + rep.GivenUp++ +} + +// endGone ends a grab whose download left the client, and searches again. +// The release is not held back: nothing was found wrong with it. +func (s *Service) endGone(ctx context.Context, q *dbq.Queries, g dbq.ListLiveGrabsRow, rep *Report) { + detail := "The download client no longer lists it: removed outside Minstrel. The album is searched for again." + if _, err := q.GiveUpGrab(ctx, dbq.GiveUpGrabParams{ID: g.ID, EndReason: EndGone, EndDetail: detail, States: running}); err != nil { + return + } + if g.WantID.Valid { + if err := q.SearchWantAgain(ctx, g.WantID); err != nil { + s.logger.Warn("lifecycle: search again", "grab", g.Name, "err", err) + } + } + if _, err := q.CountRemediation(ctx, dbq.CountRemediationParams{ + Client: g.Client, InfoHash: g.InfoHash, Trigger: Gone, Action: actGivenUp}); err != nil { + s.logger.Warn("lifecycle: stamp the ledger", "grab", g.Name, "err", err) + } + if n, err := q.SettleInterventionsFor(ctx, dbq.SettleInterventionsForParams{ + Client: g.Client, InfoHash: g.InfoHash}); err == nil { + rep.Settled += int(n) + } + s.logger.Info("lifecycle: a download left its client", "grab", g.Name) + rep.Gone++ +} + +// seed removes an imported download once it has shared enough. +// +// Absence is not a verdict: a restarting client answers with a partial list. +// A seeding download missing from it is left until its time is up, then +// recorded as released; there is nothing to delete. +func (s *Service) seed(ctx context.Context, q *dbq.Queries, d downloader.Driver, g dbq.ListSeedingGrabsRow, + t downloader.Torrent, ok bool, set Settings, rep *Report) { + overdue := set.SeedTime > 0 && g.ImportedAt.Valid && s.now().Sub(g.ImportedAt.Time) >= set.SeedTime + if !ok { + if overdue { + s.release(ctx, q, g.ID, rep) + } + return + } + if err := q.SetGrabRatio(ctx, dbq.SetGrabRatioParams{ID: g.ID, Ratio: float32(t.Ratio)}); err != nil { + s.logger.Warn("lifecycle: record the ratio", "grab", g.Name, "err", err) + } + if t.Ratio < set.SeedRatio && !overdue { + return + } + s.remove(ctx, q, d, g.Client, g.InfoHash, g.ID, g.Name, rep) + s.logger.Info("lifecycle: seeded enough, removed the download", "grab", g.Name, "ratio", t.Ratio, "overdue", overdue) + s.release(ctx, q, g.ID, rep) +} + +func (s *Service) release(ctx context.Context, q *dbq.Queries, id pgtype.UUID, rep *Report) { + if err := q.ReleaseSeed(ctx, id); err != nil { + s.logger.Warn("lifecycle: record the seeding ended", "err", err) + return + } + rep.Released++ +} + +// remove deletes a download and its files. A client that refuses leaves an +// open not_removed row, which every pass tries again. +func (s *Service) remove(ctx context.Context, q *dbq.Queries, d downloader.Driver, code, hash string, + grab pgtype.UUID, name string, rep *Report) bool { + if err := d.Delete(ctx, true, hash); err != nil { + s.logger.Warn("lifecycle: remove the download", "grab", name, "err", err) + s.raise(ctx, q, code, hash, grab, name, NotRemoved, notRemoved(err), rep) + return false + } + return true +} + +// removeAgain asks the client once more to remove a download left behind, +// unless it has been taken again since. +func (s *Service) removeAgain(ctx context.Context, q *dbq.Queries, d downloader.Driver, l dbq.DownloadIntervention, + t downloader.Torrent, ok, listed bool, rep *Report) { + if !ok { + if listed { + s.settle(ctx, q, l.Client, l.InfoHash, NotRemoved, rep) + } + return + } + live, err := q.LiveGrabOnHash(ctx, l.InfoHash) + if err != nil { + return + } + if live { + s.settle(ctx, q, l.Client, l.InfoHash, NotRemoved, rep) + return + } + if err := d.Delete(ctx, true, l.InfoHash); err != nil { + s.raise(ctx, q, l.Client, l.InfoHash, l.GrabID, t.Name, NotRemoved, notRemoved(err), rep) + return + } + if _, err := q.CountRemediation(ctx, dbq.CountRemediationParams{ + Client: l.Client, InfoHash: l.InfoHash, Trigger: NotRemoved, Action: actRemoved}); err != nil { + s.logger.Warn("lifecycle: stamp the ledger", "hash", l.InfoHash, "err", err) + } + s.logger.Info("lifecycle: removed a download left behind", "name", l.Name) + rep.Removed++ +} + +// raise records a condition or agrees with the open row that says it. The +// bool is false when the write failed: then nothing is acted on, since +// without the row's count there is no bound. +func (s *Service) raise(ctx context.Context, q *dbq.Queries, code, hash string, grab pgtype.UUID, name, trigger, + detail string, rep *Report) (dbq.DownloadIntervention, bool) { + row, err := q.RaiseIntervention(ctx, dbq.RaiseInterventionParams{ + Client: code, InfoHash: hash, GrabID: grab, Name: name, Trigger: trigger, Detail: detail}) + if err != nil { + s.logger.Warn("lifecycle: record", "client", code, "hash", hash, "trigger", trigger, "err", err) + return row, false + } + rep.Raised++ + return row, true +} + +func (s *Service) settle(ctx context.Context, q *dbq.Queries, code, hash, trigger string, rep *Report) { + n, err := q.SettleIntervention(ctx, dbq.SettleInterventionParams{Client: code, InfoHash: hash, Trigger: trigger}) + if err != nil { + s.logger.Warn("lifecycle: settle", "client", code, "hash", hash, "trigger", trigger, "err", err) + return + } + rep.Settled += int(n) +} + +// ErrNotAbandonable: the download is not running; an import, or a person's +// decision on one, has it. +var ErrNotAbandonable = errors.New("lifecycle: that download is not running") + +// Abandon is a person giving up on a running download: the release is set +// aside for good (their word about it), the download removed with its files, +// and the album searched for again. +func (s *Service) Abandon(ctx context.Context, grabID, by pgtype.UUID) (dbq.Grab, error) { + q := dbq.New(s.pool) + g, err := q.GetGrab(ctx, grabID) + if err != nil { + return dbq.Grab{}, err + } + if g.EndedAt.Valid || !isRunning(g.ImportState) { + return dbq.Grab{}, ErrNotAbandonable + } + const why = "Given up by you before it finished. The album is searched for again." + ended, err := q.GiveUpGrab(ctx, dbq.GiveUpGrabParams{ID: g.ID, EndReason: EndAbandoned, EndDetail: why, States: running}) + if err != nil { + return dbq.Grab{}, ErrNotAbandonable + } + if g.InfoHash != "" { + if err := acquire.NewMemory(s.pool).Block(ctx, dbq.BlockReleaseParams{ + InfoHash: g.InfoHash, Name: g.Name, Reason: acquire.BlockOperator, Detail: "Given up by you before it finished.", + WantID: g.WantID, BlockedBy: by, + }); err != nil { + s.logger.Warn("lifecycle: set the release aside", "grab", g.Name, "err", err) + } + var rep Report + if d, err := s.ClientFor(ctx, g.Client); err != nil { + s.raise(ctx, q, g.Client, g.InfoHash, g.ID, g.Name, NotRemoved, notRemoved(err), &rep) + } else if s.remove(ctx, q, d, g.Client, g.InfoHash, g.ID, g.Name, &rep) { + _, _ = q.SettleInterventionsFor(ctx, dbq.SettleInterventionsForParams{Client: g.Client, InfoHash: g.InfoHash}) + } + } + if g.WantID.Valid { + if err := q.SearchWantAgain(ctx, g.WantID); err != nil { + s.logger.Warn("lifecycle: search again", "grab", g.Name, "err", err) + } + } + return ended, nil +} + +func isRunning(state string) bool { + for _, r := range running { + if state == r { + return true + } + } + return false +} + +// reasonFor is a condition in a person's words. +func reasonFor(trigger string) string { + switch trigger { + case NoPeers: + return "nobody was sharing it" + case NoMetadata: + return "its file list never arrived" + case ClientError: + return "the download client gave up on it" + } + return "it stalled" +} + +func notRemoved(err error) string { + return "Given up on, but the download client would not remove it: " + firstLine(err.Error()) + + ". Minstrel asks again every ten minutes." +} + +// span is a duration as a person says it. +func span(d time.Duration) string { + switch { + case d < 2*time.Hour: + return fmt.Sprintf("%d minutes", int(d.Minutes())) + case d < 48*time.Hour: + return fmt.Sprintf("%d hours", int(d.Hours())) + } + return fmt.Sprintf("%d days", int(d.Hours()/24)) +} + +func capital(s string) string { + if s == "" { + return s + } + return strings.ToUpper(s[:1]) + s[1:] +} + +func firstLine(s string) string { + if i := strings.IndexByte(s, '\n'); i >= 0 { + return strings.TrimSpace(s[:i]) + } + return strings.TrimSpace(s) +} diff --git a/internal/lifecycle/lifecycle_test.go b/internal/lifecycle/lifecycle_test.go new file mode 100644 index 00000000..f5cd4c60 --- /dev/null +++ b/internal/lifecycle/lifecycle_test.go @@ -0,0 +1,441 @@ +package lifecycle + +import ( + "context" + "errors" + "io" + "log/slog" + "os" + "testing" + "time" + + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgtype" + "github.com/jackc/pgx/v5/pgxpool" + + "git.fabledsword.com/bvandeusen/minstrel/internal/acquire" + "git.fabledsword.com/bvandeusen/minstrel/internal/db" + "git.fabledsword.com/bvandeusen/minstrel/internal/db/dbq" + "git.fabledsword.com/bvandeusen/minstrel/internal/dbtest" + "git.fabledsword.com/bvandeusen/minstrel/internal/downloader" + "git.fabledsword.com/bvandeusen/minstrel/internal/wanted" +) + +func newPool(t *testing.T) *pgxpool.Pool { + t.Helper() + if testing.Short() { + t.Skip("skipping integration test in -short mode") + } + dsn := os.Getenv("MINSTREL_TEST_DATABASE_URL") + if dsn == "" { + t.Skip("MINSTREL_TEST_DATABASE_URL not set") + } + if err := db.Migrate(dsn, slog.New(slog.NewTextHandler(io.Discard, nil))); err != nil { + t.Fatalf("migrate: %v", err) + } + pool, err := pgxpool.New(context.Background(), dsn) + if err != nil { + t.Fatalf("pool: %v", err) + } + t.Cleanup(pool.Close) + dbtest.ResetDB(t, pool) + return pool +} + +const ( + hash = "0123456789abcdef0123456789abcdef01234567" + other = "fedcba9876543210fedcba9876543210fedcba98" +) + +// fakeClient lists the torrents it holds and records what it is asked. +type fakeClient struct { + downloader.Driver + held map[string]downloader.Torrent + err error + deleteErr error + deleted []string + reannounce int + recheck int +} + +func (c *fakeClient) Torrents(_ context.Context, hashes ...string) ([]downloader.Torrent, error) { + if c.err != nil { + return nil, c.err + } + var out []downloader.Torrent + if len(hashes) == 0 { + for _, t := range c.held { + out = append(out, t) + } + return out, nil + } + for _, h := range hashes { + if t, ok := c.held[h]; ok { + out = append(out, t) + } + } + return out, nil +} + +func (c *fakeClient) Delete(_ context.Context, _ bool, hashes ...string) error { + if c.deleteErr != nil { + return c.deleteErr + } + c.deleted = append(c.deleted, hashes...) + for _, h := range hashes { + delete(c.held, h) + } + return nil +} + +func (c *fakeClient) Reannounce(context.Context, ...string) error { c.reannounce++; return nil } +func (c *fakeClient) Recheck(context.Context, ...string) error { c.recheck++; return nil } + +type stack struct { + q *dbq.Queries + s *Service + client *fakeClient + want dbq.Want + grab dbq.Grab + clock time.Time +} + +// newStack wants RAM and has a grab of it running in a fake client holding +// t under hash. +func newStack(t *testing.T, tor downloader.Torrent) *stack { + t.Helper() + ctx := context.Background() + pool := newPool(t) + q := dbq.New(pool) + w, err := wanted.WantWith(ctx, q, wanted.WantParams{GroupMBID: "rg-ram", ArtistMBID: "mb-dp", ArtistName: "Daft Punk", + Title: "Random Access Memories", PrimaryType: "Album", FirstReleaseDate: "2013-05-17", Reason: wanted.ReasonRequest}) + if err != nil { + t.Fatal(err) + } + g, err := acquire.NewMemory(pool).Grabbed(ctx, dbq.InsertGrabParams{WantID: w.ID, InfoHash: hash, + Name: "Daft Punk - RAM [FLAC]", Key: hash, Client: "qbittorrent", SavePath: "/downloads", Origin: acquire.OriginSchedule}, + pgtype.UUID{}, "") + if err != nil { + t.Fatal(err) + } + if err := q.MarkWantSearched(ctx, w.ID); err != nil { + t.Fatal(err) + } + tor.Hash = hash + st := &stack{q: q, client: &fakeClient{held: map[string]downloader.Torrent{hash: tor}}, want: w, grab: g, + clock: time.Now()} + st.s = New(pool, slog.New(slog.NewTextHandler(io.Discard, nil))) + st.s.ClientFor = func(context.Context, string) (downloader.Driver, error) { return st.client, nil } + st.s.Now = func() time.Time { return st.clock } + return st +} + +func (st *stack) grabNow(t *testing.T) dbq.Grab { + t.Helper() + g, err := st.q.GetGrab(context.Background(), st.grab.ID) + if err != nil { + t.Fatal(err) + } + return g +} + +func (st *stack) searchedAgain(t *testing.T) bool { + t.Helper() + w, err := st.q.GetWantByReleaseGroup(context.Background(), "rg-ram") + if err != nil { + t.Fatal(err) + } + return !w.SearchedAt.Valid +} + +func (st *stack) ledger(t *testing.T) []dbq.DownloadIntervention { + t.Helper() + rows, err := st.s.pool.Query(context.Background(), `SELECT * FROM download_interventions ORDER BY first_seen, id`) + if err != nil { + t.Fatal(err) + } + out, err := pgx.CollectRows(rows, pgx.RowToStructByPos[dbq.DownloadIntervention]) + if err != nil { + t.Fatal(err) + } + return out +} + +func TestLifecycle_AFailedDownloadIsGivenUpAtOnce_Integration(t *testing.T) { + st := newStack(t, downloader.Torrent{Name: "Daft Punk - RAM [FLAC]", Progress: 0.4, State: "error", + FailReason: "No space left on device"}) + rep := st.s.Pass(context.Background()) + if rep.GivenUp != 1 { + t.Fatalf("pass = %+v", rep) + } + g := st.grabNow(t) + if !g.EndedAt.Valid || g.EndReason != EndFailed { + t.Fatalf("grab ended=%v reason=%q", g.EndedAt.Valid, g.EndReason) + } + if len(st.client.deleted) != 1 || !st.searchedAgain(t) { + t.Errorf("deleted %v, searched again %v", st.client.deleted, st.searchedAgain(t)) + } + blocks, err := st.q.ListReleaseBlocks(context.Background()) + if err != nil || len(blocks) != 1 || blocks[0].Reason != acquire.BlockStalled || !blocks[0].ExpiresAt.Valid { + t.Fatalf("blocks = %+v %v", blocks, err) + } + if d := blocks[0].ExpiresAt.Time.Sub(st.clock); d < StallBlock-time.Minute || d > StallBlock+time.Minute { + t.Errorf("held back for %s", d) + } + rows := st.ledger(t) + if len(rows) != 1 || rows[0].Trigger != ClientError || rows[0].Action != actGivenUp || rows[0].Outcome != "resolved" || + rows[0].Detail != "No space left on device" { + t.Errorf("ledger = %+v", rows) + } +} + +func TestLifecycle_ADownloadNobodySharesIsRetriedThenGivenUp_Integration(t *testing.T) { + st := newStack(t, downloader.Torrent{Name: "Daft Punk - RAM [FLAC]", Progress: 0.3, State: "stalledDL"}) + ctx := context.Background() + + // Within the stall setting: nothing is wrong yet. + if rep := st.s.Pass(ctx); rep.Retried+rep.GivenUp != 0 || len(st.ledger(t)) != 0 { + t.Fatalf("a young download was judged: %+v", rep) + } + + st.clock = st.clock.Add(7 * time.Hour) + for i := 1; i <= Retries; i++ { + if rep := st.s.Pass(ctx); rep.Retried != 1 || rep.GivenUp != 0 { + t.Fatalf("pass %d = %+v", i, rep) + } + } + if st.client.reannounce != Retries || st.client.recheck != Retries { + t.Errorf("reannounced %d, rechecked %d", st.client.reannounce, st.client.recheck) + } + if st.grabNow(t).EndedAt.Valid { + t.Fatal("given up before the retries ran out") + } + if rep := st.s.Pass(ctx); rep.GivenUp != 1 { + t.Fatalf("last pass = %+v", rep) + } + g := st.grabNow(t) + if g.EndReason != EndStalled || len(st.client.deleted) != 1 || !st.searchedAgain(t) { + t.Errorf("grab %q: %s; deleted %v", g.EndReason, g.EndDetail, st.client.deleted) + } + rows := st.ledger(t) + if len(rows) != 1 || rows[0].Trigger != NoPeers || rows[0].Remediations != Retries+1 || rows[0].Outcome != "resolved" { + t.Errorf("ledger = %+v", rows) + } +} + +func TestLifecycle_AStalledDownloadThatRecoversIsSettled_Integration(t *testing.T) { + st := newStack(t, downloader.Torrent{Name: "Daft Punk - RAM [FLAC]", Progress: 0.3, State: "stalledDL"}) + ctx := context.Background() + st.clock = st.clock.Add(7 * time.Hour) + st.s.Pass(ctx) + t2 := st.client.held[hash] + t2.NumSeeds = 3 + st.client.held[hash] = t2 + st.s.Pass(ctx) + rows := st.ledger(t) + if len(rows) != 1 || rows[0].Outcome != "resolved" || rows[0].Action != actRetried { + t.Fatalf("ledger = %+v", rows) + } + if st.grabNow(t).EndedAt.Valid { + t.Error("a download that found peers was given up") + } +} + +func TestLifecycle_TheStallSettingOffNeverGivesUp_Integration(t *testing.T) { + st := newStack(t, downloader.Torrent{Name: "Daft Punk - RAM [FLAC]", Progress: 0.3, State: "stalledDL"}) + ctx := context.Background() + if _, err := st.s.pool.Exec(ctx, `UPDATE acquisition_settings SET stall_hours = 0`); err != nil { + t.Fatal(err) + } + st.clock = st.clock.Add(30 * 24 * time.Hour) + for range Retries + 2 { + if rep := st.s.Pass(ctx); rep.Retried+rep.GivenUp != 0 { + t.Fatalf("pass = %+v", rep) + } + } +} + +func TestLifecycle_ADownloadGoneFromTheClientEndsAfterHalfAnHour_Integration(t *testing.T) { + st := newStack(t, downloader.Torrent{Name: "Daft Punk - RAM [FLAC]", Progress: 0.3, State: "downloading", NumSeeds: 4}) + ctx := context.Background() + delete(st.client.held, hash) + st.client.held[other] = downloader.Torrent{Hash: other, Name: "Someone else's", Progress: 1, State: "uploading"} + + if rep := st.s.Pass(ctx); rep.Gone != 0 { + t.Fatalf("ended at first sight: %+v", rep) + } + st.clock = st.clock.Add(31 * time.Minute) + if rep := st.s.Pass(ctx); rep.Gone != 1 { + t.Fatalf("pass = %+v", rep) + } + g := st.grabNow(t) + if g.EndReason != EndGone || !st.searchedAgain(t) { + t.Errorf("grab %q, searched again %v", g.EndReason, st.searchedAgain(t)) + } + if blocks, _ := st.q.ListReleaseBlocks(ctx); len(blocks) != 0 { + t.Errorf("a release nothing was wrong with was held back: %+v", blocks) + } +} + +func TestLifecycle_AnEmptyListIsNotAMeasurement_Integration(t *testing.T) { + st := newStack(t, downloader.Torrent{Name: "Daft Punk - RAM [FLAC]", Progress: 0.3, State: "downloading"}) + ctx := context.Background() + delete(st.client.held, hash) + st.clock = st.clock.Add(24 * time.Hour) + for range 3 { + if rep := st.s.Pass(ctx); rep.Gone != 0 { + t.Fatalf("pass = %+v", rep) + } + } + if len(st.ledger(t)) != 0 || st.grabNow(t).EndedAt.Valid { + t.Error("a client listing nothing was taken to have lost the download") + } +} + +func TestLifecycle_AnUnreachableClientIsRecordedAndSettled_Integration(t *testing.T) { + st := newStack(t, downloader.Torrent{Name: "Daft Punk - RAM [FLAC]", Progress: 0.3, State: "stalledDL"}) + ctx := context.Background() + st.clock = st.clock.Add(7 * time.Hour) + st.client.err = errors.New("dial tcp: connection refused") + st.s.Pass(ctx) + down, err := st.q.ListClientsDown(ctx) + if err != nil || len(down) != 1 || down[0].Client != "qbittorrent" { + t.Fatalf("clients down = %+v %v", down, err) + } + if st.client.reannounce != 0 { + t.Error("a download was judged through a client that did not answer") + } + st.client.err = nil + st.s.Pass(ctx) + if down, _ := st.q.ListClientsDown(ctx); len(down) != 0 { + t.Errorf("still down: %+v", down) + } +} + +func TestLifecycle_AnImportedDownloadSeedsThenIsRemoved_Integration(t *testing.T) { + st := newStack(t, downloader.Torrent{Name: "Daft Punk - RAM [FLAC]", Progress: 1, State: "uploading", Ratio: 0.8}) + ctx := context.Background() + if _, err := st.q.RecordImport(ctx, dbq.RecordImportParams{ID: st.grab.ID, ImportState: "imported", + ImportReason: "Imported.", ImportDetail: []byte("{}"), ImportedFiles: 3, EndReason: "imported"}); err != nil { + t.Fatal(err) + } + if !st.grabNow(t).Seeding { + t.Fatal("an import did not leave its download seeding") + } + if rep := st.s.Pass(ctx); rep.Released != 0 || len(st.client.deleted) != 0 { + t.Fatalf("removed before the ratio: %+v", rep) + } + if g := st.grabNow(t); g.Ratio < 0.79 || g.Ratio > 0.81 { + t.Errorf("ratio recorded %v", g.Ratio) + } + tor := st.client.held[hash] + tor.Ratio = 2.1 + st.client.held[hash] = tor + if rep := st.s.Pass(ctx); rep.Released != 1 || len(st.client.deleted) != 1 { + t.Fatalf("pass = %+v, deleted %v", rep, st.client.deleted) + } + if !st.grabNow(t).SeedReleasedAt.Valid { + t.Error("the release was not recorded") + } +} + +func TestLifecycle_AnImportedDownloadIsRemovedWhenItsDaysAreUp_Integration(t *testing.T) { + st := newStack(t, downloader.Torrent{Name: "Daft Punk - RAM [FLAC]", Progress: 1, State: "uploading", Ratio: 0.1}) + ctx := context.Background() + if _, err := st.q.RecordImport(ctx, dbq.RecordImportParams{ID: st.grab.ID, ImportState: "imported", + ImportReason: "Imported.", ImportDetail: []byte("{}"), ImportedFiles: 3, EndReason: "imported"}); err != nil { + t.Fatal(err) + } + st.clock = st.clock.Add(15 * 24 * time.Hour) + if rep := st.s.Pass(ctx); rep.Released != 1 || len(st.client.deleted) != 1 { + t.Fatalf("pass = %+v", rep) + } +} + +func TestLifecycle_ADownloadTheClientWouldNotRemoveIsAskedAgain_Integration(t *testing.T) { + st := newStack(t, downloader.Torrent{Name: "Daft Punk - RAM [FLAC]", Progress: 0.4, State: "error"}) + ctx := context.Background() + st.client.deleteErr = errors.New("HTTP 500") + if rep := st.s.Pass(ctx); rep.GivenUp != 1 { + t.Fatalf("pass = %+v", rep) + } + if !st.grabNow(t).EndedAt.Valid { + t.Fatal("a refused removal held the grab") + } + open := 0 + for _, r := range st.ledger(t) { + if r.Trigger == NotRemoved && r.Outcome == "open" { + open++ + } + } + if open != 1 { + t.Fatalf("ledger = %+v", st.ledger(t)) + } + st.client.deleteErr = nil + if rep := st.s.Pass(ctx); rep.Removed != 1 || len(st.client.deleted) != 1 { + t.Fatalf("pass = %+v", rep) + } + for _, r := range st.ledger(t) { + if r.Outcome == "open" { + t.Errorf("still open: %+v", r) + } + } +} + +func TestLifecycle_APersonGivingUpSetsTheReleaseAsideForGood_Integration(t *testing.T) { + st := newStack(t, downloader.Torrent{Name: "Daft Punk - RAM [FLAC]", Progress: 0.4, State: "downloading"}) + ctx := context.Background() + if _, err := st.s.Abandon(ctx, st.grab.ID, pgtype.UUID{}); err != nil { + t.Fatal(err) + } + g := st.grabNow(t) + if g.EndReason != EndAbandoned || len(st.client.deleted) != 1 || !st.searchedAgain(t) { + t.Errorf("grab %q, deleted %v", g.EndReason, st.client.deleted) + } + blocks, _ := st.q.ListReleaseBlocks(ctx) + if len(blocks) != 1 || blocks[0].Reason != acquire.BlockOperator || blocks[0].ExpiresAt.Valid { + t.Errorf("blocks = %+v", blocks) + } + if _, err := st.s.Abandon(ctx, st.grab.ID, pgtype.UUID{}); !errors.Is(err, ErrNotAbandonable) { + t.Errorf("abandoned twice: %v", err) + } +} + +func TestLifecycle_AHoldExpiresButABlockForGoodIsNeverShortened_Integration(t *testing.T) { + st := newStack(t, downloader.Torrent{Name: "x", Progress: 0.4, State: "downloading"}) + ctx := context.Background() + mem := acquire.NewMemory(st.s.pool) + past := pgtype.Timestamptz{Time: time.Now().Add(-time.Hour), Valid: true} + if err := mem.Block(ctx, dbq.BlockReleaseParams{InfoHash: other, Name: "old", Reason: acquire.BlockStalled, ExpiresAt: past}); err != nil { + t.Fatal(err) + } + if blocks, _ := st.q.ListReleaseBlocks(ctx); len(blocks) != 0 { + t.Errorf("an expired hold still holds: %+v", blocks) + } + if n, err := st.q.DeleteExpiredBlocks(ctx); err != nil || n != 1 { + t.Errorf("lifted %d %v", n, err) + } + + if err := mem.Block(ctx, dbq.BlockReleaseParams{InfoHash: hash, Name: "x", Reason: acquire.BlockOperator}); err != nil { + t.Fatal(err) + } + soon := pgtype.Timestamptz{Time: time.Now().Add(time.Hour), Valid: true} + if err := mem.Block(ctx, dbq.BlockReleaseParams{InfoHash: hash, Name: "x", Reason: acquire.BlockStalled, ExpiresAt: soon}); err != nil { + t.Fatal(err) + } + blocks, _ := st.q.ListReleaseBlocks(ctx) + if len(blocks) != 1 || blocks[0].Reason != acquire.BlockOperator || blocks[0].ExpiresAt.Valid { + t.Errorf("a block for good was shortened: %+v", blocks) + } +} + +func TestSpan(t *testing.T) { + for _, c := range []struct { + d time.Duration + want string + }{{45 * time.Minute, "45 minutes"}, {7 * time.Hour, "7 hours"}, {3 * 24 * time.Hour, "3 days"}} { + if got := span(c.d); got != c.want { + t.Errorf("span(%s) = %q, want %q", c.d, got, c.want) + } + } +} diff --git a/web/src/lib/api/types.ts b/web/src/lib/api/types.ts index c0cc5728..738e16e6 100644 --- a/web/src/lib/api/types.ts +++ b/web/src/lib/api/types.ts @@ -630,6 +630,8 @@ export type WantGrab = { progress: number; import_state: ImportState; import_reason: string; + // Why an ended download ended (#5523). + end_detail: string; imported_at: string | null; imported_files: number; }; @@ -710,6 +712,12 @@ export type SearchSettings = { // scan_paths), and the library's folders (#5522). import_root: string; scan_paths: string[]; + // The download lifecycle (#5523): an imported download seeds to this + // ratio or for this many days (0: removed once imported; no time limit), + // and one nobody shares is given up after this many hours (0: never). + seed_ratio: number; + seed_days: number; + stall_hours: number; }; export type WantCounts = { @@ -761,17 +769,42 @@ export type Download = { files: string[]; skipped: number; rehomed_from?: string; + // Why an ended download ended; for an imported one, its seeding (#5523). + end_detail: string; + seeding: boolean; + ratio: number; + seed_released_at: string | null; + // What is wrong with it now, if anything. + condition: DownloadCondition | null; }; -export type DownloadsPage = { items: Download[]; limit: number; offset: number }; +// An open condition on a download, from the lifecycle's ledger (#5523). +export type DownloadCondition = { + trigger: 'no_peers' | 'no_metadata' | 'client_error' | 'client_unreachable' | 'gone' | 'not_removed'; + detail: string; + retries: number; + since: string; +}; + +// A download client not answering now. +export type ClientDown = { client: string; detail: string; since: string }; + +export type DownloadsPage = { + items: Download[]; + limit: number; + offset: number; + clients_down: ClientDown[]; +}; // A release set aside: by a person, by its files, or by an import's check. export type ReleaseBlock = { info_hash: string; name: string; - reason: 'operator' | 'files' | 'import'; + reason: 'operator' | 'files' | 'import' | 'stalled'; detail: string; blocked_at: string; + // When a release given up on may be taken again; null is for good. + expires_at: string | null; want_title: string; want_artist: string; }; diff --git a/web/src/lib/api/wanted.ts b/web/src/lib/api/wanted.ts index f09aa9aa..147e5d28 100644 --- a/web/src/lib/api/wanted.ts +++ b/web/src/lib/api/wanted.ts @@ -160,6 +160,9 @@ export async function putSearchSettings(body: { search_interval_hours: number; allowed_qualities: string[]; import_root: string; + seed_ratio: number; + seed_days: number; + stall_hours: number; }): Promise { return api.put('/api/admin/acquisition/settings', body); } @@ -199,6 +202,12 @@ export async function refuseDownload(id: string): Promise { await api.post(`/api/admin/downloads/${id}/refuse`, {}); } +// A running download given up: the release is set aside for good, the +// download removed and the album searched for again (#5523). +export async function abandonDownload(id: string): Promise { + await api.post(`/api/admin/downloads/${id}/abandon`, {}); +} + export async function listReleaseBlocks(offset: number = 0): Promise { return api.get(`/api/admin/release-blocks?offset=${offset}`); } diff --git a/web/src/lib/components/SearchSettingsCard.svelte b/web/src/lib/components/SearchSettingsCard.svelte index f7831722..f256d50a 100644 --- a/web/src/lib/components/SearchSettingsCard.svelte +++ b/web/src/lib/components/SearchSettingsCard.svelte @@ -11,13 +11,23 @@ // for, whether the best release is taken without asking, and which // qualities are taken at all. The schedule wakes every ten minutes and // keeps to each indexer's searches-per-hour. A finished download is - // checked and filed into one of the library's folders (#5522). + // checked and filed into one of the library's folders (#5522); it then + // seeds until the ratio or the days below, and a download nobody shares + // is given up after the hours below (#5523). const client = useQueryClient(); const store = createSearchSettingsQuery(); const query = $derived($store); - type Form = { auto_grab: boolean; search_interval_hours: number; allowed: string[]; import_root: string }; + type Form = { + auto_grab: boolean; + search_interval_hours: number; + allowed: string[]; + import_root: string; + seed_ratio: number; + seed_days: number; + stall_hours: number; + }; // The form is the server's settings until edited; saved holds what a save // returned until the query catches up. @@ -32,7 +42,10 @@ auto_grab: s.auto_grab, search_interval_hours: s.search_interval_hours, allowed: [...s.allowed_qualities], - import_root: s.import_root + import_root: s.import_root, + seed_ratio: s.seed_ratio, + seed_days: s.seed_days, + stall_hours: s.stall_hours }; } @@ -55,7 +68,10 @@ auto_grab: form.auto_grab, search_interval_hours: Number(form.search_interval_hours), allowed_qualities: form.allowed, - import_root: form.import_root + import_root: form.import_root, + seed_ratio: Number(form.seed_ratio), + seed_days: Number(form.seed_days), + stall_hours: Number(form.stall_hours) }); edited = null; await client.invalidateQueries({ queryKey: qk.searchSettings() }); @@ -158,7 +174,7 @@ File new music into A download that checks out is copied here as Artist/Album (Year)/Artist - Album - NN - - Title, tagged with its MusicBrainz ids. The download itself keeps seeding. + Title, tagged with its MusicBrainz ids. edit({ seed_ratio: Number(e.currentTarget.value) })} + class="w-20 rounded border border-border bg-background px-2 py-1 text-sm text-text-primary + focus-visible:outline-solid focus-visible:outline-2 focus-visible:outline-accent" + /> + or for + edit({ seed_days: Number(e.currentTarget.value) })} + class="w-20 rounded border border-border bg-background px-2 py-1 text-sm text-text-primary + focus-visible:outline-solid focus-visible:outline-2 focus-visible:outline-accent" + /> + days + + + + +

{lastPass(query.data)}

diff --git a/web/src/lib/components/SearchSettingsCard.test.ts b/web/src/lib/components/SearchSettingsCard.test.ts index b93a0c86..b3cf0c4f 100644 --- a/web/src/lib/components/SearchSettingsCard.test.ts +++ b/web/src/lib/components/SearchSettingsCard.test.ts @@ -24,7 +24,10 @@ const settings: SearchSettings = { last_pass_at: new Date(Date.now() - 5 * 60_000).toISOString(), last_pass: { looked: 12, grabbed: 3, waiting: 2, nothing: 7, failed: 0, paced: true }, import_root: '', - scan_paths: ['/music', '/music2'] + scan_paths: ['/music', '/music2'], + seed_ratio: 2, + seed_days: 14, + stall_hours: 6 }; afterEach(() => vi.clearAllMocks()); @@ -60,7 +63,10 @@ describe('SearchSettingsCard', () => { auto_grab: false, search_interval_hours: 24, allowed_qualities: ['lossless', 'lossy_high', 'unknown'], - import_root: '' + import_root: '', + seed_ratio: 2, + seed_days: 14, + stall_hours: 6 }) ); }); @@ -82,6 +88,20 @@ describe('SearchSettingsCard', () => { expect(screen.getByText('/music')).toBeTruthy(); }); + test('seeding and giving up on a stalled download are set here', async () => { + setup(); + expect((screen.getByLabelText(/seed ratio/i) as HTMLInputElement).value).toBe('2'); + await fireEvent.input(screen.getByLabelText(/seed ratio/i), { target: { value: '1.5' } }); + await fireEvent.input(screen.getByLabelText(/seed days/i), { target: { value: '7' } }); + await fireEvent.input(screen.getByLabelText(/stall hours/i), { target: { value: '0' } }); + await fireEvent.click(screen.getByRole('button', { name: /^save$/i })); + await waitFor(() => + expect(putSearchSettings).toHaveBeenCalledWith( + expect.objectContaining({ seed_ratio: 1.5, seed_days: 7, stall_hours: 0 }) + ) + ); + }); + test('before the first pass it says so', () => { setup({ ...settings, last_pass_at: null, last_pass: {} }); expect(screen.getByTestId('last-pass').textContent).toMatch(/hasn’t run yet/); diff --git a/web/src/lib/components/wanted/BlockList.svelte b/web/src/lib/components/wanted/BlockList.svelte index cd41332a..33b80bd3 100644 --- a/web/src/lib/components/wanted/BlockList.svelte +++ b/web/src/lib/components/wanted/BlockList.svelte @@ -8,9 +8,10 @@ import ListContinuation from '#lib/components/ListContinuation.svelte'; import type { ReleaseBlock } from '#lib/api/types.js'; - // The Blocked tab (M502 #5521–#5522): releases the search will never take + // The Blocked tab (M502 #5521–#5523): releases the search will not take // again — set aside by a person, by what their files were, or by an - // import's check that found they weren't the album. + // import's check that found they weren't the album; and, for a week, + // releases given up on because they would not finish. const client = useQueryClient(); const blocksStore = createReleaseBlocksQuery(); @@ -35,8 +36,13 @@ function by(b: ReleaseBlock): string { if (b.reason === 'operator') return 'Set aside by you'; if (b.reason === 'files') return 'Set aside for its files'; + if (b.reason === 'stalled') return 'Given up on: it would not finish'; return 'Refused on import'; } + + // A release given up on is held back for a while, then may be taken again. + const until = (b: ReleaseBlock) => + b.expires_at ? ` · held back until ${new Date(b.expires_at).toLocaleDateString()}` : ''; {#if blocks.isPending} @@ -61,7 +67,7 @@ {#if b.want_title}
For {b.want_artist} – {b.want_title}
{/if} -
{by(b)} {relativeTime(b.blocked_at)}
+
{by(b)} {relativeTime(b.blocked_at)}{until(b)}
{#if b.detail}

{b.detail}

{/if} diff --git a/web/src/lib/components/wanted/BlockList.test.ts b/web/src/lib/components/wanted/BlockList.test.ts index bd3574c3..ae4817e0 100644 --- a/web/src/lib/components/wanted/BlockList.test.ts +++ b/web/src/lib/components/wanted/BlockList.test.ts @@ -18,6 +18,7 @@ const block: ReleaseBlock = { reason: 'import', detail: 'The album tag says another version.', blocked_at: new Date(Date.now() - 3_600_000).toISOString(), + expires_at: null, want_title: 'Random Access Memories', want_artist: 'Daft Punk' }; @@ -43,6 +44,12 @@ describe('BlockList', () => { await waitFor(() => expect(unblockRelease).toHaveBeenCalledWith('abc123')); }); + test('a release given up on says until when it is held back', () => { + setup([{ ...block, reason: 'stalled', detail: 'Nobody was sharing it.', expires_at: '2026-10-17T12:00:00Z' }]); + const line = screen.getByText(/given up on: it would not finish/i).textContent ?? ''; + expect(line).toMatch(/held back until/); + }); + test('with nothing blocked it says so', () => { setup([]); expect(screen.getByText(/no releases are blocked/i)).toBeTruthy(); diff --git a/web/src/lib/components/wanted/DownloadList.svelte b/web/src/lib/components/wanted/DownloadList.svelte index 183babbb..7d6d0150 100644 --- a/web/src/lib/components/wanted/DownloadList.svelte +++ b/web/src/lib/components/wanted/DownloadList.svelte @@ -1,23 +1,32 @@ +{#each clientsDown as c (c.client)} +

+ Can't reach the download client {c.client} since {relativeTime(c.since)}: {c.detail}. Its + downloads are left as they are until it answers. +

+{/each} + {#if downloads.isPending}

Reading the downloads…

{:else if downloads.isError && !downloads.isFetchNextPageError} @@ -147,6 +203,12 @@
{/if} + {#if running(d) && d.condition} +

{condition(d.condition)}

+ {/if} + {#if ending(d)} +

{ending(d)}

+ {/if} {#if d.import_reason}

{d.import_reason} @@ -158,12 +220,21 @@ {#if filed(d)}

{filed(d)}

{/if} + {#if seeded(d)} +

{seeded(d)}

+ {/if} {#if d.rehomed_from}

Taken for {d.rehomed_from}; it turned out to be this album.

{/if}

{when(d)}

- {#if needsYou(d)} + {#if running(d)} +
+ +
+ {:else if needsYou(d)}
{#if d.import_state === 'review'}
+ {#if w.held_at} + + {/if} {#if !w.held_at}
+ {#if showUnmatched} +
    + {#each library.unmatched as u (u.album_id)} +
  • + {u.title} + · {u.artist} +
    {u.detail}
    +
    • + {/each} +
+ {#if library.unmatched.length < library.unlisted} +

+ The first {library.unmatched.length} of {library.unlisted}. +

+ {/if} + {/if} + {/if} +
{/if} {/if} diff --git a/web/src/lib/components/wanted/WantList.test.ts b/web/src/lib/components/wanted/WantList.test.ts index b4e5b8a3..e7d7c283 100644 --- a/web/src/lib/components/wanted/WantList.test.ts +++ b/web/src/lib/components/wanted/WantList.test.ts @@ -1,39 +1,25 @@ -import { afterEach, describe, expect, test, vi } from "vitest"; -import { fireEvent, render, screen, waitFor } from "@testing-library/svelte"; -import { mockInfiniteQuery, mockQuery } from "../../../test-utils/query"; -import type { - LibraryListing, - Want, - WantGrab, - WantsPage, -} from "#lib/api/types.js"; +import { afterEach, describe, expect, test, vi } from 'vitest'; +import { fireEvent, render, screen, waitFor } from '@testing-library/svelte'; +import { mockInfiniteQuery, mockQuery } from '../../../test-utils/query'; +import type { LibraryListing, Want, WantGrab, WantsPage } from '#lib/api/types.js'; -vi.mock("#lib/api/wanted.js", () => ({ +vi.mock('#lib/api/wanted.js', () => ({ createWantsQuery: vi.fn(), removeWant: vi.fn().mockResolvedValue(undefined), searchWant: vi.fn().mockResolvedValue({ - outcome: "nothing", - considered: 0, - best_release: "", - best_verdict: "", - narrowed: [], + outcome: 'nothing', considered: 0, best_release: '', best_verdict: '', narrowed: [] }), createCandidatesQuery: vi.fn(), grabCandidate: vi.fn(), blockCandidate: vi.fn(), - unblockRelease: vi.fn(), + unblockRelease: vi.fn() })); -vi.mock("#lib/stores/toast.svelte.js", () => ({ pushToast: vi.fn() })); +vi.mock('#lib/stores/toast.svelte.js', () => ({ pushToast: vi.fn() })); -import WantList from "./WantList.svelte"; -import { pushToast } from "#lib/stores/toast.svelte.js"; -import { - createCandidatesQuery, - createWantsQuery, - removeWant, - searchWant, -} from "#lib/api/wanted.js"; -import type { WantState } from "#lib/api/types.js"; +import WantList from './WantList.svelte'; +import { pushToast } from '#lib/stores/toast.svelte.js'; +import { createCandidatesQuery, createWantsQuery, removeWant, searchWant } from '#lib/api/wanted.js'; +import type { WantState } from '#lib/api/types.js'; const DAY = 24 * 3_600_000; const ago = (ms: number) => new Date(Date.now() - ms).toISOString(); @@ -42,305 +28,203 @@ function want(id: string, extra: Partial = {}): Want { return { id, release_group_mbid: `rg-${id}`, - title: "The Ballad of Darren", - artist_mbid: "blur", - artist_name: "Blur", - artist_id: "art-1", - primary_type: "Album", + title: 'The Ballad of Darren', + artist_mbid: 'blur', + artist_name: 'Blur', + artist_id: 'art-1', + primary_type: 'Album', secondary_types: [], - year: "2023", - reason: "follow", + year: '2023', + reason: 'follow', wanted_at: ago(2 * DAY), held_at: null, - image_url: "", - ...extra, + image_url: '', + ...extra }; } function page( items: Want[], - library: LibraryListing = { listed_at: ago(DAY), unlisted: 0 }, + library: LibraryListing = { listed_at: ago(DAY), unlisted: 0, unmatched: [] } ): WantsPage { return { items, total: items.length, limit: 50, offset: 0, - counts: { - wanted: 2, - missing: 1, - held: 4, - following: 1, - activity: 1, - attention: 0, - blocked: 0, - }, - library, + counts: { wanted: 2, missing: 1, held: 4, following: 1, activity: 1, attention: 0, blocked: 0 }, + library }; } function grab(extra: Partial = {}): WantGrab { return { - id: "g1", - name: "Blur - Blur (1997) [FLAC]", + id: 'g1', + name: 'Blur - Blur (1997) [FLAC]', grabbed_at: ago(60_000), - origin: "schedule", - quality: "lossless", + origin: 'schedule', + quality: 'lossless', live: true, - end_reason: "", + end_reason: '', progress: 0, - import_state: "", - import_reason: "", - end_detail: "", + import_state: '', + import_reason: '', + end_detail: '', imported_at: null, imported_files: 0, - ...extra, + ...extra }; } afterEach(() => vi.clearAllMocks()); function setup( - items: Want[] = [ - want("w1"), - want("w2", { reason: "request", requested_by: "sam", title: "Parklife" }), - ], - state: WantState = "wanted", - library?: LibraryListing, + items: Want[] = [want('w1'), want('w2', { reason: 'request', requested_by: 'sam', title: 'Parklife' })], + state: WantState = 'wanted', + library?: LibraryListing ) { vi.mocked(createWantsQuery).mockReturnValue( - mockInfiniteQuery({ - pages: [page(items, library)], - }) as unknown as ReturnType, + mockInfiniteQuery({ pages: [page(items, library)] }) as unknown as ReturnType ); vi.mocked(createCandidatesQuery).mockReturnValue( - mockQuery({ data: [] }) as unknown as ReturnType< - typeof createCandidatesQuery - >, + mockQuery({ data: [] }) as unknown as ReturnType ); return render(WantList, { list: state }); } -describe("WantList", () => { - test("asks for its own state", () => { - setup([], "missing"); - expect(createWantsQuery).toHaveBeenCalledWith("missing"); +describe('WantList', () => { + test('asks for its own state', () => { + setup([], 'missing'); + expect(createWantsQuery).toHaveBeenCalledWith('missing'); expect(screen.getByText(/no missing albums are wanted back/i)).toBeTruthy(); }); - test("a want says why it is wanted", () => { + test('a want says why it is wanted', () => { setup(); - const why = screen - .getAllByTestId("want-why") - .map((e) => e.textContent ?? ""); + const why = screen.getAllByTestId('want-why').map((e) => e.textContent ?? ''); expect(why[0]).toMatch(/new from a followed artist/i); expect(why[1]).toMatch(/requested by sam/i); }); - test("remove takes a want off the list", async () => { + test('remove takes a want off the list', async () => { setup(); - await fireEvent.click( - screen.getByRole("button", { name: /stop wanting parklife/i }), - ); - await waitFor(() => expect(removeWant).toHaveBeenCalledWith("w2")); + await fireEvent.click(screen.getByRole('button', { name: /stop wanting parklife/i })); + await waitFor(() => expect(removeWant).toHaveBeenCalledWith('w2')); }); - test("a held album can be taken off the list, and is not searched for", async () => { - setup( - [want("w1", { held_at: ago(DAY), album_id: "al-1", reason: "library" })], - "held", - ); - expect( - screen.queryByRole("button", { name: /stop wanting|search for/i }), - ).toBeNull(); - expect( - screen - .getByRole("link", { name: /the ballad of darren/i }) - .getAttribute("href"), - ).toBe("/albums/al-1"); - expect(screen.getByTestId("want-why").textContent).toMatch( - /in your library when the list began/i, - ); - await fireEvent.click( - screen.getByRole("button", { - name: /take the ballad of darren off the list/i, - }), - ); - await waitFor(() => expect(removeWant).toHaveBeenCalledWith("w1")); + test('a held album can be taken off the list, and is not searched for', async () => { + setup([want('w1', { held_at: ago(DAY), album_id: 'al-1', reason: 'library' })], 'held'); + expect(screen.queryByRole('button', { name: /stop wanting|search for/i })).toBeNull(); + expect(screen.getByRole('link', { name: /the ballad of darren/i }).getAttribute('href')).toBe('/albums/al-1'); + expect(screen.getByTestId('want-why').textContent).toMatch(/in your library when the list began/i); + await fireEvent.click(screen.getByRole('button', { name: /take the ballad of darren off the list/i })); + await waitFor(() => expect(removeWant).toHaveBeenCalledWith('w1')); }); - test("the held tab says while the library is being listed, and what could not be", () => { - setup([], "held", { listed_at: null, unlisted: 0 }); - expect(screen.getByTestId("library-listing").textContent).toMatch( - /being added to this list/i, - ); + test('the held tab says while the library is being listed', () => { + setup([], 'held', { listed_at: null, unlisted: 0, unmatched: [] }); + expect(screen.getByTestId('library-listing').textContent).toMatch(/being added to this list/i); }); - test("the held tab names how many albums nothing names to MusicBrainz", () => { - setup([want("w1", { held_at: ago(DAY) })], "held", { + test('the held tab names the albums no MusicBrainz album matched, on request', async () => { + setup([want('w1', { held_at: ago(DAY) })], 'held', { listed_at: ago(DAY), unlisted: 12, + unmatched: [ + { album_id: 'al-9', title: 'Demo Tape', artist: 'Blur', detail: 'MusicBrainz has no album called "Demo Tape" by Blur.' } + ] }); - expect(screen.getByTestId("library-listing").textContent).toMatch( - /12 albums in your library\s+aren't on this list/i, - ); + expect(screen.getByTestId('library-listing').textContent).toMatch(/12 albums in your library\s+aren't on this list/i); + expect(screen.queryByTestId('unmatched-albums')).toBeNull(); + await fireEvent.click(screen.getByRole('button', { name: /show them/i })); + const list = screen.getByTestId('unmatched-albums'); + expect(list.textContent).toMatch(/Demo Tape/); + expect(list.textContent).toMatch(/has no album called/); + expect(screen.getByRole('link', { name: 'Demo Tape' }).getAttribute('href')).toBe('/albums/al-9'); + expect(screen.getByText(/the first 1 of 12/i)).toBeTruthy(); }); - test("an empty list explains what lands there", () => { + test('an empty list explains what lands there', () => { setup([]); expect(screen.getByText(/nothing is wanted/i)).toBeTruthy(); }); - test("each open want says what its last search found", () => { + test('each open want says what its last search found', () => { setup([ - want("w1"), - want("w2", { - title: "Parklife", - search: { - searched_at: ago(3_600_000), - outcome: "nothing", - considered: 4, - best_release: "x", - best_verdict: "rejected", - error: "", - }, - }), - want("w3", { title: "Blur", grab: grab() }), - want("w4", { - title: "13", - grab: grab({ import_state: "downloading", progress: 0.42 }), + want('w1'), + want('w2', { + title: 'Parklife', + search: { searched_at: ago(3_600_000), outcome: 'nothing', considered: 4, best_release: 'x', best_verdict: 'rejected', error: '' } }), + want('w3', { title: 'Blur', grab: grab() }), + want('w4', { title: '13', grab: grab({ import_state: 'downloading', progress: 0.42 }) }) ]); - const lines = screen - .getAllByTestId("want-search") - .map((e) => e.textContent ?? ""); + const lines = screen.getAllByTestId('want-search').map((e) => e.textContent ?? ''); expect(lines[0]).toMatch(/not searched yet/i); expect(lines[1]).toMatch(/4 found, none of them this album/i); - expect(lines[2]).toMatch( - /downloading blur - blur \(1997\) \[flac\], taken by the schedule/i, - ); + expect(lines[2]).toMatch(/downloading blur - blur \(1997\) \[flac\], taken by the schedule/i); expect(lines[3]).toMatch(/downloading 42%/i); }); - test("a download waiting on a person says why and points to Activity", () => { + test('a download waiting on a person says why and points to Activity', () => { setup([ - want("w1", { - grab: grab({ - import_state: "review", - import_reason: "Only the lengths agree. A person decides.", - }), - }), + want('w1', { grab: grab({ import_state: 'review', import_reason: 'Only the lengths agree. A person decides.' }) }) ]); - const line = screen.getByTestId("want-search"); + const line = screen.getByTestId('want-search'); expect(line.textContent).toMatch(/needs you: only the lengths agree/i); - expect( - screen - .getByRole("link", { name: /decide on activity/i }) - .getAttribute("href"), - ).toBe("/wanted/activity"); + expect(screen.getByRole('link', { name: /decide on activity/i }).getAttribute('href')).toBe('/wanted/activity'); // A live download can't be searched for again. - expect( - ( - screen.getByRole("button", { - name: /search for the ballad of darren now/i, - }) as HTMLButtonElement - ).disabled, - ).toBe(true); + expect((screen.getByRole('button', { name: /search for the ballad of darren now/i }) as HTMLButtonElement).disabled).toBe(true); }); - test("a refused download is named beside the search that follows it", () => { + test('a refused download is named beside the search that follows it', () => { setup([ - want("w1", { - search: { - searched_at: ago(3_600_000), - outcome: "nothing", - considered: 0, - best_release: "", - best_verdict: "", - error: "", - }, - grab: grab({ - live: false, - end_reason: "refused", - import_state: "refused", - import_reason: "Another album.", - }), - }), + want('w1', { + search: { searched_at: ago(3_600_000), outcome: 'nothing', considered: 0, best_release: '', best_verdict: '', error: '' }, + grab: grab({ live: false, end_reason: 'refused', import_state: 'refused', import_reason: 'Another album.' }) + }) ]); - expect(screen.getByTestId("want-search").textContent).toMatch( - /last download refused: another album/i, - ); + expect(screen.getByTestId('want-search').textContent).toMatch(/last download refused: another album/i); }); - test("a download given up on is named beside the search that follows it", () => { + test('a download given up on is named beside the search that follows it', () => { setup([ - want("w1", { - search: { - searched_at: ago(3_600_000), - outcome: "waiting", - considered: 4, - best_release: "", - best_verdict: "", - error: "", - }, - grab: grab({ - live: false, - end_reason: "stalled", - import_state: "downloading", - end_detail: "Nobody was sharing it.", - }), - }), + want('w1', { + search: { searched_at: ago(3_600_000), outcome: 'waiting', considered: 4, best_release: '', best_verdict: '', error: '' }, + grab: grab({ live: false, end_reason: 'stalled', import_state: 'downloading', end_detail: 'Nobody was sharing it.' }) + }) ]); - expect(screen.getByTestId("want-search").textContent).toMatch( - /last download given up: nobody was sharing it/i, - ); + expect(screen.getByTestId('want-search').textContent).toMatch(/last download given up: nobody was sharing it/i); }); - test("a held album says how it arrived", () => { + test('a held album says how it arrived', () => { setup( [ - want("w1", { + want('w1', { held_at: ago(DAY), - album_id: "al-1", - grab: grab({ - live: false, - end_reason: "imported", - import_state: "imported", - imported_at: ago(DAY), - imported_files: 11, - }), - }), + album_id: 'al-1', + grab: grab({ live: false, end_reason: 'imported', import_state: 'imported', imported_at: ago(DAY), imported_files: 11 }) + }) ], - "held", - ); - expect(screen.getByTestId("want-arrived").textContent).toMatch( - /imported 11 tracks .* from blur - blur/i, + 'held' ); + expect(screen.getByTestId('want-arrived').textContent).toMatch(/imported 11 tracks .* from blur - blur/i); }); - test("Search runs one now and says what came of it", async () => { + test('Search runs one now and says what came of it', async () => { setup(); - await fireEvent.click( - screen.getByRole("button", { name: /search for parklife now/i }), - ); - await waitFor(() => expect(searchWant).toHaveBeenCalledWith("w2")); - await waitFor(() => - expect(pushToast).toHaveBeenCalledWith( - "Nothing found for Parklife.", - "info", - ), - ); + await fireEvent.click(screen.getByRole('button', { name: /search for parklife now/i })); + await waitFor(() => expect(searchWant).toHaveBeenCalledWith('w2')); + await waitFor(() => expect(pushToast).toHaveBeenCalledWith('Nothing found for Parklife.', 'info')); }); - test("Releases opens what the last search found", async () => { + test('Releases opens what the last search found', async () => { setup(); - expect(screen.queryByTestId("want-candidates")).toBeNull(); - const toggle = screen.getByRole("button", { - name: /releases found for parklife/i, - }); + expect(screen.queryByTestId('want-candidates')).toBeNull(); + const toggle = screen.getByRole('button', { name: /releases found for parklife/i }); await fireEvent.click(toggle); - expect(toggle.getAttribute("aria-expanded")).toBe("true"); - expect(screen.getByTestId("want-candidates")).toBeTruthy(); - expect(createCandidatesQuery).toHaveBeenCalledWith("w2"); + expect(toggle.getAttribute('aria-expanded')).toBe('true'); + expect(screen.getByTestId('want-candidates')).toBeTruthy(); + expect(createCandidatesQuery).toHaveBeenCalledWith('w2'); }); }); diff --git a/web/src/routes/wanted/wanted.test.ts b/web/src/routes/wanted/wanted.test.ts index 547b9c1a..4b1bb38d 100644 --- a/web/src/routes/wanted/wanted.test.ts +++ b/web/src/routes/wanted/wanted.test.ts @@ -1,35 +1,23 @@ -import { afterEach, describe, expect, test, vi } from "vitest"; -import { render, screen } from "@testing-library/svelte"; -import { createRawSnippet } from "svelte"; -import { mockInfiniteQuery } from "../../test-utils/query"; -import { pageUrlModule } from "#test-utils/mocks/appState.js"; -import type { WantCounts, WantsPage } from "#lib/api/types.js"; +import { afterEach, describe, expect, test, vi } from 'vitest'; +import { render, screen } from '@testing-library/svelte'; +import { createRawSnippet } from 'svelte'; +import { mockInfiniteQuery } from '../../test-utils/query'; +import { pageUrlModule } from '#test-utils/mocks/appState.js'; +import type { WantCounts, WantsPage } from '#lib/api/types.js'; -const state = vi.hoisted(() => ({ - pageUrl: new URL("http://localhost/wanted"), -})); -vi.mock("$app/state", () => pageUrlModule(state)); -vi.mock("#lib/api/wanted.js", () => ({ createWantsQuery: vi.fn() })); +const state = vi.hoisted(() => ({ pageUrl: new URL('http://localhost/wanted') })); +vi.mock('$app/state', () => pageUrlModule(state)); +vi.mock('#lib/api/wanted.js', () => ({ createWantsQuery: vi.fn() })); -import WantedLayout from "./+layout.svelte"; -import { createWantsQuery } from "#lib/api/wanted.js"; +import WantedLayout from './+layout.svelte'; +import { createWantsQuery } from '#lib/api/wanted.js'; -const counts: WantCounts = { - wanted: 5, - missing: 1, - held: 9, - following: 2, - activity: 3, - attention: 0, - blocked: 4, -}; -const children = createRawSnippet(() => ({ - render: () => `

body

`, -})); +const counts: WantCounts = { wanted: 5, missing: 1, held: 9, following: 2, activity: 3, attention: 0, blocked: 4 }; +const children = createRawSnippet(() => ({ render: () => `

body

` })); afterEach(() => vi.clearAllMocks()); -function setup(c: WantCounts = counts, path = "/wanted") { +function setup(c: WantCounts = counts, path = '/wanted') { state.pageUrl = new URL(`http://localhost${path}`); const page: WantsPage = { items: [], @@ -37,58 +25,44 @@ function setup(c: WantCounts = counts, path = "/wanted") { limit: 50, offset: 0, counts: c, - library: { listed_at: null, unlisted: 0 }, + library: { listed_at: null, unlisted: 0, unmatched: [] } }; vi.mocked(createWantsQuery).mockReturnValue( - mockInfiniteQuery({ pages: [page] }) as unknown as ReturnType< - typeof createWantsQuery - >, + mockInfiniteQuery({ pages: [page] }) as unknown as ReturnType ); // eslint-disable-next-line @typescript-eslint/no-explicit-any return render(WantedLayout, { children } as any); } -describe("/wanted layout", () => { - test("tabs are links carrying their counts, the current one marked", () => { - setup(counts, "/wanted/activity"); - const nav = screen.getByRole("navigation", { name: /wanted sections/i }); - const links = Array.from(nav.querySelectorAll("a")).map((a) => [ - a.getAttribute("href"), - a.textContent?.replace(/\s+/g, " ").trim(), - ]); +describe('/wanted layout', () => { + test('tabs are links carrying their counts, the current one marked', () => { + setup(counts, '/wanted/activity'); + const nav = screen.getByRole('navigation', { name: /wanted sections/i }); + const links = Array.from(nav.querySelectorAll('a')).map((a) => [a.getAttribute('href'), a.textContent?.replace(/\s+/g, ' ').trim()]); expect(links).toEqual([ - ["/wanted", "Wanted 5"], - ["/wanted/activity", "Activity 3"], - ["/wanted/missing", "Missing files 1"], - ["/wanted/held", "Held 9"], - ["/wanted/following", "Following 2"], - ["/wanted/blocked", "Blocked 4"], + ['/wanted', 'Wanted 5'], + ['/wanted/activity', 'Activity 3'], + ['/wanted/missing', 'Missing files 1'], + ['/wanted/held', 'Held 9'], + ['/wanted/following', 'Following 2'], + ['/wanted/blocked', 'Blocked 4'] ]); - expect(screen.getByRole("link", { name: /activity/i })).toHaveAttribute( - "aria-current", - "page", - ); - expect(screen.getByRole("link", { name: /^wanted/i })).not.toHaveAttribute( - "aria-current", - ); - expect(screen.getByTestId("tab-body")).toBeTruthy(); + expect(screen.getByRole('link', { name: /activity/i })).toHaveAttribute('aria-current', 'page'); + expect(screen.getByRole('link', { name: /^wanted/i })).not.toHaveAttribute('aria-current'); + expect(screen.getByTestId('tab-body')).toBeTruthy(); }); - test("when nothing needs a person the verdict says so", () => { + test('when nothing needs a person the verdict says so', () => { setup(); - expect(screen.getByTestId("wanted-verdict").textContent).toMatch( - /5 albums wanted, 3 downloading\. Nothing needs you\./, - ); - expect(screen.queryByTestId("tab-attention")).toBeNull(); + expect(screen.getByTestId('wanted-verdict').textContent).toMatch(/5 albums wanted, 3 downloading\. Nothing needs you\./); + expect(screen.queryByTestId('tab-attention')).toBeNull(); }); - test("downloads waiting on a person lead the page and mark Activity", () => { + test('downloads waiting on a person lead the page and mark Activity', () => { setup({ ...counts, attention: 2 }); - const verdict = screen.getByTestId("wanted-verdict"); + const verdict = screen.getByTestId('wanted-verdict'); expect(verdict.textContent).toMatch(/2 downloads need you/); - expect( - screen.getByRole("link", { name: /see activity/i }).getAttribute("href"), - ).toBe("/wanted/activity"); - expect(screen.getByTestId("tab-attention").textContent).toBe("2"); + expect(screen.getByRole('link', { name: /see activity/i }).getAttribute('href')).toBe('/wanted/activity'); + expect(screen.getByTestId('tab-attention').textContent).toBe('2'); }); }); -- 2.54.0 From 80d2fdc3aba91949d98d30dc97158b416558a316 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Sat, 10 Oct 2026 17:29:42 -0400 Subject: [PATCH 020/117] fix: under the album's own name, matching track titles are enough to match it; a refusal says which track differed and how (M502 #5569) The first run on the library refused singles whose one track has the album's title ("0 of 1 agree") and near-complete albums (11 of 14), because every track's length had to fit as well. Lengths now only decide between groups whose names merely overlap. The reason for a refusal names the first track that differed, with both titles and lengths, so what is left can be judged. Migration 0087 clears the refusals so those albums are matched again. Co-Authored-By: Claude Opus 5.5 --- .../migrations/0087_library_rematch.down.sql | 2 + .../db/migrations/0087_library_rematch.up.sql | 9 ++ internal/wanted/library_test.go | 14 ++- internal/wanted/match.go | 88 +++++++++++++------ 4 files changed, 83 insertions(+), 30 deletions(-) create mode 100644 internal/db/migrations/0087_library_rematch.down.sql create mode 100644 internal/db/migrations/0087_library_rematch.up.sql diff --git a/internal/db/migrations/0087_library_rematch.down.sql b/internal/db/migrations/0087_library_rematch.down.sql new file mode 100644 index 00000000..66b5e525 --- /dev/null +++ b/internal/db/migrations/0087_library_rematch.down.sql @@ -0,0 +1,2 @@ +-- Nothing to undo: the refused albums are matched again either way. +SELECT 1; diff --git a/internal/db/migrations/0087_library_rematch.up.sql b/internal/db/migrations/0087_library_rematch.up.sql new file mode 100644 index 00000000..7347fff3 --- /dev/null +++ b/internal/db/migrations/0087_library_rematch.up.sql @@ -0,0 +1,9 @@ +-- The first matching (#5569) required every track's length to fit as well +-- as its title, and refused singles and near-complete albums the library +-- plainly holds. Matching now takes the titles alone under the album's own +-- name: the albums it refused are matched again. +DELETE FROM library_album_matches WHERE outcome = 'unmatched'; + +UPDATE acquisition_settings + SET library_listed_at = NULL, library_unlisted = 0 + WHERE library_listing_started_at IS NOT NULL; diff --git a/internal/wanted/library_test.go b/internal/wanted/library_test.go index eb865d1a..76410b7e 100644 --- a/internal/wanted/library_test.go +++ b/internal/wanted/library_test.go @@ -252,10 +252,22 @@ func TestTrackAgreement(t *testing.T) { {"the right title at the wrong length", []libraryTrack{{Number: 2, Title: "Song", LengthMs: 400000}}, 0}, {"a track counted once", []libraryTrack{{Title: "Song", LengthMs: 200000}, {Title: "Song", LengthMs: 200000}}, 1}, } { - if got := trackAgreement(c.tracks, edition); got != c.want { + if got, _ := trackAgreement(c.tracks, edition, false); got != c.want { t.Errorf("%s: %d agree, want %d", c.name, got, c.want) } } + // Under the album's own name the titles are enough; the reason a track + // did not agree names both sides. + edit := []libraryTrack{{Number: 2, Title: "Song (Radio Edit)", LengthMs: 150000}} + if n, miss := trackAgreement(edit, edition, false); n != 0 || miss != `"Song (Radio Edit)" runs 2:30 here; MusicBrainz's track 2 is "Song", 3:20` { + t.Errorf("by length: %d, %q", n, miss) + } + if n, _ := trackAgreement(edit, edition, true); n != 1 { + t.Errorf("by title alone: %d agree", n) + } + if _, miss := trackAgreement([]libraryTrack{{Title: "Other", LengthMs: 61000}}, edition, true); miss != `no track of its edition is like "Other" (1:01)` { + t.Errorf("unplaced: %q", miss) + } if agrees(3, 4) || !agrees(4, 5) || agrees(0, 0) { t.Error("agrees: 80% of the album's tracks, and an album has tracks") } diff --git a/internal/wanted/match.go b/internal/wanted/match.go index bcdf3ea8..04c7527e 100644 --- a/internal/wanted/match.go +++ b/internal/wanted/match.go @@ -101,10 +101,14 @@ func tracksToMatch(rows []dbq.ListAlbumTracksToMatchRow) []libraryTrack { } // trackAgreement counts the album's tracks an edition agrees with: the same -// recording, or a length that fits and a title that agrees. A track is -// paired by its disc and number first, then by title, so an album missing -// some tracks still pairs the ones it has. -func trackAgreement(tracks []libraryTrack, edition []editionTrack) int { +// recording, or a title that agrees at a length that fits. When the group is +// called just what the album is called, the titles alone are enough: the +// lengths are there to tell apart albums whose names only overlap, and a +// rip's single may be the edit MusicBrainz lists at another length. A track +// is paired by its disc and number first, then by title, so an album missing +// some tracks still pairs the ones it has. miss says how the first track +// that did not agree differs, for the reason a match was refused. +func trackAgreement(tracks []libraryTrack, edition []editionTrack, titlesEnough bool) (n int, miss string) { byPos := make(map[[2]int]int, len(edition)) for i, e := range edition { byPos[[2]int{max(e.Medium, 1), e.Position}] = i @@ -113,27 +117,47 @@ func trackAgreement(tracks []libraryTrack, edition []editionTrack) int { if t.Recording != "" && strings.EqualFold(t.Recording, e.Recording) { return true } - return release.LengthFits(t.LengthMs, e.LengthMs) && release.TitleAgrees(t.Title, e.Title) + return release.TitleAgrees(t.Title, e.Title) && (titlesEnough || release.LengthFits(t.LengthMs, e.LengthMs)) } used := make([]bool, len(edition)) - n := 0 for _, t := range tracks { + at, placed := -1, false if t.Number > 0 { - if i, ok := byPos[[2]int{max(t.Disc, 1), t.Number}]; ok && !used[i] && agree(t, edition[i]) { - used[i] = true - n++ - continue + if i, ok := byPos[[2]int{max(t.Disc, 1), t.Number}]; ok { + at = i + placed = !used[i] && agree(t, edition[i]) + if placed { + used[i] = true + } } } for i, e := range edition { - if !used[i] && agree(t, e) { - used[i] = true - n++ - break + if !placed && !used[i] && agree(t, e) { + used[i], placed = true, true } } + switch { + case placed: + n++ + case miss != "": + case at >= 0: + e := edition[at] + miss = fmt.Sprintf("%q runs %s here; MusicBrainz's track %d is %q, %s", + t.Title, minutes(t.LengthMs), e.Position, e.Title, minutes(e.LengthMs)) + default: + miss = fmt.Sprintf("no track of its edition is like %q (%s)", t.Title, minutes(t.LengthMs)) + } } - return n + return n, miss +} + +// minutes writes a length as m:ss, or "no length". +func minutes(ms int) string { + if ms <= 0 { + return "no length" + } + sec := (ms + 500) / 1000 + return fmt.Sprintf("%d:%02d", sec/60, sec%60) } // agrees: enough of the album's tracks agree for the edition to be it. @@ -173,14 +197,14 @@ func (s *Service) matchAlbum(ctx context.Context, a dbq.ListAlbumsToMatchRow, tr // discography first, then a search. tried := map[string]bool{} var best albumMatch - bestN, bestEdition, considered := -1, 0, 0 + bestN, bestEdition, considered, bestMiss := -1, 0, 0, "" check := func(cands []candidate) error { for _, c := range cands { if tried[c.Group] || len(tried) >= matchCandidates { continue } tried[c.Group] = true - n, size, err := s.agreementWith(ctx, c.Group, tracks) + n, size, miss, err := s.agreementWith(ctx, c.Group, tracks, sameTitle(c.Title, a.Title)) if err != nil { return err } @@ -189,7 +213,7 @@ func (s *Service) matchAlbum(ctx context.Context, a dbq.ListAlbumsToMatchRow, tr // one; then the edition nearest the album's length. st, bt := sameTitle(c.Title, a.Title), sameTitle(best.Title, a.Title) if n > bestN || (n == bestN && ((st && !bt) || (st == bt && abs(size-len(tracks)) < abs(bestEdition-len(tracks))))) { - best, bestN, bestEdition = albumMatch{candidate: c}, n, size + best, bestN, bestEdition, bestMiss = albumMatch{candidate: c}, n, size, miss } } return nil @@ -217,9 +241,15 @@ func (s *Service) matchAlbum(ctx context.Context, a dbq.ListAlbumsToMatchRow, tr if considered == 0 { return albumMatch{Detail: fmt.Sprintf("MusicBrainz has no album called %q by %s.", a.Title, a.ArtistName)}, nil } - return albumMatch{Detail: fmt.Sprintf( - "MusicBrainz has %d %s called %q by %s, but none has its tracks: the closest, %q, agrees on %d of its %d.", - considered, plural(considered, "album", "albums"), a.Title, a.ArtistName, best.Title, max(bestN, 0), len(tracks))}, nil + detail := fmt.Sprintf("MusicBrainz has %d %s called %q by %s, but none has its tracks: the closest, %q, agrees on %d of its %d", + considered, plural(considered, "album", "albums"), a.Title, a.ArtistName, best.Title, max(bestN, 0), len(tracks)) + switch { + case bestN < 0: + detail += " (MusicBrainz lists no edition of it)" + case bestMiss != "": + detail += ": " + bestMiss + } + return albumMatch{Detail: detail + "."}, nil } // groupCandidate reads a group the tags name. A group MusicBrainz does not @@ -296,15 +326,15 @@ func (s *Service) searchCandidates(ctx context.Context, a dbq.ListAlbumsToMatchR } // agreementWith is how many of the album's tracks the group's best edition -// agrees with, and that edition's length. A group MusicBrainz no longer has -// agrees with none. -func (s *Service) agreementWith(ctx context.Context, group string, tracks []libraryTrack) (n, size int, err error) { +// agrees with, that edition's length, and how it differs. A group +// MusicBrainz no longer has, or lists no edition of, agrees with none (-1). +func (s *Service) agreementWith(ctx context.Context, group string, tracks []libraryTrack, titlesEnough bool) (n, size int, miss string, err error) { eds, err := s.cat.Releases(ctx, group) switch { case errors.Is(err, catalogue.ErrNotFound): - return -1, 0, nil + return -1, 0, "", nil case err != nil: - return 0, 0, fmt.Errorf("%w: %v", errMusicBrainzAway, err) + return 0, 0, "", fmt.Errorf("%w: %v", errMusicBrainzAway, err) } n = -1 for _, e := range eds.Editions { @@ -313,11 +343,11 @@ func (s *Service) agreementWith(ctx context.Context, group string, tracks []libr edition = append(edition, editionTrack{Medium: int(t.MediumPosition), Position: int(t.Position), Title: t.Title, LengthMs: int(t.LengthMs), Recording: t.RecordingMbid}) } - if got := trackAgreement(tracks, edition); got > n || (got == n && abs(len(edition)-len(tracks)) < abs(size-len(tracks))) { - n, size = got, len(edition) + if got, m := trackAgreement(tracks, edition, titlesEnough); got > n || (got == n && abs(len(edition)-len(tracks)) < abs(size-len(tracks))) { + n, size, miss = got, len(edition), m } } - return n, size, nil + return n, size, miss, nil } // creditedArtist is the artist a want is filed under: the library's own -- 2.54.0 From bf409d5f3f2bac924f12f451c21491c121e93489 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Sat, 10 Oct 2026 17:44:06 -0400 Subject: [PATCH 021/117] fix: a second copy of a track counts once, joining words don't part two titles, and a newer matcher retries an older one's refusals (M502 #5569) The second run's refusals were mostly albums holding a track twice (a two-track single with three files), which no edition can pair, and release names that differ only in "+" against "/". Tracks are now read once per disc, number and title, and "and", "&" and "+" count for nothing when titles are compared; an edition's own title also counts as the album's name. library_album_matches records which matcher version decided each album. Refusals by an older version are matched again, even after the listing is done, so an improvement reaches them without a migration. Co-Authored-By: Claude Opus 5.5 --- internal/db/dbq/models.go | 1 + internal/db/dbq/wanted.sql.go | 45 +++++++++++++------ .../0088_library_matcher_version.down.sql | 1 + .../0088_library_matcher_version.up.sql | 4 ++ internal/db/queries/wanted.sql | 30 ++++++++----- internal/wanted/library.go | 28 +++++++++--- internal/wanted/library_test.go | 41 +++++++++++++++++ internal/wanted/match.go | 30 ++++++++++--- 8 files changed, 146 insertions(+), 34 deletions(-) create mode 100644 internal/db/migrations/0088_library_matcher_version.down.sql create mode 100644 internal/db/migrations/0088_library_matcher_version.up.sql diff --git a/internal/db/dbq/models.go b/internal/db/dbq/models.go index 883db4df..84b05f94 100644 --- a/internal/db/dbq/models.go +++ b/internal/db/dbq/models.go @@ -494,6 +494,7 @@ type LibraryAlbumMatch struct { ReleaseGroupMbid string Detail string DecidedAt pgtype.Timestamptz + Matcher int32 } type LibraryChange struct { diff --git a/internal/db/dbq/wanted.sql.go b/internal/db/dbq/wanted.sql.go index fbe40c14..4699d712 100644 --- a/internal/db/dbq/wanted.sql.go +++ b/internal/db/dbq/wanted.sql.go @@ -20,6 +20,19 @@ func (q *Queries) ClearDefaultReleaseTypeProfile(ctx context.Context) error { return err } +const countAlbumsToRematch = `-- name: CountAlbumsToRematch :one +SELECT COUNT(*) FROM library_album_matches + WHERE outcome = 'unmatched' AND matcher < $1::int +` + +// Refusals by an older matcher, which a finished listing goes back to. +func (q *Queries) CountAlbumsToRematch(ctx context.Context, matcher int32) (int64, error) { + row := q.db.QueryRow(ctx, countAlbumsToRematch, matcher) + var count int64 + err := row.Scan(&count) + return count, err +} + const countWants = `-- name: CountWants :one SELECT COUNT(*) FILTER (WHERE fulfilled_at IS NULL AND reason <> 'missing')::bigint AS wanted, COUNT(*) FILTER (WHERE fulfilled_at IS NULL AND reason = 'missing')::bigint AS missing, @@ -126,9 +139,9 @@ func (q *Queries) DeleteWant(ctx context.Context, id pgtype.UUID) (int64, error) const finishLibraryListing = `-- name: FinishLibraryListing :one UPDATE acquisition_settings - SET library_listed_at = now(), + SET library_listed_at = COALESCE(library_listed_at, now()), library_unlisted = (SELECT COUNT(*) FROM library_album_matches WHERE outcome = 'unmatched') - WHERE id = true AND library_listed_at IS NULL + WHERE id = true RETURNING library_listed_at, library_unlisted ` @@ -137,7 +150,8 @@ type FinishLibraryListingRow struct { LibraryUnlisted int32 } -// The listing is done: count the albums no match was found for. +// The listing is done, or done again after a newer matcher's retries: +// count the albums no match was found for. The first finish is kept. func (q *Queries) FinishLibraryListing(ctx context.Context) (FinishLibraryListingRow, error) { row := q.db.QueryRow(ctx, finishLibraryListing) var i FinishLibraryListingRow @@ -405,13 +419,15 @@ SELECT a.id, a.title, a.mbid, a.release_group_mbid, ar.name AS artist_name, ar.m JOIN artists ar ON ar.id = a.artist_id WHERE a.created_at <= $1::timestamptz AND EXISTS (SELECT 1 FROM tracks t WHERE t.album_id = a.id) - AND NOT EXISTS (SELECT 1 FROM library_album_matches m WHERE m.album_id = a.id) + AND NOT EXISTS (SELECT 1 FROM library_album_matches m + WHERE m.album_id = a.id AND (m.outcome = 'listed' OR m.matcher >= $2::int)) ORDER BY a.created_at, a.id - LIMIT $2 + LIMIT $3 ` type ListAlbumsToMatchParams struct { Cutoff pgtype.Timestamptz + Matcher int32 MaxRows int32 } @@ -424,11 +440,11 @@ type ListAlbumsToMatchRow struct { ArtistMbid *string } -// Library albums from before the cutoff that the listing has not decided: -// each is matched to MusicBrainz (#5569). An album row with no tracks is -// not an album in the library. +// Library albums from before the cutoff that the listing has not decided, +// or that an older matcher refused: each is matched to MusicBrainz (#5569). +// An album row with no tracks is not an album in the library. func (q *Queries) ListAlbumsToMatch(ctx context.Context, arg ListAlbumsToMatchParams) ([]ListAlbumsToMatchRow, error) { - rows, err := q.db.Query(ctx, listAlbumsToMatch, arg.Cutoff, arg.MaxRows) + rows, err := q.db.Query(ctx, listAlbumsToMatch, arg.Cutoff, arg.Matcher, arg.MaxRows) if err != nil { return nil, err } @@ -937,11 +953,12 @@ func (q *Queries) MarkGroupSeen(ctx context.Context, arg MarkGroupSeenParams) er } const recordAlbumMatch = `-- name: RecordAlbumMatch :exec -INSERT INTO library_album_matches (album_id, outcome, method, release_group_mbid, detail) -VALUES ($1, $2, $3, $4, $5) +INSERT INTO library_album_matches (album_id, outcome, method, release_group_mbid, detail, matcher) +VALUES ($1, $2, $3, $4, $5, + $6) ON CONFLICT (album_id) DO UPDATE - SET outcome = EXCLUDED.outcome, method = EXCLUDED.method, - release_group_mbid = EXCLUDED.release_group_mbid, detail = EXCLUDED.detail, decided_at = now() + SET outcome = EXCLUDED.outcome, method = EXCLUDED.method, release_group_mbid = EXCLUDED.release_group_mbid, + detail = EXCLUDED.detail, matcher = EXCLUDED.matcher, decided_at = now() ` type RecordAlbumMatchParams struct { @@ -950,6 +967,7 @@ type RecordAlbumMatchParams struct { Method string ReleaseGroupMbid string Detail string + Matcher int32 } func (q *Queries) RecordAlbumMatch(ctx context.Context, arg RecordAlbumMatchParams) error { @@ -959,6 +977,7 @@ func (q *Queries) RecordAlbumMatch(ctx context.Context, arg RecordAlbumMatchPara arg.Method, arg.ReleaseGroupMbid, arg.Detail, + arg.Matcher, ) return err } diff --git a/internal/db/migrations/0088_library_matcher_version.down.sql b/internal/db/migrations/0088_library_matcher_version.down.sql new file mode 100644 index 00000000..1e22094a --- /dev/null +++ b/internal/db/migrations/0088_library_matcher_version.down.sql @@ -0,0 +1 @@ +ALTER TABLE library_album_matches DROP COLUMN matcher; diff --git a/internal/db/migrations/0088_library_matcher_version.up.sql b/internal/db/migrations/0088_library_matcher_version.up.sql new file mode 100644 index 00000000..112d7d6b --- /dev/null +++ b/internal/db/migrations/0088_library_matcher_version.up.sql @@ -0,0 +1,4 @@ +-- Which version of the matching (#5569) decided an album. A refusal by an +-- older matcher is tried again by a newer one, so improving the matching +-- reaches the albums it refused without a migration each time. +ALTER TABLE library_album_matches ADD COLUMN matcher integer NOT NULL DEFAULT 0; diff --git a/internal/db/queries/wanted.sql b/internal/db/queries/wanted.sql index 2cb967b1..49077412 100644 --- a/internal/db/queries/wanted.sql +++ b/internal/db/queries/wanted.sql @@ -272,18 +272,24 @@ SELECT a.id, 'listed', 'tags', a.release_group_mbid ON CONFLICT (album_id) DO NOTHING; -- name: ListAlbumsToMatch :many --- Library albums from before the cutoff that the listing has not decided: --- each is matched to MusicBrainz (#5569). An album row with no tracks is --- not an album in the library. +-- Library albums from before the cutoff that the listing has not decided, +-- or that an older matcher refused: each is matched to MusicBrainz (#5569). +-- An album row with no tracks is not an album in the library. SELECT a.id, a.title, a.mbid, a.release_group_mbid, ar.name AS artist_name, ar.mbid AS artist_mbid FROM albums a JOIN artists ar ON ar.id = a.artist_id WHERE a.created_at <= sqlc.arg(cutoff)::timestamptz AND EXISTS (SELECT 1 FROM tracks t WHERE t.album_id = a.id) - AND NOT EXISTS (SELECT 1 FROM library_album_matches m WHERE m.album_id = a.id) + AND NOT EXISTS (SELECT 1 FROM library_album_matches m + WHERE m.album_id = a.id AND (m.outcome = 'listed' OR m.matcher >= sqlc.arg(matcher)::int)) ORDER BY a.created_at, a.id LIMIT sqlc.arg(max_rows); +-- name: CountAlbumsToRematch :one +-- Refusals by an older matcher, which a finished listing goes back to. +SELECT COUNT(*) FROM library_album_matches + WHERE outcome = 'unmatched' AND matcher < sqlc.arg(matcher)::int; + -- name: ListAlbumTracksToMatch :many -- What an album's tracks say about themselves, to check a match against. SELECT title, disc_number, track_number, duration_ms, mbid @@ -308,18 +314,20 @@ VALUES (sqlc.arg(release_group_mbid), sqlc.arg(artist_mbid), sqlc.arg(artist_nam ON CONFLICT (release_group_mbid) DO NOTHING; -- name: RecordAlbumMatch :exec -INSERT INTO library_album_matches (album_id, outcome, method, release_group_mbid, detail) -VALUES (sqlc.arg(album_id), sqlc.arg(outcome), sqlc.arg(method), sqlc.arg(release_group_mbid), sqlc.arg(detail)) +INSERT INTO library_album_matches (album_id, outcome, method, release_group_mbid, detail, matcher) +VALUES (sqlc.arg(album_id), sqlc.arg(outcome), sqlc.arg(method), sqlc.arg(release_group_mbid), sqlc.arg(detail), + sqlc.arg(matcher)) ON CONFLICT (album_id) DO UPDATE - SET outcome = EXCLUDED.outcome, method = EXCLUDED.method, - release_group_mbid = EXCLUDED.release_group_mbid, detail = EXCLUDED.detail, decided_at = now(); + SET outcome = EXCLUDED.outcome, method = EXCLUDED.method, release_group_mbid = EXCLUDED.release_group_mbid, + detail = EXCLUDED.detail, matcher = EXCLUDED.matcher, decided_at = now(); -- name: FinishLibraryListing :one --- The listing is done: count the albums no match was found for. +-- The listing is done, or done again after a newer matcher's retries: +-- count the albums no match was found for. The first finish is kept. UPDATE acquisition_settings - SET library_listed_at = now(), + SET library_listed_at = COALESCE(library_listed_at, now()), library_unlisted = (SELECT COUNT(*) FROM library_album_matches WHERE outcome = 'unmatched') - WHERE id = true AND library_listed_at IS NULL + WHERE id = true RETURNING library_listed_at, library_unlisted; -- name: ListUnmatchedLibraryAlbums :many diff --git a/internal/wanted/library.go b/internal/wanted/library.go index 8642d846..ea56759c 100644 --- a/internal/wanted/library.go +++ b/internal/wanted/library.go @@ -23,6 +23,15 @@ const libraryMatchBudget = 2 * time.Minute // libraryMatchBatch is how many albums one query hands the matching. const libraryMatchBatch = 25 +// libraryMatcher is the matching's version. Raise it when the matching +// changes in a way that could match an album it refused: albums an older +// version refused are matched again, even after the listing is done. +// +// 1 tags, release, discography and search, checked by title and length +// 2 titles alone under the album's own name; a refusal names the track +// 3 a second copy of a track counts once; joining words count for nothing +const libraryMatcher = 3 + // listLibrary puts the library's albums on the wanted list as held, once // (operator, 2026-10-10: "make a record saying that the import has // happened so later changes like unwanting hold and dont auto import from @@ -39,7 +48,8 @@ const libraryMatchBatch = 25 // of the albums no match was found for. // // Each album decided is recorded in library_album_matches, so an album taken -// off the list stays off, and once done the listing never runs again. +// off the list stays off. Once done the listing runs again only for albums +// an older matcher refused (libraryMatcher). func (s *Service) listLibrary(ctx context.Context, res *PassResult) error { q := dbq.New(s.pool) state, err := q.GetLibraryListing(ctx) @@ -47,7 +57,13 @@ func (s *Service) listLibrary(ctx context.Context, res *PassResult) error { return fmt.Errorf("wanted: read the library listing: %w", err) } if state.LibraryListedAt.Valid { - return nil + stale, err := q.CountAlbumsToRematch(ctx, libraryMatcher) + if err != nil { + return fmt.Errorf("wanted: count refusals to retry: %w", err) + } + if stale == 0 { + return nil + } } cutoff := state.LibraryListingStartedAt if !cutoff.Valid { @@ -67,7 +83,8 @@ func (s *Service) listLibrary(ctx context.Context, res *PassResult) error { deadline := s.now().Add(libraryMatchBudget) for { - albums, err := q.ListAlbumsToMatch(ctx, dbq.ListAlbumsToMatchParams{Cutoff: cutoff, MaxRows: libraryMatchBatch}) + albums, err := q.ListAlbumsToMatch(ctx, dbq.ListAlbumsToMatchParams{ + Cutoff: cutoff, Matcher: libraryMatcher, MaxRows: libraryMatchBatch}) if err != nil { return fmt.Errorf("wanted: albums to match: %w", err) } @@ -107,7 +124,8 @@ func (s *Service) listAlbum(ctx context.Context, q *dbq.Queries, a dbq.ListAlbum } if !m.OK { s.logger.Info("wanted: no MusicBrainz match for a library album", "album", a.Title, "artist", a.ArtistName, "why", m.Detail) - return 0, q.RecordAlbumMatch(ctx, dbq.RecordAlbumMatchParams{AlbumID: a.ID, Outcome: "unmatched", Detail: m.Detail}) + return 0, q.RecordAlbumMatch(ctx, dbq.RecordAlbumMatchParams{ + AlbumID: a.ID, Outcome: "unmatched", Detail: m.Detail, Matcher: libraryMatcher}) } var listed int64 err = pgx.BeginFunc(ctx, s.pool, func(tx pgx.Tx) error { @@ -124,7 +142,7 @@ func (s *Service) listAlbum(ctx context.Context, q *dbq.Queries, a dbq.ListAlbum } listed = n return tq.RecordAlbumMatch(ctx, dbq.RecordAlbumMatchParams{ - AlbumID: a.ID, Outcome: "listed", Method: m.Method, ReleaseGroupMbid: m.Group, + AlbumID: a.ID, Outcome: "listed", Method: m.Method, ReleaseGroupMbid: m.Group, Matcher: libraryMatcher, }) }) if err != nil { diff --git a/internal/wanted/library_test.go b/internal/wanted/library_test.go index 76410b7e..029252ab 100644 --- a/internal/wanted/library_test.go +++ b/internal/wanted/library_test.go @@ -268,7 +268,48 @@ func TestTrackAgreement(t *testing.T) { if _, miss := trackAgreement([]libraryTrack{{Title: "Other", LengthMs: 61000}}, edition, true); miss != `no track of its edition is like "Other" (1:01)` { t.Errorf("unplaced: %q", miss) } + if !sameTitle("Brain Stew + Jaded", "Brain Stew / Jaded") || sameTitle("+++", "Grind & Hustle") || sameTitle("Blur", "The Best of Blur") { + t.Error("sameTitle: joining words count for nothing, and a title of none is no title") + } + n := int32(1) + copies := tracksToMatch([]dbq.ListAlbumTracksToMatchRow{ + {Title: "Song", TrackNumber: &n, DurationMs: 1}, {Title: "song", TrackNumber: &n, DurationMs: 2}, {Title: "Other", TrackNumber: &n}}) + if len(copies) != 2 { + t.Errorf("a second copy of a track is read once: %+v", copies) + } if agrees(3, 4) || !agrees(4, 5) || agrees(0, 0) { t.Error("agrees: 80% of the album's tracks, and an album has tracks") } } + +// A refusal by an older matcher is matched again once the matching improves, +// even after the listing is done; a refusal by this one is not. +func TestListLibrary_RetriesAnOlderMatchersRefusals_Integration(t *testing.T) { + pool := newPool(t) + ctx := context.Background() + q := dbq.New(pool) + s := newService(t, pool, newFakeMB(t)) + ballad := seedAlbum(t, q, blur, "The Ballad of Darren", "", []string{"The Ballad", "St. Charles Square"}) + bootleg := seedAlbum(t, q, blur, "Bootleg", "", []string{"Intro"}) + for _, a := range []pgtype.UUID{ballad.ID, bootleg.ID} { + if err := q.RecordAlbumMatch(ctx, dbq.RecordAlbumMatchParams{AlbumID: a, Outcome: "unmatched", Detail: "old", Matcher: 1}); err != nil { + t.Fatal(err) + } + } + if _, err := pool.Exec(ctx, `UPDATE library_album_matches SET matcher = $1 WHERE album_id = $2`, libraryMatcher, bootleg.ID); err != nil { + t.Fatal(err) + } + // The reset leaves the listing done. + if err := s.listLibrary(ctx, &PassResult{}); err != nil { + t.Fatal(err) + } + if outcome, method, _, _ := matchOf(t, pool, ballad.ID); outcome != "listed" || method != matchDiscography { + t.Errorf("the older refusal was decided %s by %s", outcome, method) + } + if _, _, _, detail := matchOf(t, pool, bootleg.ID); detail != "old" { + t.Errorf("this matcher's own refusal was retried: %q", detail) + } + if state, _ := q.GetLibraryListing(ctx); state.LibraryUnlisted != 1 { + t.Errorf("unlisted = %d, want the bootleg alone", state.LibraryUnlisted) + } +} diff --git a/internal/wanted/match.go b/internal/wanted/match.go index 04c7527e..9d9d8c99 100644 --- a/internal/wanted/match.go +++ b/internal/wanted/match.go @@ -85,8 +85,13 @@ type albumMatch struct { Detail string } +// tracksToMatch reads an album's tracks, each once: a second copy of a +// track (the same disc, number and title, as a duplicate file in another +// format is) would otherwise count against the match as a track no edition +// has. func tracksToMatch(rows []dbq.ListAlbumTracksToMatchRow) []libraryTrack { out := make([]libraryTrack, 0, len(rows)) + seen := make(map[string]bool, len(rows)) for _, r := range rows { t := libraryTrack{Title: r.Title, LengthMs: int(r.DurationMs), Recording: deref(r.Mbid)} if r.DiscNumber != nil { @@ -95,6 +100,11 @@ func tracksToMatch(rows []dbq.ListAlbumTracksToMatchRow) []libraryTrack { if r.TrackNumber != nil { t.Number = int(*r.TrackNumber) } + key := fmt.Sprintf("%d/%d/%s", max(t.Disc, 1), t.Number, strings.Join(release.Tokens(t.Title), " ")) + if seen[key] { + continue + } + seen[key] = true out = append(out, t) } return out @@ -165,9 +175,15 @@ func agrees(n, tracks int) bool { return tracks > 0 && n*10 >= tracks*matchAgreement } -// sameTitle: the same words, not merely one holding the other. +// sameTitle: the same words, not merely one holding the other. "And", +// "&" and "+" join two titles however a release spells it ("Brain Stew + +// Jaded", "Brain Stew / Jaded"), so they count for nothing. func sameTitle(a, b string) bool { - return slices.Equal(release.Tokens(a), release.Tokens(b)) + words := func(s string) []string { + return slices.DeleteFunc(release.Tokens(s), func(w string) bool { return w == "and" }) + } + wa, wb := words(a), words(b) + return len(wa) > 0 && slices.Equal(wa, wb) } // matchAlbum finds the release group a library album is. A MusicBrainz @@ -204,7 +220,9 @@ func (s *Service) matchAlbum(ctx context.Context, a dbq.ListAlbumsToMatchRow, tr continue } tried[c.Group] = true - n, size, miss, err := s.agreementWith(ctx, c.Group, tracks, sameTitle(c.Title, a.Title)) + n, size, miss, err := s.agreementWith(ctx, c.Group, tracks, func(edition string) bool { + return sameTitle(c.Title, a.Title) || sameTitle(edition, a.Title) + }) if err != nil { return err } @@ -328,7 +346,9 @@ func (s *Service) searchCandidates(ctx context.Context, a dbq.ListAlbumsToMatchR // agreementWith is how many of the album's tracks the group's best edition // agrees with, that edition's length, and how it differs. A group // MusicBrainz no longer has, or lists no edition of, agrees with none (-1). -func (s *Service) agreementWith(ctx context.Context, group string, tracks []libraryTrack, titlesEnough bool) (n, size int, miss string, err error) { +// ownName says whether an edition, by its title, is called what the album +// is, so that its track titles are enough. +func (s *Service) agreementWith(ctx context.Context, group string, tracks []libraryTrack, ownName func(edition string) bool) (n, size int, miss string, err error) { eds, err := s.cat.Releases(ctx, group) switch { case errors.Is(err, catalogue.ErrNotFound): @@ -343,7 +363,7 @@ func (s *Service) agreementWith(ctx context.Context, group string, tracks []libr edition = append(edition, editionTrack{Medium: int(t.MediumPosition), Position: int(t.Position), Title: t.Title, LengthMs: int(t.LengthMs), Recording: t.RecordingMbid}) } - if got, m := trackAgreement(tracks, edition, titlesEnough); got > n || (got == n && abs(len(edition)-len(tracks)) < abs(size-len(tracks))) { + if got, m := trackAgreement(tracks, edition, ownName(e.Release.Title)); got > n || (got == n && abs(len(edition)-len(tracks)) < abs(size-len(tracks))) { n, size, miss = got, len(edition), m } } -- 2.54.0 From 1936fe77c647f8cda64df450a067dd33c5f845e8 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Sat, 10 Oct 2026 17:51:42 -0400 Subject: [PATCH 022/117] fix: the matcher-retry test stamps the listing's cutoff after its albums exist, as a listing done before them would not see them (M502 #5569) Co-Authored-By: Claude Opus 5.5 --- internal/wanted/library_test.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/internal/wanted/library_test.go b/internal/wanted/library_test.go index 029252ab..7f1a6187 100644 --- a/internal/wanted/library_test.go +++ b/internal/wanted/library_test.go @@ -299,7 +299,10 @@ func TestListLibrary_RetriesAnOlderMatchersRefusals_Integration(t *testing.T) { if _, err := pool.Exec(ctx, `UPDATE library_album_matches SET matcher = $1 WHERE album_id = $2`, libraryMatcher, bootleg.ID); err != nil { t.Fatal(err) } - // The reset leaves the listing done. + // A listing done, with these albums from before its cutoff. + if _, err := pool.Exec(ctx, `UPDATE acquisition_settings SET library_listing_started_at = now(), library_listed_at = now()`); err != nil { + t.Fatal(err) + } if err := s.listLibrary(ctx, &PassResult{}); err != nil { t.Fatal(err) } -- 2.54.0 From 7ef1fca90c58e60d1bc82a2b2c73dc27b8eea1f0 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Sat, 10 Oct 2026 18:14:56 -0400 Subject: [PATCH 023/117] fix: an album holding every track of an edition, and more besides, is that release group (M502 #5569) Savant's "Void" folder holds all 23 tracks of the VOID (DLC) edition plus 20 more, and was refused at 23 of 43. A whole edition of three or more tracks now matches whatever else the folder holds; singles and two-track releases still need most of the album's tracks, since many albums hold their songs. Matcher version 4 retries the refusals. Co-Authored-By: Claude Opus 5.5 --- internal/wanted/library.go | 3 ++- internal/wanted/library_test.go | 5 ++++- internal/wanted/match.go | 13 ++++++++----- 3 files changed, 14 insertions(+), 7 deletions(-) diff --git a/internal/wanted/library.go b/internal/wanted/library.go index ea56759c..051fbbf1 100644 --- a/internal/wanted/library.go +++ b/internal/wanted/library.go @@ -30,7 +30,8 @@ const libraryMatchBatch = 25 // 1 tags, release, discography and search, checked by title and length // 2 titles alone under the album's own name; a refusal names the track // 3 a second copy of a track counts once; joining words count for nothing -const libraryMatcher = 3 +// 4 an album holding every track of an edition, and more, is that group +const libraryMatcher = 4 // listLibrary puts the library's albums on the wanted list as held, once // (operator, 2026-10-10: "make a record saying that the import has diff --git a/internal/wanted/library_test.go b/internal/wanted/library_test.go index 7f1a6187..f503d67d 100644 --- a/internal/wanted/library_test.go +++ b/internal/wanted/library_test.go @@ -277,9 +277,12 @@ func TestTrackAgreement(t *testing.T) { if len(copies) != 2 { t.Errorf("a second copy of a track is read once: %+v", copies) } - if agrees(3, 4) || !agrees(4, 5) || agrees(0, 0) { + if agrees(3, 4, 0) || !agrees(4, 5, 0) || agrees(0, 0, 0) { t.Error("agrees: 80% of the album's tracks, and an album has tracks") } + if !agrees(23, 43, 23) || agrees(2, 10, 2) || agrees(22, 43, 23) { + t.Error("agrees: a whole edition of three or more tracks, with more besides") + } } // A refusal by an older matcher is matched again once the matching improves, diff --git a/internal/wanted/match.go b/internal/wanted/match.go index 9d9d8c99..f98f7b1f 100644 --- a/internal/wanted/match.go +++ b/internal/wanted/match.go @@ -170,9 +170,12 @@ func minutes(ms int) string { return fmt.Sprintf("%d:%02d", sec/60, sec%60) } -// agrees: enough of the album's tracks agree for the edition to be it. -func agrees(n, tracks int) bool { - return tracks > 0 && n*10 >= tracks*matchAgreement +// agrees: enough of the album's tracks agree for the edition to be it, or +// the album holds every track of the edition and more besides (a folder +// with the deluxe edition and its demos). A single or a two-track release +// is not taken on containment: too many albums hold its songs. +func agrees(n, tracks, edition int) bool { + return tracks > 0 && (n*10 >= tracks*matchAgreement || (edition >= 3 && n == edition)) } // sameTitle: the same words, not merely one holding the other. "And", @@ -243,7 +246,7 @@ func (s *Service) matchAlbum(ctx context.Context, a dbq.ListAlbumsToMatchRow, tr if err := check(discography); err != nil { return albumMatch{}, err } - if !agrees(bestN, len(tracks)) { + if !agrees(bestN, len(tracks), bestEdition) { found, err := s.searchCandidates(ctx, a) if err != nil { return albumMatch{}, err @@ -252,7 +255,7 @@ func (s *Service) matchAlbum(ctx context.Context, a dbq.ListAlbumsToMatchRow, tr return albumMatch{}, err } } - if agrees(bestN, len(tracks)) { + if agrees(bestN, len(tracks), bestEdition) { best.OK = true return best, nil } -- 2.54.0 From ca57133c8a7511a086ea9cbdca459ce638f94a67 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Sat, 10 Oct 2026 18:39:35 -0400 Subject: [PATCH 024/117] fix: a release group whose title names a live, remix or demo version is wanted only when the profile allows that version (M502 #5573) MusicBrainz types My Chemical Romance's "Common People (Recorded For BBC Radio 1)" as a plain Single, so a studio-only follow wanted it. Passes now reads the title's qualifier (brackets, or after a dash) for the version it names and treats it as a secondary type; a version no type names, an acoustic or instrumental one, no profile passes. The follow check takes off the list an open follow want the rule now refuses. Judge counts bbc, sessions and unplugged as live. Co-Authored-By: Claude Opus 5.5 --- internal/release/match.go | 2 +- internal/release/release_test.go | 2 + internal/release/version.go | 79 ++++++++++++++++++++++++++++++++ internal/release/version_test.go | 32 +++++++++++++ internal/wanted/profiles.go | 11 ++++- internal/wanted/wanted.go | 35 +++++++++++++- internal/wanted/wanted_test.go | 44 ++++++++++++------ 7 files changed, 188 insertions(+), 17 deletions(-) create mode 100644 internal/release/version.go create mode 100644 internal/release/version_test.go diff --git a/internal/release/match.go b/internal/release/match.go index 77ae1575..3fb25006 100644 --- a/internal/release/match.go +++ b/internal/release/match.go @@ -55,7 +55,7 @@ var versions = []struct { {[]string{"karaoke"}, "a karaoke version", ""}, {[]string{"acapella", "acappella", "accapella", "acapellas"}, "an a cappella version", ""}, {[]string{"remix", "remixes", "remixed"}, "a remix release", "Remix"}, - {[]string{"live", "bootleg"}, "a live recording", "Live"}, + {[]string{"live", "bootleg", "bbc", "sessions", "unplugged"}, "a live recording", "Live"}, {[]string{"demo", "demos"}, "demo recordings", "Demo"}, // Not "covers": in a release name that is usually the artwork scans. {[]string{"tribute", "lullaby", "lullabies"}, "a tribute release", ""}, diff --git a/internal/release/release_test.go b/internal/release/release_test.go index a5c07015..0bde6e41 100644 --- a/internal/release/release_test.go +++ b/internal/release/release_test.go @@ -62,6 +62,8 @@ func TestJudge_TheAlbumAndWhatIsNot(t *testing.T) { "Daft Punk - Random Access Memories (Instrumentals) [MP3]": VetoVersion, "Daft Punk - Random Access Memories Remixes": VetoVersion, "Daft Punk - Random Access Memories (Live) FLAC": VetoVersion, + "Daft Punk - Random Access Memories (BBC Sessions) FLAC": VetoVersion, + "Daft Punk - Random Access Memories Unplugged [MP3]": VetoVersion, "Daft Punk - Discovery (2001) [FLAC]": VetoTitle, "Random Access Memories (2013) [FLAC]": VetoArtist, "Daft Punk - Discography 1997-2013 FLAC": VetoTitle, diff --git a/internal/release/version.go b/internal/release/version.go new file mode 100644 index 00000000..c90dbf50 --- /dev/null +++ b/internal/release/version.go @@ -0,0 +1,79 @@ +package release + +import "strings" + +// TitleVersion is the version of a recording a release group's title names +// in its qualifier: the words in brackets, or after a dash. MusicBrainz +// does not always type such a group: My Chemical Romance's "Common People +// (Recorded For BBC Radio 1)" is a plain Single (#5573). Typ is the +// MusicBrainz secondary type the version is, or "" for one no type names +// (an acoustic or instrumental version); Word is the word that said so. +type TitleVersion struct { + Word string + What string + Typ string +} + +// titleVersions are the qualifier words that name a version, in the order +// they are looked for: "rough mix" is a demo before "mix" is a remix. +var titleVersions = []struct { + words []string + what string + typ string +}{ + {[]string{"demo", "demos", "outtake", "outtakes", "rehearsal", "rough mix"}, "demo recordings", "Demo"}, + {[]string{"live", "bootleg", "bbc", "session", "sessions", "unplugged", "recorded for", "in concert"}, "a live recording", "Live"}, + {[]string{"remix", "remixes", "remixed", "rmx", "mix", "mixes"}, "a remix", "Remix"}, + {[]string{"instrumental", "instrumentals", "acoustic", "karaoke", "acapella", "acappella", "a cappella", + "drumless", "nightcore", "slowed", "sped up"}, "another version", ""}, + {[]string{"commentary", "interview", "interviews"}, "commentary or interviews", "Interview"}, +} + +// VersionInTitle reads a release group's title for the version it names, +// if any. Only the qualifier counts, so "Live Through This" and "Live +// Forever" name no version. +func VersionInTitle(title string) (TitleVersion, bool) { + for _, q := range qualifiers(title) { + tokens := Tokens(q) + for _, v := range titleVersions { + for _, w := range v.words { + if contains(tokens, strings.Fields(w)) { + return TitleVersion{Word: w, What: v.what, Typ: v.typ}, true + } + } + } + } + return TitleVersion{}, false +} + +// qualifiers are the parts of a title in brackets, and what follows a +// spaced dash. +func qualifiers(title string) []string { + var out []string + depth, start := 0, 0 + for i, c := range title { + switch c { + case '(', '[', '{': + if depth == 0 { + start = i + 1 + } + depth++ + case ')', ']', '}': + if depth > 0 { + depth-- + if depth == 0 { + out = append(out, title[start:i]) + } + } + } + } + if depth > 0 { + out = append(out, title[start:]) + } + for _, dash := range []string{" - ", " – ", " — "} { + if i := strings.Index(title, dash); i >= 0 { + out = append(out, title[i+len(dash):]) + } + } + return out +} diff --git a/internal/release/version_test.go b/internal/release/version_test.go new file mode 100644 index 00000000..c6a877c9 --- /dev/null +++ b/internal/release/version_test.go @@ -0,0 +1,32 @@ +package release + +import "testing" + +func TestVersionInTitle(t *testing.T) { + for title, want := range map[string]string{ + // My Chemical Romance's groups MusicBrainz types as plain Singles (#5573). + "Common People (Recorded For BBC Radio 1)": "Live", + "Welcome to the Black Parade (Steve Aoki remix)": "Remix", + "The Five of Us Are Dying (Rough mix)": "Demo", + "I'm Not Okay (I Promise) [outtake version 2]": "Demo", + "Na Na Na (live from Sessions@AOL)": "Live", + "Song - Live at Wembley": "Live", + "Song – Club Mix": "Remix", + "Helena (Acoustic)": "", + "Teenagers (a cappella)": "", + "Hybrid Theory (20th Anniversary Edition)": "none", + "Live Through This": "none", + "Live Forever": "none", + "The Black Parade": "none", + "Warped Tour ’05: Bootleg Series (live at Warped Tour": "Live", // unclosed + } { + v, ok := VersionInTitle(title) + got := v.Typ + if !ok { + got = "none" + } + if got != want { + t.Errorf("VersionInTitle(%q) = %q (%q), want %q", title, got, v.Word, want) + } + } +} diff --git a/internal/wanted/profiles.go b/internal/wanted/profiles.go index d0a466b1..3cd86f3b 100644 --- a/internal/wanted/profiles.go +++ b/internal/wanted/profiles.go @@ -12,6 +12,7 @@ import ( "github.com/jackc/pgx/v5/pgtype" "git.fabledsword.com/bvandeusen/minstrel/internal/db/dbq" + "git.fabledsword.com/bvandeusen/minstrel/internal/release" ) // PrimaryTypes and SecondaryTypes are MusicBrainz's release-group types, in @@ -37,7 +38,12 @@ var ( // on profile p wants: its primary type is listed, and every secondary type // it carries is allowed. A group with no secondary type is a studio release, // which every profile passes on its primary type alone. -func Passes(p dbq.ReleaseTypeProfile, primary string, secondary []string) bool { +// +// A version its title names counts as a secondary type MusicBrainz left +// off (#5573): "Common People (Recorded For BBC Radio 1)" is live whatever +// its types say, and a version no type names, an acoustic one, no profile +// passes. +func Passes(p dbq.ReleaseTypeProfile, title, primary string, secondary []string) bool { if !containsFold(p.PrimaryTypes, primary) { return false } @@ -46,6 +52,9 @@ func Passes(p dbq.ReleaseTypeProfile, primary string, secondary []string) bool { return false } } + if v, ok := release.VersionInTitle(title); ok { + return v.Typ != "" && containsFold(p.SecondaryTypes, v.Typ) + } return true } diff --git a/internal/wanted/wanted.go b/internal/wanted/wanted.go index 6f21ec0c..3efbf930 100644 --- a/internal/wanted/wanted.go +++ b/internal/wanted/wanted.go @@ -21,6 +21,7 @@ import ( "git.fabledsword.com/bvandeusen/minstrel/internal/catalogue" "git.fabledsword.com/bvandeusen/minstrel/internal/db/dbq" "git.fabledsword.com/bvandeusen/minstrel/internal/musicbrainz" + "git.fabledsword.com/bvandeusen/minstrel/internal/release" "git.fabledsword.com/bvandeusen/minstrel/internal/tags" ) @@ -275,7 +276,13 @@ func (s *Service) applyFollow(ctx context.Context, q *dbq.Queries, f dbq.Follow) wanted := 0 for _, g := range groups { judge := (baseline && f.WantExisting) || (!baseline && !g.Seen && s.recent(g.FirstReleaseDate, f.BaselineAt.Time)) - if judge && Passes(profile, g.PrimaryType, g.SecondaryTypes) { + passes := Passes(profile, g.Title, g.PrimaryType, g.SecondaryTypes) + if !passes { + if err := s.dropRefusedVersion(ctx, q, g.Mbid, g.Title); err != nil { + return wanted, err + } + } + if judge && passes { held, err := q.LibraryHoldsReleaseGroup(ctx, dbq.LibraryHoldsReleaseGroupParams{ GroupMbid: g.Mbid, Title: g.Title, ArtistMbid: f.ArtistMbid, }) @@ -305,6 +312,32 @@ func (s *Service) applyFollow(ctx context.Context, q *dbq.Queries, f dbq.Follow) return wanted, q.MarkFollowChecked(ctx, f.ArtistMbid) } +// dropRefusedVersion takes off the list a group the follow wanted before +// Passes read the version its title names (#5573), while it is still only +// wanted: one the operator holds, or wanted for another reason, stays. +func (s *Service) dropRefusedVersion(ctx context.Context, q *dbq.Queries, group, title string) error { + v, ok := release.VersionInTitle(title) + if !ok { + return nil + } + w, err := q.GetWantByReleaseGroup(ctx, group) + if errors.Is(err, pgx.ErrNoRows) { + return nil + } + if err != nil { + return fmt.Errorf("read want %s: %w", title, err) + } + if w.Reason != ReasonFollow || w.FulfilledAt.Valid { + return nil + } + if _, err := q.DeleteWant(ctx, w.ID); err != nil { + return fmt.Errorf("drop want %s: %w", title, err) + } + s.logger.Info("wanted: a followed artist's release is another version, taken off the list", + "want", title, "version", v.What, "word", v.Word) + return nil +} + // recent reports whether a release dated date (MusicBrainz's partial date: // "2026", "2026-10", "2026-10-09", or empty when not yet known) came out no // earlier than NewReleaseGrace before the follow's baseline. A partial date diff --git a/internal/wanted/wanted_test.go b/internal/wanted/wanted_test.go index e0f5270e..a25e9cdf 100644 --- a/internal/wanted/wanted_test.go +++ b/internal/wanted/wanted_test.go @@ -30,21 +30,29 @@ func TestPasses_PrimaryListedAndEverySecondaryAllowed(t *testing.T) { withLive := dbq.ReleaseTypeProfile{PrimaryTypes: []string{"Album"}, SecondaryTypes: []string{"Live"}} for _, c := range []struct { profile dbq.ReleaseTypeProfile + title string primary string secondary []string want bool }{ - {standard, "Album", nil, true}, - {standard, "single", nil, true}, // MusicBrainz's case is not relied on - {standard, "Album", []string{"Live"}, false}, - {standard, "Broadcast", nil, false}, - {withLive, "Album", []string{"Live"}, true}, - {withLive, "Album", []string{"Live", "Compilation"}, false}, - {withLive, "EP", nil, false}, + {standard, "Revenge", "Album", nil, true}, + {standard, "Helena", "single", nil, true}, // MusicBrainz's case is not relied on + {standard, "Live at Koko", "Album", []string{"Live"}, false}, + {standard, "Radio Show", "Broadcast", nil, false}, + {withLive, "Live at Koko", "Album", []string{"Live"}, true}, + {withLive, "Live & Rare", "Album", []string{"Live", "Compilation"}, false}, + {withLive, "Like Phantoms, Forever", "EP", nil, false}, + // MusicBrainz types these plain Singles; their titles say otherwise (#5573). + {standard, "Common People (Recorded For BBC Radio 1)", "Single", nil, false}, + {standard, "Welcome to the Black Parade (Steve Aoki remix)", "Single", nil, false}, + {standard, "The Five of Us Are Dying (Rough mix)", "Single", nil, false}, + {withLive, "Desolation Row (live at BBC Maida Vale)", "Album", nil, true}, + {withLive, "Helena (acoustic)", "Album", nil, false}, + {standard, "Live Through This", "Album", nil, true}, } { - if got := Passes(c.profile, c.primary, c.secondary); got != c.want { - t.Errorf("Passes(%v/%v, %s %v) = %v, want %v", c.profile.PrimaryTypes, c.profile.SecondaryTypes, - c.primary, c.secondary, got, c.want) + if got := Passes(c.profile, c.title, c.primary, c.secondary); got != c.want { + t.Errorf("Passes(%v/%v, %q %s %v) = %v, want %v", c.profile.PrimaryTypes, c.profile.SecondaryTypes, + c.title, c.primary, c.secondary, got, c.want) } } } @@ -177,6 +185,7 @@ const ( theBallad = "11111111-0000-4000-8000-000000000003" // 2023 album, not held newSingle = "11111111-0000-4000-8000-000000000004" // appears after the baseline oldSingle = "11111111-0000-4000-8000-000000000005" // 1997 single, catalogued late + bbcSingle = "11111111-0000-4000-8000-000000000006" // live, typed a plain Single (#5573) parkRelease = "22222222-0000-4000-8000-000000000001" ) @@ -206,12 +215,13 @@ func newFakeMB(t *testing.T) *fakeMB { case r.URL.Path == "/release-group" && r.URL.Query().Get("artist") == blur: groups := group(parklife, "Parklife", "Album", "", "1994-04-25") + "," + group(liveAtWemb, "Live at Wembley Stadium", "Album", `"Live"`, "2009-07-02") + "," + - group(theBallad, "The Ballad of Darren", "Album", "", "2023-07-21") - n := 3 + group(theBallad, "The Ballad of Darren", "Album", "", "2023-07-21") + "," + + group(bbcSingle, "Song 2 (Recorded For BBC Radio 1)", "Single", "", "2023-08-01") + n := 4 if f.added.Load() { groups += "," + group(newSingle, "New Single", "Single", "", "") + "," + group(oldSingle, "Old Single", "Single", "", "1997-02-03") - n = 5 + n = 6 } _, _ = fmt.Fprintf(w, `{"release-group-count": %d, "release-groups": [%s]}`, n, groups) case r.URL.Path == "/release-group/"+theBallad: @@ -328,6 +338,11 @@ func TestFollow_WantExistingWantsWhatPassesAndIsNotHeld_Integration(t *testing.T f := newFakeMB(t) s := newService(t, pool, f) _ = seedHeld(t, q, "Parklife", parkRelease, parklife, 2) + // Wanted before the follow read the version a title names (#5573). + if _, err := WantWith(ctx, q, WantParams{GroupMBID: bbcSingle, ArtistMBID: blur, ArtistName: "Blur", + Title: "Song 2 (Recorded For BBC Radio 1)", PrimaryType: "Single", Reason: ReasonFollow}); err != nil { + t.Fatal(err) + } if _, err := s.Follow(ctx, FollowParams{ArtistMBID: blur, ArtistName: "Blur", WantExisting: true}); err != nil { t.Fatal(err) @@ -338,7 +353,8 @@ func TestFollow_WantExistingWantsWhatPassesAndIsNotHeld_Integration(t *testing.T if _, err := q.GetWantByReleaseGroup(ctx, theBallad); err != nil { t.Errorf("the studio album not held should be wanted: %v", err) } - for group, why := range map[string]string{parklife: "held", liveAtWemb: "live, and Standard is studio only"} { + for group, why := range map[string]string{parklife: "held", liveAtWemb: "live, and Standard is studio only", + bbcSingle: "recorded for the BBC, which its title says, and Standard is studio only"} { if _, err := q.GetWantByReleaseGroup(ctx, group); err == nil { t.Errorf("%s wanted, but it is %s", group, why) } -- 2.54.0 From a4fbe44bd5c70fd816fa33c2b5846fc90bdc99cd Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Sat, 10 Oct 2026 18:43:32 -0400 Subject: [PATCH 025/117] =?UTF-8?q?feat:=20the=20Wanted=20view=20lists=20a?= =?UTF-8?q?rtists=20A=E2=80=93Z=20with=20their=20albums=20inside,=20each?= =?UTF-8?q?=20album's=20tracks=20a=20click=20away=20(M502=20#5572)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The list runs like the library: artists by the library's sort name, each artist's albums oldest first, grouped under the artist (linked into the library when it holds them). A filter narrows it to an artist or album by name, on the server, so it finds what has not been scrolled to. Tracks opens an album's track list: the library's own for an album it holds, a lost file marked, otherwise MusicBrainz's standard edition, named. Co-Authored-By: Claude Opus 5.5 --- internal/api/admin_wanted.go | 136 +++++++++- internal/api/admin_wanted_test.go | 213 ++++++++++++++++ internal/api/api.go | 1 + internal/db/dbq/wanted.sql.go | 89 ++++++- internal/db/queries/wanted.sql | 28 ++- web/src/lib/api/queries.ts | 3 +- web/src/lib/api/types.ts | 7 + web/src/lib/api/wanted.ts | 32 ++- web/src/lib/components/wanted/WantList.svelte | 235 +++++++++++------- .../lib/components/wanted/WantList.test.ts | 72 +++++- .../lib/components/wanted/WantTracks.svelte | 52 ++++ 11 files changed, 753 insertions(+), 115 deletions(-) create mode 100644 internal/api/admin_wanted_test.go create mode 100644 web/src/lib/components/wanted/WantTracks.svelte diff --git a/internal/api/admin_wanted.go b/internal/api/admin_wanted.go index 7ed0b292..852d9cda 100644 --- a/internal/api/admin_wanted.go +++ b/internal/api/admin_wanted.go @@ -4,12 +4,14 @@ import ( "errors" "net/http" "strconv" + "strings" "github.com/go-chi/chi/v5" "github.com/jackc/pgx/v5" "github.com/jackc/pgx/v5/pgtype" "git.fabledsword.com/bvandeusen/minstrel/internal/apierror" + "git.fabledsword.com/bvandeusen/minstrel/internal/catalogue" "git.fabledsword.com/bvandeusen/minstrel/internal/db/dbq" "git.fabledsword.com/bvandeusen/minstrel/internal/musicbrainz" "git.fabledsword.com/bvandeusen/minstrel/internal/wanted" @@ -111,8 +113,9 @@ const unmatchedShown = 200 const wantsPageSize = 50 -// handleListWants implements GET /api/admin/wanted?state=wanted|missing|held&offset=. -// wanted is everything open but missing files; held is newest first. +// handleListWants implements GET /api/admin/wanted?state=wanted|missing|held&q=&offset=. +// wanted is everything open but missing files. Each list runs artist by +// artist, A–Z, like the library (#5572); q narrows it to an artist or album. func (h *handlers) handleListWants(w http.ResponseWriter, r *http.Request) { state := r.URL.Query().Get("state") if state == "" { @@ -127,7 +130,8 @@ func (h *handlers) handleListWants(w http.ResponseWriter, r *http.Request) { offset = 0 } q := dbq.New(h.pool) - rows, err := q.ListWants(r.Context(), dbq.ListWantsParams{State: state, Skip: int32(offset), MaxRows: wantsPageSize}) + filter := strings.TrimSpace(r.URL.Query().Get("q")) + rows, err := q.ListWants(r.Context(), dbq.ListWantsParams{State: state, Q: filter, Skip: int32(offset), MaxRows: wantsPageSize}) if err != nil { writeErrWithLog(w, h.logger, "admin wanted: list", apierror.Internal(err)) return @@ -173,6 +177,12 @@ func (h *handlers) handleListWants(w http.ResponseWriter, r *http.Request) { default: out.Total = counts.Wanted } + if filter != "" { + out.Total = 0 + if len(rows) > 0 { + out.Total = rows[0].Total + } + } for _, row := range rows { v := wantView{ ID: uuidToString(row.ID), ReleaseGroupMBID: row.ReleaseGroupMbid, Title: row.Title, @@ -227,6 +237,126 @@ func (h *handlers) handleListWants(w http.ResponseWriter, r *http.Request) { writeJSON(w, http.StatusOK, out) } +// wantTracksView is an album's track list for the Wanted view (#5572): the +// library's own tracks for an album it holds or held, else MusicBrainz's +// standard edition of what is wanted. +type wantTracksView struct { + // library, musicbrainz, or none when MusicBrainz lists no edition. + Source string `json:"source"` + // Edition names the MusicBrainz release the list is from. + Edition string `json:"edition,omitempty"` + Tracks []wantTrackView `json:"tracks"` +} + +type wantTrackView struct { + Disc int32 `json:"disc"` + Number int32 `json:"number"` + Title string `json:"title"` + LengthMs int32 `json:"length_ms"` + // Missing: the library's file for it is gone. + Missing bool `json:"missing"` +} + +// handleWantTracks implements GET /api/admin/wanted/{id}/tracks. +func (h *handlers) handleWantTracks(w http.ResponseWriter, r *http.Request) { + id, ok := requireURLUUID(w, r, "id") + if !ok { + return + } + q := dbq.New(h.pool) + want, err := q.GetWantAlbum(r.Context(), id) + if errors.Is(err, pgx.ErrNoRows) { + writeErr(w, apierror.NotFound("want")) + return + } + if err != nil { + writeErrWithLog(w, h.logger, "admin wanted: tracks", apierror.Internal(err)) + return + } + out := wantTracksView{Tracks: []wantTrackView{}} + if want.AlbumID.Valid { + rows, err := q.ListLibraryAlbumTracks(r.Context(), want.AlbumID) + if err != nil { + writeErrWithLog(w, h.logger, "admin wanted: library tracks", apierror.Internal(err)) + return + } + out.Source = "library" + for _, t := range rows { + v := wantTrackView{Title: t.Title, LengthMs: t.DurationMs, Missing: t.Missing} + if t.DiscNumber != nil { + v.Disc = *t.DiscNumber + } + if t.TrackNumber != nil { + v.Number = *t.TrackNumber + } + out.Tracks = append(out.Tracks, v) + } + writeJSON(w, http.StatusOK, out) + return + } + eds, err := h.catalogue.Releases(r.Context(), want.ReleaseGroupMbid) + switch { + case errors.Is(err, catalogue.ErrNotFound): + out.Source = "none" + writeJSON(w, http.StatusOK, out) + return + case errors.Is(err, musicbrainz.ErrUnreachable), errors.Is(err, musicbrainz.ErrBadResponse): + writeErr(w, &apierror.Error{Status: 503, Code: "musicbrainz_unreachable", Message: "MusicBrainz can't be reached", Cause: err}) + return + case err != nil: + writeErrWithLog(w, h.logger, "admin wanted: editions", apierror.Internal(err)) + return + } + e, ok := standardEdition(eds.Editions) + if !ok { + out.Source = "none" + writeJSON(w, http.StatusOK, out) + return + } + out.Source = "musicbrainz" + out.Edition = editionLabel(e.Release) + for _, t := range e.Tracks { + out.Tracks = append(out.Tracks, wantTrackView{Disc: t.MediumPosition, Number: t.Position, Title: t.Title, LengthMs: t.LengthMs}) + } + writeJSON(w, http.StatusOK, out) +} + +// standardEdition is the edition a person would call the album: an official +// one with the fewest tracks, else any with the fewest. The catalogue lists +// official editions first by date, so a tie goes to the earliest. +func standardEdition(eds []catalogue.Edition) (catalogue.Edition, bool) { + best := -1 + for i, e := range eds { + if len(e.Tracks) == 0 { + continue + } + if best < 0 { + best = i + continue + } + b := eds[best] + bo, eo := b.Release.Status == "Official", e.Release.Status == "Official" + if (eo && !bo) || (eo == bo && len(e.Tracks) < len(b.Tracks)) { + best = i + } + } + if best < 0 { + return catalogue.Edition{}, false + } + return eds[best], true +} + +// editionLabel names a release for a person: "2023 · CD · GB". +func editionLabel(r dbq.MbRelease) string { + var parts []string + for _, p := range []string{first4(r.ReleaseDate), r.Formats, r.Country} { + if p != "" { + parts = append(parts, p) + } + } + return strings.Join(parts, " · ") +} + func first4(date string) string { if len(date) < 4 { return "" diff --git a/internal/api/admin_wanted_test.go b/internal/api/admin_wanted_test.go new file mode 100644 index 00000000..db7f9baa --- /dev/null +++ b/internal/api/admin_wanted_test.go @@ -0,0 +1,213 @@ +package api + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "net/url" + "testing" + + "github.com/go-chi/chi/v5" + + "git.fabledsword.com/bvandeusen/minstrel/internal/auth" + "git.fabledsword.com/bvandeusen/minstrel/internal/catalogue" + "git.fabledsword.com/bvandeusen/minstrel/internal/db/dbq" + "git.fabledsword.com/bvandeusen/minstrel/internal/musicbrainz" + "git.fabledsword.com/bvandeusen/minstrel/internal/wanted" +) + +func doWantedReq(t *testing.T, h *handlers, path string, user dbq.User) *httptest.ResponseRecorder { + t.Helper() + r := chi.NewRouter() + r.Route("/api/admin", func(admin chi.Router) { + admin.Use(auth.RequireAdmin()) + admin.Get("/wanted", h.handleListWants) + admin.Get("/wanted/{id}/tracks", h.handleWantTracks) + }) + w := httptest.NewRecorder() + r.ServeHTTP(w, withUser(httptest.NewRequest(http.MethodGet, path, nil), user)) + return w +} + +func seedWant(t *testing.T, h *handlers, group, artistMBID, artist, title, date string) dbq.Want { + t.Helper() + w, err := wanted.WantWith(context.Background(), dbq.New(h.pool), wanted.WantParams{ + GroupMBID: group, ArtistMBID: artistMBID, ArtistName: artist, Title: title, + PrimaryType: "Album", FirstReleaseDate: date, Reason: wanted.ReasonFollow, + }) + if err != nil { + t.Fatal(err) + } + return w +} + +func listWanted(t *testing.T, h *handlers, user dbq.User, q string) wantsPage { + t.Helper() + w := doWantedReq(t, h, "/api/admin/wanted?"+url.Values{"q": {q}}.Encode(), user) + if w.Code != http.StatusOK { + t.Fatalf("status = %d; body = %s", w.Code, w.Body.String()) + } + var page wantsPage + if err := json.Unmarshal(w.Body.Bytes(), &page); err != nil { + t.Fatal(err) + } + return page +} + +// The list runs like the library (#5572): artists A–Z by the library's sort +// name where the library holds the artist, each artist's albums oldest +// first; q narrows it to an artist or an album. +func TestListWants_ArtistsAToZThenTheirAlbumsByDate(t *testing.T) { + h, _ := testHandlersWithClientFn(t) + admin := seedUser(t, h.pool, "admin", "pw", true) + ctx := context.Background() + beatles := "a0000000-0000-4000-8000-000000000002" + if _, err := dbq.New(h.pool).UpsertArtist(ctx, dbq.UpsertArtistParams{Name: "The Beatles", SortName: "Beatles, The", Mbid: &beatles}); err != nil { + t.Fatal(err) + } + seedWant(t, h, "g0000000-0000-4000-8000-000000000001", "a0000000-0000-4000-8000-000000000003", "Zeds", "Zed Album", "2001") + seedWant(t, h, "g0000000-0000-4000-8000-000000000002", beatles, "The Beatles", "Abbey Road", "1969-09-26") + seedWant(t, h, "g0000000-0000-4000-8000-000000000003", beatles, "The Beatles", "Please Please Me", "1963-03-22") + seedWant(t, h, "g0000000-0000-4000-8000-000000000004", "a0000000-0000-4000-8000-000000000001", "Camel", "Moonmadness", "1976") + + page := listWanted(t, h, admin, "") + var got []string + for _, it := range page.Items { + got = append(got, it.ArtistName+"/"+it.Title) + } + want := "[The Beatles/Please Please Me The Beatles/Abbey Road Camel/Moonmadness Zeds/Zed Album]" + if fmt.Sprint(got) != want { + t.Errorf("order = %v, want %s", got, want) + } + if page.Items[0].ArtistID == "" || page.Items[2].ArtistID != "" { + t.Errorf("artist ids = %q, %q: only an artist the library holds has one", page.Items[0].ArtistID, page.Items[2].ArtistID) + } + + page = listWanted(t, h, admin, "BEATLES") + if len(page.Items) != 2 || page.Total != 2 { + t.Errorf("q=BEATLES: %d items, total %d; want the two Beatles albums", len(page.Items), page.Total) + } + page = listWanted(t, h, admin, "moon") + if len(page.Items) != 1 || page.Items[0].Title != "Moonmadness" || page.Total != 1 { + t.Errorf("q=moon: %+v, total %d", page.Items, page.Total) + } + if page = listWanted(t, h, admin, "nothing like it"); len(page.Items) != 0 || page.Total != 0 { + t.Errorf("no match: %d items, total %d", len(page.Items), page.Total) + } +} + +// fakeEditions is a MusicBrainz holding one group with two editions, the +// shorter official one the standard, or 503 when down is set. +func fakeEditions(t *testing.T, group string, down bool) *httptest.Server { + t.Helper() + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if down { + w.WriteHeader(http.StatusServiceUnavailable) + return + } + switch { + case r.URL.Path == "/release-group/"+group: + _, _ = fmt.Fprintf(w, `{"id": %q, "title": "Moonmadness", "primary-type": "Album", "first-release-date": "1976"}`, group) + case r.URL.Path == "/release" && r.URL.Query().Get("release-group") == group: + _, _ = fmt.Fprint(w, `{"release-count": 2, "releases": [ + {"id": "r-deluxe", "title": "Moonmadness", "status": "Official", "date": "2009", "country": "GB", + "media": [{"position": 1, "format": "CD", "tracks": [ + {"position": 1, "number": "1", "title": "Aristillus", "length": 120000, "recording": {"id": "rec-1"}}, + {"position": 2, "number": "2", "title": "Song Within a Song", "length": 433000, "recording": {"id": "rec-2"}}, + {"position": 3, "number": "3", "title": "Bonus", "length": 200000, "recording": {"id": "rec-3"}}]}]}, + {"id": "r-orig", "title": "Moonmadness", "status": "Official", "date": "1976-03", "country": "GB", + "media": [{"position": 1, "format": "12\" Vinyl", "tracks": [ + {"position": 1, "number": "A1", "title": "Aristillus", "length": 120000, "recording": {"id": "rec-1"}}, + {"position": 2, "number": "A2", "title": "Song Within a Song", "length": 433000, "recording": {"id": "rec-2"}}]}]}]}`) + default: + w.WriteHeader(http.StatusNotFound) + } + })) + t.Cleanup(srv.Close) + return srv +} + +func wantTracks(t *testing.T, h *handlers, user dbq.User, id string) (int, wantTracksView) { + t.Helper() + w := doWantedReq(t, h, "/api/admin/wanted/"+id+"/tracks", user) + var out wantTracksView + if w.Code == http.StatusOK { + if err := json.Unmarshal(w.Body.Bytes(), &out); err != nil { + t.Fatal(err) + } + } + return w.Code, out +} + +// An album not yet held shows MusicBrainz's standard edition: the shortest +// official one. +func TestWantTracks_StandardEditionFromMusicBrainz(t *testing.T) { + const group = "g0000000-0000-4000-8000-000000000010" + for _, down := range []bool{false, true} { + h, pool := testHandlersWithClientFn(t) + admin := seedUser(t, h.pool, "admin", "pw", true) + h.catalogue = catalogue.New(pool, &musicbrainz.Client{BaseURL: fakeEditions(t, group, down).URL, + Pace: &musicbrainz.Pacer{}, Retries: -1}, h.logger) + w := seedWant(t, h, group, "a0000000-0000-4000-8000-000000000001", "Camel", "Moonmadness", "1976") + + code, out := wantTracks(t, h, admin, uuidToString(w.ID)) + if down { + if code != http.StatusServiceUnavailable { + t.Errorf("MusicBrainz down: status %d, want 503", code) + } + continue + } + if code != http.StatusOK || out.Source != "musicbrainz" || len(out.Tracks) != 2 { + t.Fatalf("status %d, %+v: want the two-track original", code, out) + } + if out.Edition != "1976 · 12\" Vinyl · GB" { + t.Errorf("edition = %q", out.Edition) + } + if tr := out.Tracks[1]; tr.Disc != 1 || tr.Number != 2 || tr.Title != "Song Within a Song" || tr.LengthMs != 433000 { + t.Errorf("track 2 = %+v", tr) + } + } +} + +// A held album shows the library's own tracks, a lost file marked. +func TestWantTracks_HeldAlbumShowsTheLibrarysTracks(t *testing.T) { + h, _ := testHandlersWithClientFn(t) + admin := seedUser(t, h.pool, "admin", "pw", true) + ctx := context.Background() + q := dbq.New(h.pool) + mbid := "a0000000-0000-4000-8000-000000000001" + artist, err := q.UpsertArtist(ctx, dbq.UpsertArtistParams{Name: "Camel", SortName: "Camel", Mbid: &mbid}) + if err != nil { + t.Fatal(err) + } + album, err := q.UpsertAlbum(ctx, dbq.UpsertAlbumParams{Title: "Moonmadness", SortTitle: "Moonmadness", ArtistID: artist.ID}) + if err != nil { + t.Fatal(err) + } + for i, title := range []string{"Aristillus", "Song Within a Song"} { + num := int32(i + 1) + if _, err := q.UpsertTrack(ctx, dbq.UpsertTrackParams{ + Title: title, AlbumID: album.ID, ArtistID: artist.ID, TrackNumber: &num, DurationMs: 120000, + FilePath: fmt.Sprintf("/music/camel/%02d.flac", i+1), FileSize: 1, FileFormat: "flac", + }); err != nil { + t.Fatal(err) + } + } + if _, err := h.pool.Exec(ctx, `UPDATE tracks SET missing_since = now() WHERE title = 'Aristillus'`); err != nil { + t.Fatal(err) + } + w := seedWant(t, h, "g0000000-0000-4000-8000-000000000011", mbid, "Camel", "Moonmadness", "1976") + if err := q.FulfillWant(ctx, dbq.FulfillWantParams{ID: w.ID, AlbumID: album.ID}); err != nil { + t.Fatal(err) + } + + code, out := wantTracks(t, h, admin, uuidToString(w.ID)) + if code != http.StatusOK || out.Source != "library" || len(out.Tracks) != 2 { + t.Fatalf("status %d, %+v", code, out) + } + if !out.Tracks[0].Missing || out.Tracks[1].Missing || out.Tracks[1].Number != 2 { + t.Errorf("tracks = %+v: the first is missing, the second on disk", out.Tracks) + } +} diff --git a/internal/api/api.go b/internal/api/api.go index 1b7a0225..68825e18 100644 --- a/internal/api/api.go +++ b/internal/api/api.go @@ -257,6 +257,7 @@ func Mount(r chi.Router, pool *pgxpool.Pool, logger *slog.Logger, events *playev admin.Get("/acquisition/settings", h.handleGetSearchSettings) admin.Put("/acquisition/settings", h.handlePutSearchSettings) admin.Get("/wanted/{id}/candidates", h.handleListCandidates) + admin.Get("/wanted/{id}/tracks", h.handleWantTracks) admin.Post("/wanted/{id}/search", h.handleSearchWant) admin.Post("/wanted/{id}/grab", h.handleGrabCandidate) admin.Post("/wanted/{id}/block", h.handleBlockCandidate) diff --git a/internal/db/dbq/wanted.sql.go b/internal/db/dbq/wanted.sql.go index 4699d712..c33da297 100644 --- a/internal/db/dbq/wanted.sql.go +++ b/internal/db/dbq/wanted.sql.go @@ -273,6 +273,24 @@ func (q *Queries) GetReleaseTypeProfile(ctx context.Context, id pgtype.UUID) (Re return i, err } +const getWantAlbum = `-- name: GetWantAlbum :one +SELECT id, release_group_mbid, album_id FROM wants WHERE id = $1 +` + +type GetWantAlbumRow struct { + ID pgtype.UUID + ReleaseGroupMbid string + AlbumID pgtype.UUID +} + +// What the Wanted view's track list needs of a want (#5572). +func (q *Queries) GetWantAlbum(ctx context.Context, id pgtype.UUID) (GetWantAlbumRow, error) { + row := q.db.QueryRow(ctx, getWantAlbum, id) + var i GetWantAlbumRow + err := row.Scan(&i.ID, &i.ReleaseGroupMbid, &i.AlbumID) + return i, err +} + const getWantByReleaseGroup = `-- name: GetWantByReleaseGroup :one SELECT id, release_group_mbid, artist_mbid, artist_name, title, primary_type, first_release_date, reason, album_id, created_by, wanted_at, fulfilled_at, created_at, updated_at, searched_at, search_outcome, search_considered, best_release, best_verdict, search_error FROM wants WHERE release_group_mbid = $1 ` @@ -616,6 +634,49 @@ func (q *Queries) ListFollowsDue(ctx context.Context, arg ListFollowsDueParams) return items, nil } +const listLibraryAlbumTracks = `-- name: ListLibraryAlbumTracks :many +SELECT title, disc_number, track_number, duration_ms, (missing_since IS NOT NULL)::bool AS missing + FROM tracks + WHERE album_id = $1 + ORDER BY disc_number NULLS FIRST, track_number NULLS LAST, title +` + +type ListLibraryAlbumTracksRow struct { + Title string + DiscNumber *int32 + TrackNumber *int32 + DurationMs int32 + Missing bool +} + +// A held or missing album's own tracks, for the Wanted view's track list; +// missing says its file is gone. +func (q *Queries) ListLibraryAlbumTracks(ctx context.Context, albumID pgtype.UUID) ([]ListLibraryAlbumTracksRow, error) { + rows, err := q.db.Query(ctx, listLibraryAlbumTracks, albumID) + if err != nil { + return nil, err + } + defer rows.Close() + var items []ListLibraryAlbumTracksRow + for rows.Next() { + var i ListLibraryAlbumTracksRow + if err := rows.Scan( + &i.Title, + &i.DiscNumber, + &i.TrackNumber, + &i.DurationMs, + &i.Missing, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + const listLibraryAlbums = `-- name: ListLibraryAlbums :execrows INSERT INTO wants (release_group_mbid, artist_mbid, artist_name, title, reason, album_id, wanted_at, fulfilled_at) SELECT DISTINCT ON (a.release_group_mbid) @@ -800,7 +861,8 @@ SELECT w.id, w.release_group_mbid, w.artist_mbid, w.artist_name, w.title, w.reas COALESCE(g.secondary_types, '{}')::text[] AS secondary_types, COALESCE(NULLIF(w.first_release_date, ''), g.first_release_date, '')::text AS first_release_date, u.username AS requested_by, - (SELECT ar.id FROM artists ar WHERE ar.mbid = w.artist_mbid) AS library_artist_id, + la.id AS library_artist_id, + COUNT(*) OVER () AS total, -- The search schedule's last look (#5521), and the download running. w.searched_at, w.search_outcome, w.search_considered, w.best_release, w.best_verdict, w.search_error, -- Its latest download, running or ended, and what the import made of @@ -813,6 +875,7 @@ SELECT w.id, w.release_group_mbid, w.artist_mbid, w.artist_name, w.title, w.reas FROM wants w LEFT JOIN users u ON u.id = w.created_by LEFT JOIN mb_release_groups g ON g.mbid = w.release_group_mbid + LEFT JOIN LATERAL (SELECT ar.id, ar.sort_name FROM artists ar WHERE ar.mbid = w.artist_mbid LIMIT 1) la ON true LEFT JOIN grabs gr ON gr.id = (SELECT x.id FROM grabs x WHERE x.want_id = w.id ORDER BY x.ended_at IS NULL DESC, x.grabbed_at DESC LIMIT 1) WHERE CASE $1::text @@ -820,12 +883,18 @@ SELECT w.id, w.release_group_mbid, w.artist_mbid, w.artist_name, w.title, w.reas WHEN 'missing' THEN w.fulfilled_at IS NULL AND w.reason = 'missing' ELSE w.fulfilled_at IS NULL AND w.reason <> 'missing' END - ORDER BY COALESCE(w.fulfilled_at, w.wanted_at) DESC, w.id - LIMIT $3 OFFSET $2 + AND ($2::text = '' + OR strpos(lower(w.artist_name), lower($2::text)) > 0 + OR strpos(lower(w.title), lower($2::text)) > 0) + ORDER BY lower(COALESCE(NULLIF(la.sort_name, ''), w.artist_name)), w.artist_mbid, + NULLIF(COALESCE(NULLIF(w.first_release_date, ''), g.first_release_date, ''), '') NULLS LAST, + lower(w.title), w.id + LIMIT $4 OFFSET $3 ` type ListWantsParams struct { State string + Q string Skip int32 MaxRows int32 } @@ -845,6 +914,7 @@ type ListWantsRow struct { FirstReleaseDate string RequestedBy *string LibraryArtistID pgtype.UUID + Total int64 SearchedAt pgtype.Timestamptz SearchOutcome string SearchConsidered int32 @@ -867,9 +937,17 @@ type ListWantsRow struct { } // The Wanted page: open wants (all but missing ones, or only missing ones), -// or what was held most recently. +// or the held ones. Ordered like the library (#5572): artists A–Z by the +// library's sort name when it holds the artist, each artist's albums by +// release date, so the page groups them as they arrive. q narrows to an +// artist or album title; total counts every match, past the page. func (q *Queries) ListWants(ctx context.Context, arg ListWantsParams) ([]ListWantsRow, error) { - rows, err := q.db.Query(ctx, listWants, arg.State, arg.Skip, arg.MaxRows) + rows, err := q.db.Query(ctx, listWants, + arg.State, + arg.Q, + arg.Skip, + arg.MaxRows, + ) if err != nil { return nil, err } @@ -892,6 +970,7 @@ func (q *Queries) ListWants(ctx context.Context, arg ListWantsParams) ([]ListWan &i.FirstReleaseDate, &i.RequestedBy, &i.LibraryArtistID, + &i.Total, &i.SearchedAt, &i.SearchOutcome, &i.SearchConsidered, diff --git a/internal/db/queries/wanted.sql b/internal/db/queries/wanted.sql index 49077412..173e582c 100644 --- a/internal/db/queries/wanted.sql +++ b/internal/db/queries/wanted.sql @@ -177,7 +177,10 @@ DELETE FROM wants WHERE id = $1; -- name: ListWants :many -- The Wanted page: open wants (all but missing ones, or only missing ones), --- or what was held most recently. +-- or the held ones. Ordered like the library (#5572): artists A–Z by the +-- library's sort name when it holds the artist, each artist's albums by +-- release date, so the page groups them as they arrive. q narrows to an +-- artist or album title; total counts every match, past the page. SELECT w.id, w.release_group_mbid, w.artist_mbid, w.artist_name, w.title, w.reason, w.album_id, w.wanted_at, w.fulfilled_at, -- A request names an album before the catalogue has read it: the @@ -186,7 +189,8 @@ SELECT w.id, w.release_group_mbid, w.artist_mbid, w.artist_name, w.title, w.reas COALESCE(g.secondary_types, '{}')::text[] AS secondary_types, COALESCE(NULLIF(w.first_release_date, ''), g.first_release_date, '')::text AS first_release_date, u.username AS requested_by, - (SELECT ar.id FROM artists ar WHERE ar.mbid = w.artist_mbid) AS library_artist_id, + la.id AS library_artist_id, + COUNT(*) OVER () AS total, -- The search schedule's last look (#5521), and the download running. w.searched_at, w.search_outcome, w.search_considered, w.best_release, w.best_verdict, w.search_error, -- Its latest download, running or ended, and what the import made of @@ -199,6 +203,7 @@ SELECT w.id, w.release_group_mbid, w.artist_mbid, w.artist_name, w.title, w.reas FROM wants w LEFT JOIN users u ON u.id = w.created_by LEFT JOIN mb_release_groups g ON g.mbid = w.release_group_mbid + LEFT JOIN LATERAL (SELECT ar.id, ar.sort_name FROM artists ar WHERE ar.mbid = w.artist_mbid LIMIT 1) la ON true LEFT JOIN grabs gr ON gr.id = (SELECT x.id FROM grabs x WHERE x.want_id = w.id ORDER BY x.ended_at IS NULL DESC, x.grabbed_at DESC LIMIT 1) WHERE CASE sqlc.arg(state)::text @@ -206,9 +211,26 @@ SELECT w.id, w.release_group_mbid, w.artist_mbid, w.artist_name, w.title, w.reas WHEN 'missing' THEN w.fulfilled_at IS NULL AND w.reason = 'missing' ELSE w.fulfilled_at IS NULL AND w.reason <> 'missing' END - ORDER BY COALESCE(w.fulfilled_at, w.wanted_at) DESC, w.id + AND (sqlc.arg(q)::text = '' + OR strpos(lower(w.artist_name), lower(sqlc.arg(q)::text)) > 0 + OR strpos(lower(w.title), lower(sqlc.arg(q)::text)) > 0) + ORDER BY lower(COALESCE(NULLIF(la.sort_name, ''), w.artist_name)), w.artist_mbid, + NULLIF(COALESCE(NULLIF(w.first_release_date, ''), g.first_release_date, ''), '') NULLS LAST, + lower(w.title), w.id LIMIT sqlc.arg(max_rows) OFFSET sqlc.arg(skip); +-- name: GetWantAlbum :one +-- What the Wanted view's track list needs of a want (#5572). +SELECT id, release_group_mbid, album_id FROM wants WHERE id = $1; + +-- name: ListLibraryAlbumTracks :many +-- A held or missing album's own tracks, for the Wanted view's track list; +-- missing says its file is gone. +SELECT title, disc_number, track_number, duration_ms, (missing_since IS NOT NULL)::bool AS missing + FROM tracks + WHERE album_id = $1 + ORDER BY disc_number NULLS FIRST, track_number NULLS LAST, title; + -- name: CountWants :one SELECT COUNT(*) FILTER (WHERE fulfilled_at IS NULL AND reason <> 'missing')::bigint AS wanted, COUNT(*) FILTER (WHERE fulfilled_at IS NULL AND reason = 'missing')::bigint AS missing, diff --git a/web/src/lib/api/queries.ts b/web/src/lib/api/queries.ts index 2877ca37..b70e78f4 100644 --- a/web/src/lib/api/queries.ts +++ b/web/src/lib/api/queries.ts @@ -41,7 +41,8 @@ export const qk = { ['adminRequests', { status: status ?? 'all' }] as const, // Wanted list (M502 #5520). Wants sit under ['adminWants'] so one prefix // invalidation refreshes every tab and its counts. - adminWants: (state: string) => ['adminWants', { state }] as const, + adminWants: (state: string, q: string = '') => ['adminWants', { state, q }] as const, + wantTracks: (id: string) => ['wantTracks', id] as const, adminFollows: () => ['adminFollows'] as const, adminFollow: (artistMbid: string) => ['adminFollow', artistMbid] as const, releaseProfiles: () => ['releaseProfiles'] as const, diff --git a/web/src/lib/api/types.ts b/web/src/lib/api/types.ts index 957fc4a3..9e3950ff 100644 --- a/web/src/lib/api/types.ts +++ b/web/src/lib/api/types.ts @@ -821,6 +821,13 @@ export type LibraryListing = { listed_at: string | null; unlisted: number; unmat export type WantsPage = Page & { counts: WantCounts; library: LibraryListing }; +// An album's tracks in the Wanted view (#5572): the library's own for an +// album it holds (a lost file marked missing), else MusicBrainz's standard +// edition, named by edition; none when MusicBrainz lists no edition. +export type WantTrack = { disc: number; number: number; title: string; length_ms: number; missing: boolean }; + +export type WantTracks = { source: 'library' | 'musicbrainz' | 'none'; edition?: string; tracks: WantTrack[] }; + export type Follow = { artist_mbid: string; artist_name: string; diff --git a/web/src/lib/api/wanted.ts b/web/src/lib/api/wanted.ts index 147e5d28..c81f6f18 100644 --- a/web/src/lib/api/wanted.ts +++ b/web/src/lib/api/wanted.ts @@ -1,4 +1,4 @@ -import { createInfiniteQuery, createQuery } from '@tanstack/svelte-query'; +import { createInfiniteQuery, createQuery, keepPreviousData } from '@tanstack/svelte-query'; import { api } from './client'; import { qk } from './queries'; import type { @@ -14,6 +14,7 @@ import type { SearchResult, SearchSettings, WantState, + WantTracks, WantsPage } from './types'; @@ -21,8 +22,11 @@ import type { // artists it follows for new releases, and the release-type profiles that // say which kinds of release a follow wants. -export async function listWants(state: WantState, offset: number = 0): Promise { - return api.get(`/api/admin/wanted?state=${state}&offset=${offset}`); +// q narrows the list to an artist or album whose name holds it (#5572). +export async function listWants(state: WantState, offset: number = 0, q: string = ''): Promise { + const params = new URLSearchParams({ state, offset: String(offset) }); + if (q) params.set('q', q); + return api.get(`/api/admin/wanted?${params}`); } export function wantsNextOffset(last: WantsPage): number | undefined { @@ -31,16 +35,32 @@ export function wantsNextOffset(last: WantsPage): number | undefined { } // Loads as the page scrolls (rule 172), one tab at a time. -export function createWantsQuery(state: WantState) { +export function createWantsQuery(state: WantState, q: string = '') { return createInfiniteQuery({ - queryKey: qk.adminWants(state), - queryFn: ({ pageParam }) => listWants(state, pageParam as number), + queryKey: qk.adminWants(state, q), + queryFn: ({ pageParam }) => listWants(state, pageParam as number, q), initialPageParam: 0, getNextPageParam: wantsNextOffset, + // A new filter keeps the list it narrows on screen until its answer comes. + placeholderData: keepPreviousData, staleTime: 30_000 }); } +// An album's tracks (#5572): the library's own for an album it holds, +// otherwise MusicBrainz's standard edition. +export async function getWantTracks(id: string): Promise { + return api.get(`/api/admin/wanted/${id}/tracks`); +} + +export function createWantTracksQuery(id: string) { + return createQuery({ + queryKey: qk.wantTracks(id), + queryFn: () => getWantTracks(id), + staleTime: 5 * 60_000 + }); +} + export async function removeWant(id: string): Promise { await api.del(`/api/admin/wanted/${id}`); } diff --git a/web/src/lib/components/wanted/WantList.svelte b/web/src/lib/components/wanted/WantList.svelte index 168f9fee..3771a890 100644 --- a/web/src/lib/components/wanted/WantList.svelte +++ b/web/src/lib/components/wanted/WantList.svelte @@ -1,5 +1,5 @@ + +
+ {#if tracks.isPending} +

Reading the tracks…

+ {:else if tracks.isError} +

+ {away ? "MusicBrainz can't be reached, so the tracks can't be shown." : "Couldn't load the tracks."} +

+ {:else if tracks.data.source === 'none' || list.length === 0} +

MusicBrainz lists no edition of this album, so there are no tracks to show.

+ {:else} + {#if tracks.data.source === 'musicbrainz'} +

+ MusicBrainz's standard edition{tracks.data.edition ? ` · ${tracks.data.edition}` : ''} +

+ {/if} +
    + {#each list as t, i (i)} +
  1. + {position(t)} + + {t.title} + + {#if t.missing}file missing{/if} + {#if t.length_ms > 0} + {formatDuration(t.length_ms / 1000)} + {/if} +
    2. + {/each} +
+ {/if} +
-- 2.54.0 From 358849de77f7763309541d81c7f9b31ca0dd34ad Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Sat, 10 Oct 2026 21:14:33 -0400 Subject: [PATCH 026/117] feat: the Wanted view becomes Collection, shaped like Lidarr: an artist catalogue first, the state sections down a left rail (M502 #5577) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The section opens on a catalogue of every artist on the list, A–Z with a letter rail, each card showing how many of their albums are held. An artist opens on their page: every album of theirs, held or not, with its status, search, releases and tracks. Wanted, Missing files, Activity, Following and Blocked sit down a left rail as the sections that show state; Held is gone, the catalogue shows it. The section moves from /wanted to /collection. Co-Authored-By: Claude Opus 5.5 --- internal/api/admin_wanted.go | 182 +++++++++++++----- internal/api/admin_wanted_test.go | 70 +++++++ internal/api/api.go | 1 + internal/db/dbq/wanted.sql.go | 117 ++++++++++- internal/db/queries/wanted.sql | 38 +++- web/src/lib/api/queries.ts | 4 +- web/src/lib/api/types.ts | 26 ++- web/src/lib/api/wanted.ts | 41 +++- .../ReacquisitionSettingsCard.svelte | 2 +- .../lib/components/SearchSettingsCard.svelte | 2 +- web/src/lib/components/Shell.svelte | 7 +- web/src/lib/components/Shell.test.ts | 8 +- web/src/lib/components/WantedTabs.svelte | 54 ------ .../wanted/CatalogueArtistCard.svelte | 46 +++++ .../components/wanted/CatalogueGrid.svelte | 110 +++++++++++ .../components/wanted/CatalogueGrid.test.ts | 103 ++++++++++ .../components/wanted/CollectionRail.svelte | 58 ++++++ web/src/lib/components/wanted/WantList.svelte | 106 ++++------ .../lib/components/wanted/WantList.test.ts | 171 +++++++++------- .../routes/admin/missing-files/+page.svelte | 2 +- .../admin/missing-files/missing-files.test.ts | 2 +- web/src/routes/admin/requests/+page.svelte | 2 +- web/src/routes/collection/+layout.svelte | 58 ++++++ .../routes/{wanted => collection}/+layout.ts | 0 web/src/routes/collection/+page.svelte | 8 + .../activity/+page.svelte | 2 +- .../collection/artist/[mbid]/+page.svelte | 15 ++ .../blocked/+page.svelte | 2 +- .../collection.test.ts} | 51 ++--- .../following/+page.svelte | 2 +- .../missing/+page.svelte | 2 +- .../{ => collection}/wanted/+page.svelte | 2 +- web/src/routes/wanted/+layout.svelte | 48 ----- web/src/routes/wanted/held/+page.svelte | 8 - 34 files changed, 994 insertions(+), 356 deletions(-) delete mode 100644 web/src/lib/components/WantedTabs.svelte create mode 100644 web/src/lib/components/wanted/CatalogueArtistCard.svelte create mode 100644 web/src/lib/components/wanted/CatalogueGrid.svelte create mode 100644 web/src/lib/components/wanted/CatalogueGrid.test.ts create mode 100644 web/src/lib/components/wanted/CollectionRail.svelte create mode 100644 web/src/routes/collection/+layout.svelte rename web/src/routes/{wanted => collection}/+layout.ts (100%) create mode 100644 web/src/routes/collection/+page.svelte rename web/src/routes/{wanted => collection}/activity/+page.svelte (67%) create mode 100644 web/src/routes/collection/artist/[mbid]/+page.svelte rename web/src/routes/{wanted => collection}/blocked/+page.svelte (66%) rename web/src/routes/{wanted/wanted.test.ts => collection/collection.test.ts} (52%) rename web/src/routes/{wanted => collection}/following/+page.svelte (78%) rename web/src/routes/{wanted => collection}/missing/+page.svelte (66%) rename web/src/routes/{ => collection}/wanted/+page.svelte (68%) delete mode 100644 web/src/routes/wanted/+layout.svelte delete mode 100644 web/src/routes/wanted/held/+page.svelte diff --git a/internal/api/admin_wanted.go b/internal/api/admin_wanted.go index 852d9cda..d58a20b1 100644 --- a/internal/api/admin_wanted.go +++ b/internal/api/admin_wanted.go @@ -69,8 +69,10 @@ type wantGrabView struct { ImportedFiles int32 `json:"imported_files"` } -// wantCounts sizes each tab of the Wanted page. +// wantCounts sizes each tab of the Wanted page. Artists: how many artists +// the list holds albums of, its main page (#5572). type wantCounts struct { + Artists int64 `json:"artists"` Wanted int64 `json:"wanted"` Missing int64 `json:"missing"` Held int64 `json:"held"` @@ -90,6 +92,34 @@ type wantsPage struct { Counts wantCounts `json:"counts"` // Library is the one-time listing of the library's albums (#5567). Library libraryListingView `json:"library"` + // Artist heads one artist's page, when the list is narrowed to them. + Artist *wantArtistView `json:"artist,omitempty"` +} + +// wantArtistView is one artist on the Wanted view's main page (#5572): how +// many of their albums the list holds, of how many, like Lidarr's artist +// index. +type wantArtistView struct { + ArtistMBID string `json:"artist_mbid"` + ArtistName string `json:"artist_name"` + SortName string `json:"sort_name"` + ArtistID string `json:"artist_id,omitempty"` + ImageURL string `json:"image_url"` + Albums int32 `json:"albums"` + Held int32 `json:"held"` + Wanted int32 `json:"wanted"` + Missing int32 `json:"missing"` + Downloading int32 `json:"downloading"` + Following bool `json:"following"` +} + +type wantArtistsPage struct { + Items []wantArtistView `json:"items"` + Total int64 `json:"total"` + Limit int `json:"limit"` + Offset int `json:"offset"` + Counts wantCounts `json:"counts"` + Library libraryListingView `json:"library"` } // libraryListingView: whether the library's albums are all on the list @@ -113,77 +143,129 @@ const unmatchedShown = 200 const wantsPageSize = 50 -// handleListWants implements GET /api/admin/wanted?state=wanted|missing|held&q=&offset=. +// wantArtistsPageSize fills a few rows of the artist grid a page. +const wantArtistsPageSize = 96 + +// wantsHeader is what every Wanted page carries: each tab's count, and how +// the one-time library listing went. The albums no match was found for are +// named only when asked (the main page's first page). +func (h *handlers) wantsHeader(r *http.Request, q *dbq.Queries, unmatched bool) (wantCounts, libraryListingView, error) { + counts, err := q.CountWants(r.Context()) + if err != nil { + return wantCounts{}, libraryListingView{}, err + } + listing, err := q.GetLibraryListing(r.Context()) + if err != nil { + return wantCounts{}, libraryListingView{}, err + } + lib := libraryListingView{Unlisted: listing.LibraryUnlisted, Unmatched: []unmatchedAlbumView{}} + if listing.LibraryListedAt.Valid { + at := formatTimestamp(listing.LibraryListedAt) + lib.ListedAt = &at + } + if unmatched && listing.LibraryUnlisted > 0 { + rows, err := q.ListUnmatchedLibraryAlbums(r.Context(), unmatchedShown) + if err != nil { + return wantCounts{}, libraryListingView{}, err + } + for _, u := range rows { + lib.Unmatched = append(lib.Unmatched, unmatchedAlbumView{ + AlbumID: uuidToString(u.AlbumID), Title: u.Title, Artist: u.ArtistName, Detail: u.Detail, + }) + } + } + return wantCounts{Artists: counts.Artists, Wanted: counts.Wanted, Missing: counts.Missing, Held: counts.Held, + Following: counts.Following, Activity: counts.Activity, Attention: counts.Attention, Blocked: counts.Blocked}, lib, nil +} + +func offsetParam(r *http.Request) int { + offset, _ := strconv.Atoi(r.URL.Query().Get("offset")) + return max(offset, 0) +} + +func wantArtistViewOf(row dbq.ListWantArtistsRow) wantArtistView { + v := wantArtistView{ + ArtistMBID: row.ArtistMbid, ArtistName: row.ArtistName, SortName: row.SortName, + ImageURL: artistArtURL(row.ArtistMbid, row.ArtistName), + Albums: row.Albums, Held: row.Held, Wanted: row.Wanted, Missing: row.Missing, + Downloading: row.Downloading, Following: row.Following, + } + if row.LibraryArtistID.Valid { + v.ArtistID = uuidToString(row.LibraryArtistID) + } + return v +} + +// handleListWantArtists implements GET /api/admin/wanted/artists?q=&offset=: +// the Wanted view's main page, every artist on the list A–Z (#5572). +func (h *handlers) handleListWantArtists(w http.ResponseWriter, r *http.Request) { + offset := offsetParam(r) + q := dbq.New(h.pool) + rows, err := q.ListWantArtists(r.Context(), dbq.ListWantArtistsParams{ + Q: strings.TrimSpace(r.URL.Query().Get("q")), Skip: int32(offset), MaxRows: wantArtistsPageSize}) + if err != nil { + writeErrWithLog(w, h.logger, "admin wanted: artists", apierror.Internal(err)) + return + } + counts, lib, err := h.wantsHeader(r, q, offset == 0) + if err != nil { + writeErrWithLog(w, h.logger, "admin wanted: counts", apierror.Internal(err)) + return + } + out := wantArtistsPage{Items: make([]wantArtistView, 0, len(rows)), Limit: wantArtistsPageSize, Offset: offset, + Counts: counts, Library: lib} + for _, row := range rows { + out.Total = row.Total + out.Items = append(out.Items, wantArtistViewOf(row)) + } + writeJSON(w, http.StatusOK, out) +} + +// handleListWants implements +// GET /api/admin/wanted?state=wanted|missing|held|all&artist=&q=&offset=. // wanted is everything open but missing files. Each list runs artist by -// artist, A–Z, like the library (#5572); q narrows it to an artist or album. +// artist, A–Z, like the library (#5572); q narrows it to an artist or +// album, and artist to one artist's page, which it then heads. func (h *handlers) handleListWants(w http.ResponseWriter, r *http.Request) { state := r.URL.Query().Get("state") if state == "" { state = "wanted" } - if state != "wanted" && state != "missing" && state != "held" { - writeErr(w, apierror.BadRequest("bad_state", "state must be wanted, missing or held")) + if state != "wanted" && state != "missing" && state != "held" && state != "all" { + writeErr(w, apierror.BadRequest("bad_state", "state must be wanted, missing, held or all")) return } - offset, _ := strconv.Atoi(r.URL.Query().Get("offset")) - if offset < 0 { - offset = 0 - } + offset := offsetParam(r) + artist := strings.TrimSpace(r.URL.Query().Get("artist")) q := dbq.New(h.pool) - filter := strings.TrimSpace(r.URL.Query().Get("q")) - rows, err := q.ListWants(r.Context(), dbq.ListWantsParams{State: state, Q: filter, Skip: int32(offset), MaxRows: wantsPageSize}) + rows, err := q.ListWants(r.Context(), dbq.ListWantsParams{State: state, ArtistMbid: artist, + Q: strings.TrimSpace(r.URL.Query().Get("q")), Skip: int32(offset), MaxRows: wantsPageSize}) if err != nil { writeErrWithLog(w, h.logger, "admin wanted: list", apierror.Internal(err)) return } - counts, err := q.CountWants(r.Context()) + counts, lib, err := h.wantsHeader(r, q, false) if err != nil { - writeErrWithLog(w, h.logger, "admin wanted: count", apierror.Internal(err)) + writeErrWithLog(w, h.logger, "admin wanted: counts", apierror.Internal(err)) return } - listing, err := q.GetLibraryListing(r.Context()) - if err != nil { - writeErrWithLog(w, h.logger, "admin wanted: library listing", apierror.Internal(err)) - return - } - out := wantsPage{ - Library: libraryListingView{Unlisted: listing.LibraryUnlisted}, - Items: make([]wantView, 0, len(rows)), Limit: wantsPageSize, Offset: offset, - Counts: wantCounts{Wanted: counts.Wanted, Missing: counts.Missing, Held: counts.Held, Following: counts.Following, - Activity: counts.Activity, Attention: counts.Attention, Blocked: counts.Blocked}, - } - out.Library.Unmatched = []unmatchedAlbumView{} - if listing.LibraryListedAt.Valid { - at := formatTimestamp(listing.LibraryListedAt) - out.Library.ListedAt = &at - } - if state == "held" && offset == 0 && listing.LibraryUnlisted > 0 { - unmatched, err := q.ListUnmatchedLibraryAlbums(r.Context(), unmatchedShown) + out := wantsPage{Library: lib, Counts: counts, + Items: make([]wantView, 0, len(rows)), Limit: wantsPageSize, Offset: offset} + if artist != "" { + heads, err := q.ListWantArtists(r.Context(), dbq.ListWantArtistsParams{ArtistMbid: artist, MaxRows: 1}) if err != nil { - writeErrWithLog(w, h.logger, "admin wanted: unmatched albums", apierror.Internal(err)) + writeErrWithLog(w, h.logger, "admin wanted: artist", apierror.Internal(err)) return } - for _, u := range unmatched { - out.Library.Unmatched = append(out.Library.Unmatched, unmatchedAlbumView{ - AlbumID: uuidToString(u.AlbumID), Title: u.Title, Artist: u.ArtistName, Detail: u.Detail, - }) - } - } - switch state { - case "missing": - out.Total = counts.Missing - case "held": - out.Total = counts.Held - default: - out.Total = counts.Wanted - } - if filter != "" { - out.Total = 0 - if len(rows) > 0 { - out.Total = rows[0].Total + if len(heads) == 0 { + writeErr(w, apierror.NotFound("artist")) + return } + v := wantArtistViewOf(heads[0]) + out.Artist = &v } for _, row := range rows { + out.Total = row.Total v := wantView{ ID: uuidToString(row.ID), ReleaseGroupMBID: row.ReleaseGroupMbid, Title: row.Title, ArtistMBID: row.ArtistMbid, ArtistName: row.ArtistName, diff --git a/internal/api/admin_wanted_test.go b/internal/api/admin_wanted_test.go index db7f9baa..ecfc70da 100644 --- a/internal/api/admin_wanted_test.go +++ b/internal/api/admin_wanted_test.go @@ -24,6 +24,7 @@ func doWantedReq(t *testing.T, h *handlers, path string, user dbq.User) *httptes r.Route("/api/admin", func(admin chi.Router) { admin.Use(auth.RequireAdmin()) admin.Get("/wanted", h.handleListWants) + admin.Get("/wanted/artists", h.handleListWantArtists) admin.Get("/wanted/{id}/tracks", h.handleWantTracks) }) w := httptest.NewRecorder() @@ -98,6 +99,75 @@ func TestListWants_ArtistsAToZThenTheirAlbumsByDate(t *testing.T) { } } +// The Collection's catalogue (#5572): every artist on the list A–Z, each +// with how many of their albums are held; an artist's page lists every album +// of theirs, held or not, under that artist. +func TestListWantArtists_CatalogueAndAnArtistsPage(t *testing.T) { + h, _ := testHandlersWithClientFn(t) + admin := seedUser(t, h.pool, "admin", "pw", true) + ctx := context.Background() + q := dbq.New(h.pool) + beatles := "a0000000-0000-4000-8000-000000000002" + camel := "a0000000-0000-4000-8000-000000000001" + if _, err := q.UpsertArtist(ctx, dbq.UpsertArtistParams{Name: "The Beatles", SortName: "Beatles, The", Mbid: &beatles}); err != nil { + t.Fatal(err) + } + seedWant(t, h, "g0000000-0000-4000-8000-000000000002", beatles, "The Beatles", "Abbey Road", "1969") + held := seedWant(t, h, "g0000000-0000-4000-8000-000000000003", beatles, "The Beatles", "Please Please Me", "1963") + if _, err := h.pool.Exec(ctx, `UPDATE wants SET fulfilled_at = now() WHERE id = $1`, held.ID); err != nil { + t.Fatal(err) + } + seedWant(t, h, "g0000000-0000-4000-8000-000000000004", camel, "Camel", "Moonmadness", "1976") + if _, err := h.pool.Exec(ctx, `INSERT INTO follows (artist_mbid, artist_name, profile_id) + SELECT $1, 'Camel', id FROM release_type_profiles WHERE is_default`, camel); err != nil { + t.Fatal(err) + } + + w := doWantedReq(t, h, "/api/admin/wanted/artists", admin) + if w.Code != http.StatusOK { + t.Fatalf("status = %d; body = %s", w.Code, w.Body.String()) + } + var page wantArtistsPage + if err := json.Unmarshal(w.Body.Bytes(), &page); err != nil { + t.Fatal(err) + } + if len(page.Items) != 2 || page.Total != 2 || page.Counts.Artists != 2 { + t.Fatalf("catalogue = %+v, total %d, artists %d", page.Items, page.Total, page.Counts.Artists) + } + b, c := page.Items[0], page.Items[1] + if b.ArtistName != "The Beatles" || b.SortName != "Beatles, The" || b.Albums != 2 || b.Held != 1 || b.Wanted != 1 || + b.ArtistID == "" || b.Following || b.ImageURL == "" { + t.Errorf("Beatles = %+v", b) + } + if c.ArtistName != "Camel" || c.Albums != 1 || c.Held != 0 || !c.Following || c.ArtistID != "" { + t.Errorf("Camel = %+v", c) + } + + // q finds an artist by an album's title. + w = doWantedReq(t, h, "/api/admin/wanted/artists?q=moon", admin) + page = wantArtistsPage{} + if err := json.Unmarshal(w.Body.Bytes(), &page); err != nil { + t.Fatal(err) + } + if len(page.Items) != 1 || page.Items[0].ArtistName != "Camel" { + t.Errorf("q=moon: %+v", page.Items) + } + + // The Beatles' page: both albums, held or not, headed by the artist. + w = doWantedReq(t, h, "/api/admin/wanted?state=all&artist="+beatles, admin) + var albums wantsPage + if err := json.Unmarshal(w.Body.Bytes(), &albums); err != nil { + t.Fatal(err) + } + if len(albums.Items) != 2 || albums.Items[0].Title != "Please Please Me" || albums.Items[0].HeldAt == nil || + albums.Artist == nil || albums.Artist.Held != 1 { + t.Errorf("artist page = %+v, head %+v", albums.Items, albums.Artist) + } + if w := doWantedReq(t, h, "/api/admin/wanted?state=all&artist=nobody", admin); w.Code != http.StatusNotFound { + t.Errorf("an artist with nothing on the list: status %d, want 404", w.Code) + } +} + // fakeEditions is a MusicBrainz holding one group with two editions, the // shorter official one the standard, or 503 when down is set. func fakeEditions(t *testing.T, group string, down bool) *httptest.Server { diff --git a/internal/api/api.go b/internal/api/api.go index 68825e18..b8d7d27a 100644 --- a/internal/api/api.go +++ b/internal/api/api.go @@ -252,6 +252,7 @@ func Mount(r chi.Router, pool *pgxpool.Pool, logger *slog.Logger, events *playev // for, the artists it follows, and the release-type // profiles that decide which of their releases count. admin.Get("/wanted", h.handleListWants) + admin.Get("/wanted/artists", h.handleListWantArtists) admin.Delete("/wanted/{id}", h.handleRemoveWant) // Searching and grabbing (M502 #5521). admin.Get("/acquisition/settings", h.handleGetSearchSettings) diff --git a/internal/db/dbq/wanted.sql.go b/internal/db/dbq/wanted.sql.go index c33da297..22469f12 100644 --- a/internal/db/dbq/wanted.sql.go +++ b/internal/db/dbq/wanted.sql.go @@ -37,6 +37,7 @@ const countWants = `-- name: CountWants :one SELECT COUNT(*) FILTER (WHERE fulfilled_at IS NULL AND reason <> 'missing')::bigint AS wanted, COUNT(*) FILTER (WHERE fulfilled_at IS NULL AND reason = 'missing')::bigint AS missing, COUNT(*) FILTER (WHERE fulfilled_at IS NOT NULL)::bigint AS held, + COUNT(DISTINCT artist_mbid)::bigint AS artists, (SELECT COUNT(*) FROM follows)::bigint AS following, -- The Wanted view's other tabs (#5522): downloads running, those -- waiting on a person, and releases set aside. @@ -52,6 +53,7 @@ type CountWantsRow struct { Wanted int64 Missing int64 Held int64 + Artists int64 Following int64 Activity int64 Attention int64 @@ -65,6 +67,7 @@ func (q *Queries) CountWants(ctx context.Context) (CountWantsRow, error) { &i.Wanted, &i.Missing, &i.Held, + &i.Artists, &i.Following, &i.Activity, &i.Attention, @@ -852,6 +855,98 @@ func (q *Queries) ListUnmatchedLibraryAlbums(ctx context.Context, maxRows int32) return items, nil } +const listWantArtists = `-- name: ListWantArtists :many +WITH a AS ( + SELECT w.artist_mbid, + (array_agg(w.artist_name ORDER BY w.wanted_at DESC))[1] AS artist_name, + COUNT(*) FILTER (WHERE w.fulfilled_at IS NOT NULL)::int AS held, + COUNT(*) FILTER (WHERE w.fulfilled_at IS NULL AND w.reason <> 'missing')::int AS wanted, + COUNT(*) FILTER (WHERE w.fulfilled_at IS NULL AND w.reason = 'missing')::int AS missing, + COUNT(*)::int AS albums, + bool_or($1::text <> '' AND strpos(lower(w.title), lower($1::text)) > 0) AS title_match + FROM wants w + WHERE $4::text = '' OR w.artist_mbid = $4::text + GROUP BY w.artist_mbid +) +SELECT a.artist_mbid, a.artist_name::text AS artist_name, a.held, a.wanted, a.missing, a.albums, + COALESCE(NULLIF(la.sort_name, ''), a.artist_name)::text AS sort_name, + la.id AS library_artist_id, + (f.artist_mbid IS NOT NULL)::bool AS following, + (SELECT COUNT(*) FROM grabs g JOIN wants x ON x.id = g.want_id + WHERE x.artist_mbid = a.artist_mbid AND g.ended_at IS NULL)::int AS downloading, + COUNT(*) OVER () AS total + FROM a + LEFT JOIN LATERAL (SELECT ar.id, ar.sort_name FROM artists ar WHERE ar.mbid = a.artist_mbid LIMIT 1) la ON true + LEFT JOIN follows f ON f.artist_mbid = a.artist_mbid + WHERE $1::text = '' + OR strpos(lower(a.artist_name), lower($1::text)) > 0 + OR a.title_match + ORDER BY lower(COALESCE(NULLIF(la.sort_name, ''), a.artist_name)), a.artist_mbid + LIMIT $3 OFFSET $2 +` + +type ListWantArtistsParams struct { + Q string + Skip int32 + MaxRows int32 + ArtistMbid string +} + +type ListWantArtistsRow struct { + ArtistMbid string + ArtistName string + Held int32 + Wanted int32 + Missing int32 + Albums int32 + SortName string + LibraryArtistID pgtype.UUID + Following bool + Downloading int32 + Total int64 +} + +// The Wanted view's main page (#5572): every artist on the list, A–Z like +// the library, with how many of their albums are held of how many are on +// the list. q matches the artist's name or any of their albums' titles; +// artist_mbid narrows to one artist, for their page's header. +func (q *Queries) ListWantArtists(ctx context.Context, arg ListWantArtistsParams) ([]ListWantArtistsRow, error) { + rows, err := q.db.Query(ctx, listWantArtists, + arg.Q, + arg.Skip, + arg.MaxRows, + arg.ArtistMbid, + ) + if err != nil { + return nil, err + } + defer rows.Close() + var items []ListWantArtistsRow + for rows.Next() { + var i ListWantArtistsRow + if err := rows.Scan( + &i.ArtistMbid, + &i.ArtistName, + &i.Held, + &i.Wanted, + &i.Missing, + &i.Albums, + &i.SortName, + &i.LibraryArtistID, + &i.Following, + &i.Downloading, + &i.Total, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + const listWants = `-- name: ListWants :many SELECT w.id, w.release_group_mbid, w.artist_mbid, w.artist_name, w.title, w.reason, w.album_id, w.wanted_at, w.fulfilled_at, @@ -879,24 +974,27 @@ SELECT w.id, w.release_group_mbid, w.artist_mbid, w.artist_name, w.title, w.reas LEFT JOIN grabs gr ON gr.id = (SELECT x.id FROM grabs x WHERE x.want_id = w.id ORDER BY x.ended_at IS NULL DESC, x.grabbed_at DESC LIMIT 1) WHERE CASE $1::text + WHEN 'all' THEN true WHEN 'held' THEN w.fulfilled_at IS NOT NULL WHEN 'missing' THEN w.fulfilled_at IS NULL AND w.reason = 'missing' ELSE w.fulfilled_at IS NULL AND w.reason <> 'missing' END - AND ($2::text = '' - OR strpos(lower(w.artist_name), lower($2::text)) > 0 - OR strpos(lower(w.title), lower($2::text)) > 0) + AND ($2::text = '' OR w.artist_mbid = $2::text) + AND ($3::text = '' + OR strpos(lower(w.artist_name), lower($3::text)) > 0 + OR strpos(lower(w.title), lower($3::text)) > 0) ORDER BY lower(COALESCE(NULLIF(la.sort_name, ''), w.artist_name)), w.artist_mbid, NULLIF(COALESCE(NULLIF(w.first_release_date, ''), g.first_release_date, ''), '') NULLS LAST, lower(w.title), w.id - LIMIT $4 OFFSET $3 + LIMIT $5 OFFSET $4 ` type ListWantsParams struct { - State string - Q string - Skip int32 - MaxRows int32 + State string + ArtistMbid string + Q string + Skip int32 + MaxRows int32 } type ListWantsRow struct { @@ -937,13 +1035,14 @@ type ListWantsRow struct { } // The Wanted page: open wants (all but missing ones, or only missing ones), -// or the held ones. Ordered like the library (#5572): artists A–Z by the +// the held ones, or all of them; artist_mbid narrows to one artist's page. Ordered like the library (#5572): artists A–Z by the // library's sort name when it holds the artist, each artist's albums by // release date, so the page groups them as they arrive. q narrows to an // artist or album title; total counts every match, past the page. func (q *Queries) ListWants(ctx context.Context, arg ListWantsParams) ([]ListWantsRow, error) { rows, err := q.db.Query(ctx, listWants, arg.State, + arg.ArtistMbid, arg.Q, arg.Skip, arg.MaxRows, diff --git a/internal/db/queries/wanted.sql b/internal/db/queries/wanted.sql index 173e582c..7aa34649 100644 --- a/internal/db/queries/wanted.sql +++ b/internal/db/queries/wanted.sql @@ -177,7 +177,7 @@ DELETE FROM wants WHERE id = $1; -- name: ListWants :many -- The Wanted page: open wants (all but missing ones, or only missing ones), --- or the held ones. Ordered like the library (#5572): artists A–Z by the +-- the held ones, or all of them; artist_mbid narrows to one artist's page. Ordered like the library (#5572): artists A–Z by the -- library's sort name when it holds the artist, each artist's albums by -- release date, so the page groups them as they arrive. q narrows to an -- artist or album title; total counts every match, past the page. @@ -207,10 +207,12 @@ SELECT w.id, w.release_group_mbid, w.artist_mbid, w.artist_name, w.title, w.reas LEFT JOIN grabs gr ON gr.id = (SELECT x.id FROM grabs x WHERE x.want_id = w.id ORDER BY x.ended_at IS NULL DESC, x.grabbed_at DESC LIMIT 1) WHERE CASE sqlc.arg(state)::text + WHEN 'all' THEN true WHEN 'held' THEN w.fulfilled_at IS NOT NULL WHEN 'missing' THEN w.fulfilled_at IS NULL AND w.reason = 'missing' ELSE w.fulfilled_at IS NULL AND w.reason <> 'missing' END + AND (sqlc.arg(artist_mbid)::text = '' OR w.artist_mbid = sqlc.arg(artist_mbid)::text) AND (sqlc.arg(q)::text = '' OR strpos(lower(w.artist_name), lower(sqlc.arg(q)::text)) > 0 OR strpos(lower(w.title), lower(sqlc.arg(q)::text)) > 0) @@ -219,6 +221,39 @@ SELECT w.id, w.release_group_mbid, w.artist_mbid, w.artist_name, w.title, w.reas lower(w.title), w.id LIMIT sqlc.arg(max_rows) OFFSET sqlc.arg(skip); +-- name: ListWantArtists :many +-- The Wanted view's main page (#5572): every artist on the list, A–Z like +-- the library, with how many of their albums are held of how many are on +-- the list. q matches the artist's name or any of their albums' titles; +-- artist_mbid narrows to one artist, for their page's header. +WITH a AS ( + SELECT w.artist_mbid, + (array_agg(w.artist_name ORDER BY w.wanted_at DESC))[1] AS artist_name, + COUNT(*) FILTER (WHERE w.fulfilled_at IS NOT NULL)::int AS held, + COUNT(*) FILTER (WHERE w.fulfilled_at IS NULL AND w.reason <> 'missing')::int AS wanted, + COUNT(*) FILTER (WHERE w.fulfilled_at IS NULL AND w.reason = 'missing')::int AS missing, + COUNT(*)::int AS albums, + bool_or(sqlc.arg(q)::text <> '' AND strpos(lower(w.title), lower(sqlc.arg(q)::text)) > 0) AS title_match + FROM wants w + WHERE sqlc.arg(artist_mbid)::text = '' OR w.artist_mbid = sqlc.arg(artist_mbid)::text + GROUP BY w.artist_mbid +) +SELECT a.artist_mbid, a.artist_name::text AS artist_name, a.held, a.wanted, a.missing, a.albums, + COALESCE(NULLIF(la.sort_name, ''), a.artist_name)::text AS sort_name, + la.id AS library_artist_id, + (f.artist_mbid IS NOT NULL)::bool AS following, + (SELECT COUNT(*) FROM grabs g JOIN wants x ON x.id = g.want_id + WHERE x.artist_mbid = a.artist_mbid AND g.ended_at IS NULL)::int AS downloading, + COUNT(*) OVER () AS total + FROM a + LEFT JOIN LATERAL (SELECT ar.id, ar.sort_name FROM artists ar WHERE ar.mbid = a.artist_mbid LIMIT 1) la ON true + LEFT JOIN follows f ON f.artist_mbid = a.artist_mbid + WHERE sqlc.arg(q)::text = '' + OR strpos(lower(a.artist_name), lower(sqlc.arg(q)::text)) > 0 + OR a.title_match + ORDER BY lower(COALESCE(NULLIF(la.sort_name, ''), a.artist_name)), a.artist_mbid + LIMIT sqlc.arg(max_rows) OFFSET sqlc.arg(skip); + -- name: GetWantAlbum :one -- What the Wanted view's track list needs of a want (#5572). SELECT id, release_group_mbid, album_id FROM wants WHERE id = $1; @@ -235,6 +270,7 @@ SELECT title, disc_number, track_number, duration_ms, (missing_since IS NOT NULL SELECT COUNT(*) FILTER (WHERE fulfilled_at IS NULL AND reason <> 'missing')::bigint AS wanted, COUNT(*) FILTER (WHERE fulfilled_at IS NULL AND reason = 'missing')::bigint AS missing, COUNT(*) FILTER (WHERE fulfilled_at IS NOT NULL)::bigint AS held, + COUNT(DISTINCT artist_mbid)::bigint AS artists, (SELECT COUNT(*) FROM follows)::bigint AS following, -- The Wanted view's other tabs (#5522): downloads running, those -- waiting on a person, and releases set aside. diff --git a/web/src/lib/api/queries.ts b/web/src/lib/api/queries.ts index b70e78f4..4dbde645 100644 --- a/web/src/lib/api/queries.ts +++ b/web/src/lib/api/queries.ts @@ -41,7 +41,9 @@ export const qk = { ['adminRequests', { status: status ?? 'all' }] as const, // Wanted list (M502 #5520). Wants sit under ['adminWants'] so one prefix // invalidation refreshes every tab and its counts. - adminWants: (state: string, q: string = '') => ['adminWants', { state, q }] as const, + adminWants: (state: string, q: string = '', artist: string = '') => + ['adminWants', { state, q, artist }] as const, + wantArtists: (q: string = '') => ['adminWants', 'artists', { q }] as const, wantTracks: (id: string) => ['wantTracks', id] as const, adminFollows: () => ['adminFollows'] as const, adminFollow: (artistMbid: string) => ['adminFollow', artistMbid] as const, diff --git a/web/src/lib/api/types.ts b/web/src/lib/api/types.ts index 9e3950ff..3684ccb3 100644 --- a/web/src/lib/api/types.ts +++ b/web/src/lib/api/types.ts @@ -571,7 +571,8 @@ export type ResolvedDuplicatesResponse = { // library: in the library when the list began (#5567). export type WantReason = 'request' | 'follow' | 'missing' | 'library'; -export type WantState = 'wanted' | 'missing' | 'held'; +// all: one artist's page, every album of theirs on the list (#5572). +export type WantState = 'wanted' | 'missing' | 'held' | 'all'; // One release group on the wanted list. held_at is set once the library // holds it; artist_id and album_id link into the library when it has them. @@ -722,6 +723,8 @@ export type SearchSettings = { }; export type WantCounts = { + // Artists in the Collection's catalogue (#5572). + artists: number; wanted: number; missing: number; held: number; @@ -819,7 +822,26 @@ export type UnmatchedAlbum = { album_id: string; title: string; artist: string; export type LibraryListing = { listed_at: string | null; unlisted: number; unmatched: UnmatchedAlbum[] }; -export type WantsPage = Page & { counts: WantCounts; library: LibraryListing }; +// artist heads the list when it is narrowed to one artist's page. +export type WantsPage = Page & { counts: WantCounts; library: LibraryListing; artist?: WantArtist }; + +// One artist in the Collection's catalogue (#5572), like Lidarr's artist +// index: how many of their albums on the list are held, of how many. +export type WantArtist = { + artist_mbid: string; + artist_name: string; + sort_name: string; + artist_id?: string; + image_url: string; + albums: number; + held: number; + wanted: number; + missing: number; + downloading: number; + following: boolean; +}; + +export type WantArtistsPage = Page & { counts: WantCounts; library: LibraryListing }; // An album's tracks in the Wanted view (#5572): the library's own for an // album it holds (a lost file marked missing), else MusicBrainz's standard diff --git a/web/src/lib/api/wanted.ts b/web/src/lib/api/wanted.ts index c81f6f18..d5a60f9f 100644 --- a/web/src/lib/api/wanted.ts +++ b/web/src/lib/api/wanted.ts @@ -13,6 +13,7 @@ import type { GrabResult, SearchResult, SearchSettings, + WantArtistsPage, WantState, WantTracks, WantsPage @@ -22,10 +23,17 @@ import type { // artists it follows for new releases, and the release-type profiles that // say which kinds of release a follow wants. -// q narrows the list to an artist or album whose name holds it (#5572). -export async function listWants(state: WantState, offset: number = 0, q: string = ''): Promise { +// q narrows the list to an artist or album whose name holds it; artist to +// one artist's page, which the response then heads (#5572). +export async function listWants( + state: WantState, + offset: number = 0, + q: string = '', + artist: string = '' +): Promise { const params = new URLSearchParams({ state, offset: String(offset) }); if (q) params.set('q', q); + if (artist) params.set('artist', artist); return api.get(`/api/admin/wanted?${params}`); } @@ -35,10 +43,10 @@ export function wantsNextOffset(last: WantsPage): number | undefined { } // Loads as the page scrolls (rule 172), one tab at a time. -export function createWantsQuery(state: WantState, q: string = '') { +export function createWantsQuery(state: WantState, q: string = '', artist: string = '') { return createInfiniteQuery({ - queryKey: qk.adminWants(state, q), - queryFn: ({ pageParam }) => listWants(state, pageParam as number, q), + queryKey: qk.adminWants(state, q, artist), + queryFn: ({ pageParam }) => listWants(state, pageParam as number, q, artist), initialPageParam: 0, getNextPageParam: wantsNextOffset, // A new filter keeps the list it narrows on screen until its answer comes. @@ -47,6 +55,29 @@ export function createWantsQuery(state: WantState, q: string = '') { }); } +// The Collection's catalogue (#5572): every artist on the list, A–Z, with +// how many of their albums are held. Its first page carries every section's +// count. +export async function listWantArtists(offset: number = 0, q: string = ''): Promise { + const params = new URLSearchParams({ offset: String(offset) }); + if (q) params.set('q', q); + return api.get(`/api/admin/wanted/artists?${params}`); +} + +export function createWantArtistsQuery(q: string = '') { + return createInfiniteQuery({ + queryKey: qk.wantArtists(q), + queryFn: ({ pageParam }) => listWantArtists(pageParam as number, q), + initialPageParam: 0, + getNextPageParam: (last: WantArtistsPage) => { + const loaded = last.offset + last.items.length; + return last.items.length === 0 || loaded >= last.total ? undefined : loaded; + }, + placeholderData: keepPreviousData, + staleTime: 30_000 + }); +} + // An album's tracks (#5572): the library's own for an album it holds, // otherwise MusicBrainz's standard edition. export async function getWantTracks(id: string): Promise { diff --git a/web/src/lib/components/ReacquisitionSettingsCard.svelte b/web/src/lib/components/ReacquisitionSettingsCard.svelte index 490b7664..437faeca 100644 --- a/web/src/lib/components/ReacquisitionSettingsCard.svelte +++ b/web/src/lib/components/ReacquisitionSettingsCard.svelte @@ -74,7 +74,7 @@

Automatic re-acquisition

When a file has been missing for a while, Minstrel can put its album back on the - wanted list + wanted list by itself. It's wanted per album, not per track: a whole folder going missing is one want, not forty.

diff --git a/web/src/lib/components/SearchSettingsCard.svelte b/web/src/lib/components/SearchSettingsCard.svelte index f256d50a..3bc7cb26 100644 --- a/web/src/lib/components/SearchSettingsCard.svelte +++ b/web/src/lib/components/SearchSettingsCard.svelte @@ -104,7 +104,7 @@

Searching and grabbing

Minstrel searches your indexers for each album on the - wanted list, judges every + wanted list, judges every release it finds, and hands the best one to your download client. Each album's releases and the reason for every verdict are on the Wanted page.

diff --git a/web/src/lib/components/Shell.svelte b/web/src/lib/components/Shell.svelte index 5caf9946..d318f4d8 100644 --- a/web/src/lib/components/Shell.svelte +++ b/web/src/lib/components/Shell.svelte @@ -1,7 +1,7 @@ diff --git a/web/src/lib/components/Shell.test.ts b/web/src/lib/components/Shell.test.ts index 92a63aab..059a0958 100644 --- a/web/src/lib/components/Shell.test.ts +++ b/web/src/lib/components/Shell.test.ts @@ -67,19 +67,19 @@ describe('Shell', () => { expect(screen.getByRole('link', { name: 'Discover' })).toHaveAttribute('href', '/discover'); }); - test('admins get Wanted after Discover; others do not', () => { + test('admins get Collection after Discover; others do not', () => { const primary = () => screen .getAllByRole('link') .filter((el) => el.closest('nav[aria-label="Primary"]') !== null) .map((el) => el.textContent?.trim()); const { unmount } = render(Shell); - expect(primary()).not.toContain('Wanted'); + expect(primary()).not.toContain('Collection'); unmount(); userState.current = { id: '1', username: 'alice', is_admin: true }; render(Shell); - expect(primary()).toEqual(['Home', 'Library', 'Discover', 'Wanted']); - expect(screen.getByRole('link', { name: 'Wanted' })).toHaveAttribute('href', '/wanted'); + expect(primary()).toEqual(['Home', 'Library', 'Discover', 'Collection']); + expect(screen.getByRole('link', { name: 'Collection' })).toHaveAttribute('href', '/collection'); }); test('main nav omits Artists/Albums/Liked/History/Playlists/Search/Settings/Admin', () => { diff --git a/web/src/lib/components/WantedTabs.svelte b/web/src/lib/components/WantedTabs.svelte deleted file mode 100644 index 3a9bf1a4..00000000 --- a/web/src/lib/components/WantedTabs.svelte +++ /dev/null @@ -1,54 +0,0 @@ - - - diff --git a/web/src/lib/components/wanted/CatalogueArtistCard.svelte b/web/src/lib/components/wanted/CatalogueArtistCard.svelte new file mode 100644 index 00000000..88a69371 --- /dev/null +++ b/web/src/lib/components/wanted/CatalogueArtistCard.svelte @@ -0,0 +1,46 @@ + + + +
+ +
+
+
+
+
+
{a.artist_name}
+
{a.held} / {a.albums} held
+ {#if notes.length} +
{notes.join(' · ')}
+ {/if} +
+ diff --git a/web/src/lib/components/wanted/CatalogueGrid.svelte b/web/src/lib/components/wanted/CatalogueGrid.svelte new file mode 100644 index 00000000..f37478c2 --- /dev/null +++ b/web/src/lib/components/wanted/CatalogueGrid.svelte @@ -0,0 +1,110 @@ + + +{#if library && !library.listed_at} +

+ Your library's albums are being added to the catalogue. Albums whose tags don't name them are + matched on MusicBrainz first, by their artist, title and tracks, about one a second. +

+{:else if library && library.unlisted > 0} +
+

+ {library.unlisted === 1 ? '1 album' : `${library.unlisted} albums`} in your library + {library.unlisted === 1 ? "isn't" : "aren't"} in the catalogue: no MusicBrainz album matched + {library.unlisted === 1 ? 'its' : 'their'} artist, title and tracks. Until + {library.unlisted === 1 ? "it's" : "they're"} matched, {library.unlisted === 1 ? "it isn't" : "they aren't"} + fetched back if {library.unlisted === 1 ? 'its' : 'their'} files go missing. +

+ {#if library.unmatched.length > 0} + + {#if showUnmatched} +
    + {#each library.unmatched as u (u.album_id)} +
  • + {u.title} + · {u.artist} +
    {u.detail}
    +
    • + {/each} +
+ {#if library.unmatched.length < library.unlisted} +

The first {library.unmatched.length} of {library.unlisted}.

+ {/if} + {/if} + {/if} +
+{/if} + +{#if !artists.isPending && (items.length > 0 || q)} +
+ +
+{/if} + +{#if artists.isPending} +

Reading the catalogue…

+{:else if artists.isError && !artists.isFetchNextPageError} +

Couldn't load the catalogue.

+{:else if items.length === 0 && q} +

+ No artist in the catalogue, or album of theirs, is named like '{q}'. +

+{:else if items.length === 0} +
+ +

The catalogue is empty.

+

+ Your library's albums land here once it's been scanned, as do the albums users request and + the releases of artists Minstrel follows. +

+
+{:else} + a.sort_name || a.artist_name} + hasMore={!!artists.hasNextPage} + onLoadMore={() => artists.fetchNextPage()} + > + {#snippet item(a: WantArtist)} + + {/snippet} + + + artists.fetchNextPage()} + /> +{/if} diff --git a/web/src/lib/components/wanted/CatalogueGrid.test.ts b/web/src/lib/components/wanted/CatalogueGrid.test.ts new file mode 100644 index 00000000..3b4bc138 --- /dev/null +++ b/web/src/lib/components/wanted/CatalogueGrid.test.ts @@ -0,0 +1,103 @@ +import { afterEach, describe, expect, test, vi } from 'vitest'; +import { fireEvent, render, screen, waitFor } from '@testing-library/svelte'; +import { mockInfiniteQuery } from '../../../test-utils/query'; +import type { LibraryListing, WantArtist, WantArtistsPage } from '#lib/api/types.js'; + +vi.mock('#lib/api/wanted.js', () => ({ createWantArtistsQuery: vi.fn() })); + +import CatalogueGrid from './CatalogueGrid.svelte'; +import { createWantArtistsQuery } from '#lib/api/wanted.js'; + +const DAY = 24 * 3_600_000; +const ago = (ms: number) => new Date(Date.now() - ms).toISOString(); + +function artist(mbid: string, name: string, extra: Partial = {}): WantArtist { + return { + artist_mbid: mbid, + artist_name: name, + sort_name: name, + image_url: `/api/catalogue/art/artist/${mbid}`, + albums: 4, + held: 3, + wanted: 1, + missing: 0, + downloading: 0, + following: false, + ...extra + }; +} + +function setup( + items: WantArtist[] = [ + artist('blur', 'Blur', { following: true, downloading: 1 }), + artist('beatles', 'The Beatles', { sort_name: 'Beatles, The', albums: 13, held: 13, wanted: 0 }) + ], + library: LibraryListing = { listed_at: ago(DAY), unlisted: 0, unmatched: [] } +) { + const page: WantArtistsPage = { + items, + total: items.length, + limit: 96, + offset: 0, + counts: { artists: items.length, wanted: 1, missing: 0, held: 16, following: 1, activity: 1, attention: 0, blocked: 0 }, + library + }; + vi.mocked(createWantArtistsQuery).mockReturnValue( + mockInfiniteQuery({ pages: [page] }) as unknown as ReturnType + ); + return render(CatalogueGrid); +} + +afterEach(() => vi.clearAllMocks()); + +describe('CatalogueGrid', () => { + test('each artist is a card: how many albums are held, and whether they are followed', () => { + setup(); + const cards = screen.getAllByTestId('catalogue-artist'); + expect(cards.map((c) => c.getAttribute('href'))).toEqual(['/collection/artist/blur', '/collection/artist/beatles']); + expect(cards[0].textContent?.replace(/\s+/g, ' ')).toMatch(/Blur 3 \/ 4 held Following · 1 downloading/); + const bar = screen.getByRole('progressbar', { name: '13 of 13 albums held' }); + expect(bar.getAttribute('aria-valuenow')).toBe('13'); + expect((bar.firstElementChild as HTMLElement).style.width).toBe('100%'); + }); + + test('the alphabet rail files an artist under their sort name', () => { + setup(); + expect(document.getElementById('alpha-B')?.textContent).toMatch(/Blur/); + }); + + test('the filter asks the server for artists by name or album', async () => { + setup(); + await fireEvent.input(screen.getByRole('searchbox', { name: /filter artists and albums/i }), { + target: { value: 'parklife' } + }); + await waitFor(() => expect(createWantArtistsQuery).toHaveBeenLastCalledWith('parklife')); + }); + + test('an empty catalogue says what lands there', () => { + setup([]); + expect(screen.getByText(/the catalogue is empty/i)).toBeTruthy(); + }); + + test('says while the library is being listed', () => { + setup([], { listed_at: null, unlisted: 0, unmatched: [] }); + expect(screen.getByTestId('library-listing').textContent).toMatch(/being added to the catalogue/i); + }); + + test('names the albums no MusicBrainz album matched, on request', async () => { + setup(undefined, { + listed_at: ago(DAY), + unlisted: 12, + unmatched: [ + { album_id: 'al-9', title: 'Demo Tape', artist: 'Blur', detail: 'MusicBrainz has no album called "Demo Tape" by Blur.' } + ] + }); + expect(screen.getByTestId('library-listing').textContent).toMatch(/12 albums in your library\s+aren't in the catalogue/i); + expect(screen.queryByTestId('unmatched-albums')).toBeNull(); + await fireEvent.click(screen.getByRole('button', { name: /show them/i })); + const list = screen.getByTestId('unmatched-albums'); + expect(list.textContent).toMatch(/Demo Tape/); + expect(screen.getByRole('link', { name: 'Demo Tape' }).getAttribute('href')).toBe('/albums/al-9'); + expect(screen.getByText(/the first 1 of 12/i)).toBeTruthy(); + }); +}); diff --git a/web/src/lib/components/wanted/CollectionRail.svelte b/web/src/lib/components/wanted/CollectionRail.svelte new file mode 100644 index 00000000..a2d2e52b --- /dev/null +++ b/web/src/lib/components/wanted/CollectionRail.svelte @@ -0,0 +1,58 @@ + + + diff --git a/web/src/lib/components/wanted/WantList.svelte b/web/src/lib/components/wanted/WantList.svelte index 3771a890..bd3ae632 100644 --- a/web/src/lib/components/wanted/WantList.svelte +++ b/web/src/lib/components/wanted/WantList.svelte @@ -7,50 +7,51 @@ import { pushToast } from '#lib/stores/toast.svelte.js'; import { relativeTime } from '#lib/utils/relativeTime.js'; import ListContinuation from '#lib/components/ListContinuation.svelte'; + import Cover from '#lib/components/Cover.svelte'; + import FollowControl from '#lib/components/FollowControl.svelte'; import QuickFilter from '#lib/components/QuickFilter.svelte'; import WantCandidates from '#lib/components/WantCandidates.svelte'; import WantTracks from './WantTracks.svelte'; import type { SearchResult, Want, WantState } from '#lib/api/types.js'; - // One tab of the Wanted view (M502 #5520–#5522): the albums still wanted, - // the missing albums wanted back, or the ones held. Each open want says - // what its last search found and how its download is getting on; Releases - // lists every candidate with its verdict, and Search runs one now. The - // library's albums were put on the list as held once (#5567), each matched to - // MusicBrainz (#5569); taking one off leaves its files and stops it being - // fetched back if they go. The ones no match was found for are named here. + // Albums on the Collection's list (M502 #5520–#5522): the ones still + // wanted, the missing ones wanted back, or, given an artist, every album + // of theirs in the catalogue, held or not, under a header (#5572). Each + // open want says what its last search found and how its download is + // getting on; Releases lists every candidate with its verdict, and Search + // runs one now. Taking a held album off leaves its files and stops it + // being fetched back if they go. // // The list reads like the library (#5572): artists A–Z, each with its // albums oldest first, and each album's tracks a click away. The filter // narrows it to an artist or album by name, on the server, so it finds // what has not been scrolled to yet. - let { list }: { list: WantState } = $props(); + let { list = 'wanted', artist = '' }: { list?: WantState; artist?: string } = $props(); const client = useQueryClient(); let filter = $state(''); const q = $derived(filter.trim()); - const wantsStore = $derived(createWantsQuery(list, q)); + const wantsStore = $derived(artist ? createWantsQuery('all', '', artist) : createWantsQuery(list, q)); const wants = $derived($wantsStore); const items = $derived((wants.data?.pages ?? []).flatMap((p) => p.items) as Want[]); - const library = $derived(wants.data?.pages[0]?.library); + const head = $derived(wants.data?.pages[0]?.artist); // The server sends the list artist by artist; each run of one artist's // albums is a section. - type ArtistSection = { key: string; name: string; artistId?: string; wants: Want[] }; + type ArtistSection = { key: string; name: string; mbid: string; wants: Want[] }; const sections = $derived.by(() => { const out: ArtistSection[] = []; for (const w of items) { const key = w.artist_mbid || w.artist_name; const last = out[out.length - 1]; if (last && last.key === key) last.wants.push(w); - else out.push({ key, name: w.artist_name, artistId: w.artist_id, wants: [w] }); + else out.push({ key, name: w.artist_name, mbid: w.artist_mbid, wants: [w] }); } return out; }); let busy = $state(null); - let showUnmatched = $state(false); let searching = $state(null); let open = $state(null); let shown = $state(null); @@ -187,53 +188,30 @@ } -{#if list === 'held' && library} - {#if !library.listed_at} -

- Your library's albums are being added to this list as held. Albums whose tags don't name them - are matched on MusicBrainz first, by their artist, title and tracks, about one a second. -

- {:else if library.unlisted > 0} -
-

- {library.unlisted === 1 ? '1 album' : `${library.unlisted} albums`} in your library - {library.unlisted === 1 ? "isn't" : "aren't"} on this list: no MusicBrainz album matched - {library.unlisted === 1 ? 'its' : 'their'} artist, title and tracks. Until - {library.unlisted === 1 ? "it's" : "they're"} matched, {library.unlisted === 1 ? "it isn't" : "they aren't"} - fetched back if {library.unlisted === 1 ? 'its' : 'their'} files go missing. -

- {#if library.unmatched.length > 0} - - {#if showUnmatched} -
    - {#each library.unmatched as u (u.album_id)} -
  • - {u.title} - · {u.artist} -
    {u.detail}
    -
    • - {/each} -
- {#if library.unmatched.length < library.unlisted} -

- The first {library.unmatched.length} of {library.unlisted}. -

- {/if} - {/if} - {/if} +{#if artist && head} +
+
+
- {/if} +
+ ← Catalogue +

{head.artist_name}

+

+ {head.held} of {albums(head.albums)} held{head.wanted ? ` · ${head.wanted} wanted` : ''}{head.missing + ? ` · ${head.missing} with files missing` + : ''}{head.downloading ? ` · ${head.downloading} downloading` : ''} +

+
+ + {#if head.artist_id} + Open in the library + {/if} +
+
+
{/if} -{#if !wants.isPending && (items.length > 0 || q)} +{#if !artist && !wants.isPending && (items.length > 0 || q)}
@@ -242,7 +220,7 @@ {#if wants.isPending}

Reading the wanted list…

{:else if wants.isError && !wants.isFetchNextPageError} -

Couldn't load the wanted list.

+

Couldn't load {artist ? "this artist's albums" : 'the wanted list'}.

{:else if items.length === 0 && q}

Nothing on this list has an artist or album named like '{q}'. @@ -251,7 +229,7 @@

- {#if list === 'held'}Nothing on the list has been held yet. + {#if artist}None of this artist's albums are in the catalogue. {:else if list === 'missing'}No missing albums are wanted back. {:else}Nothing is wanted.{/if}

@@ -268,16 +246,16 @@
{#each sections as a, i (a.key + ':' + i)}
-

+ {#if !artist}

- {#if a.artistId} - {a.name} + {#if a.mbid} + {a.name} {:else} {a.name} {/if} {albums(a.wants.length)} -

+ {/if}
{:else if arrived(w)} diff --git a/web/src/lib/components/wanted/WantList.test.ts b/web/src/lib/components/wanted/WantList.test.ts index 48f4f426..2ad923ea 100644 --- a/web/src/lib/components/wanted/WantList.test.ts +++ b/web/src/lib/components/wanted/WantList.test.ts @@ -3,18 +3,26 @@ import { fireEvent, render, screen, waitFor } from '@testing-library/svelte'; import { mockInfiniteQuery, mockQuery } from '../../../test-utils/query'; import type { LibraryListing, Want, WantGrab, WantsPage } from '#lib/api/types.js'; -vi.mock('#lib/api/wanted.js', () => ({ - createWantsQuery: vi.fn(), - removeWant: vi.fn().mockResolvedValue(undefined), - searchWant: vi.fn().mockResolvedValue({ - outcome: 'nothing', considered: 0, best_release: '', best_verdict: '', narrowed: [] - }), - createCandidatesQuery: vi.fn(), - createWantTracksQuery: vi.fn(), - grabCandidate: vi.fn(), - blockCandidate: vi.fn(), - unblockRelease: vi.fn() -})); +vi.mock('#lib/api/wanted.js', async () => { + const { mockQuery } = await import('../../../test-utils/query'); + return { + // The artist page's FollowControl. + createFollowQuery: vi.fn(() => mockQuery({ data: { following: true } })), + createReleaseProfilesQuery: vi.fn(() => mockQuery({ data: { profiles: [], primary_types: [], secondary_types: [] } })), + putFollow: vi.fn(), + deleteFollow: vi.fn(), + createWantsQuery: vi.fn(), + removeWant: vi.fn().mockResolvedValue(undefined), + searchWant: vi.fn().mockResolvedValue({ + outcome: 'nothing', considered: 0, best_release: '', best_verdict: '', narrowed: [] + }), + createCandidatesQuery: vi.fn(), + createWantTracksQuery: vi.fn(), + grabCandidate: vi.fn(), + blockCandidate: vi.fn(), + unblockRelease: vi.fn() + }; +}); vi.mock('#lib/stores/toast.svelte.js', () => ({ pushToast: vi.fn() })); import WantList from './WantList.svelte'; @@ -27,53 +35,55 @@ const ago = (ms: number) => new Date(Date.now() - ms).toISOString(); function want(id: string, extra: Partial = {}): Want { return { - id, - release_group_mbid: `rg-${id}`, - title: 'The Ballad of Darren', - artist_mbid: 'blur', - artist_name: 'Blur', - artist_id: 'art-1', - primary_type: 'Album', - secondary_types: [], - year: '2023', - reason: 'follow', - wanted_at: ago(2 * DAY), - held_at: null, - image_url: '', - ...extra + id, + release_group_mbid: `rg-${id}`, + title: 'The Ballad of Darren', + artist_mbid: 'blur', + artist_name: 'Blur', + artist_id: 'art-1', + primary_type: 'Album', + secondary_types: [], + year: '2023', + reason: 'follow', + wanted_at: ago(2 * DAY), + held_at: null, + image_url: '', + ...extra }; } function page( items: Want[], - library: LibraryListing = { listed_at: ago(DAY), unlisted: 0, unmatched: [] } + library: LibraryListing = { listed_at: ago(DAY), unlisted: 0, unmatched: [] }, + artist?: WantsPage['artist'] ): WantsPage { return { - items, - total: items.length, - limit: 50, - offset: 0, - counts: { wanted: 2, missing: 1, held: 4, following: 1, activity: 1, attention: 0, blocked: 0 }, - library + artist, + items, + total: items.length, + limit: 50, + offset: 0, + counts: { artists: 2, wanted: 2, missing: 1, held: 4, following: 1, activity: 1, attention: 0, blocked: 0 }, + library }; } function grab(extra: Partial = {}): WantGrab { return { - id: 'g1', - name: 'Blur - Blur (1997) [FLAC]', - grabbed_at: ago(60_000), - origin: 'schedule', - quality: 'lossless', - live: true, - end_reason: '', - progress: 0, - import_state: '', - import_reason: '', - end_detail: '', - imported_at: null, - imported_files: 0, - ...extra + id: 'g1', + name: 'Blur - Blur (1997) [FLAC]', + grabbed_at: ago(60_000), + origin: 'schedule', + quality: 'lossless', + live: true, + end_reason: '', + progress: 0, + import_state: '', + import_reason: '', + end_detail: '', + imported_at: null, + imported_files: 0, + ...extra }; } @@ -133,29 +143,6 @@ describe('WantList', () => { await waitFor(() => expect(removeWant).toHaveBeenCalledWith('w1')); }); - test('the held tab says while the library is being listed', () => { - setup([], 'held', { listed_at: null, unlisted: 0, unmatched: [] }); - expect(screen.getByTestId('library-listing').textContent).toMatch(/being added to this list/i); - }); - - test('the held tab names the albums no MusicBrainz album matched, on request', async () => { - setup([want('w1', { held_at: ago(DAY) })], 'held', { - listed_at: ago(DAY), - unlisted: 12, - unmatched: [ - { album_id: 'al-9', title: 'Demo Tape', artist: 'Blur', detail: 'MusicBrainz has no album called "Demo Tape" by Blur.' } - ] - }); - expect(screen.getByTestId('library-listing').textContent).toMatch(/12 albums in your library\s+aren't on this list/i); - expect(screen.queryByTestId('unmatched-albums')).toBeNull(); - await fireEvent.click(screen.getByRole('button', { name: /show them/i })); - const list = screen.getByTestId('unmatched-albums'); - expect(list.textContent).toMatch(/Demo Tape/); - expect(list.textContent).toMatch(/has no album called/); - expect(screen.getByRole('link', { name: 'Demo Tape' }).getAttribute('href')).toBe('/albums/al-9'); - expect(screen.getByText(/the first 1 of 12/i)).toBeTruthy(); - }); - test('an empty list explains what lands there', () => { setup([]); expect(screen.getByText(/nothing is wanted/i)).toBeTruthy(); @@ -184,7 +171,7 @@ describe('WantList', () => { ]); const line = screen.getByTestId('want-search'); expect(line.textContent).toMatch(/needs you: only the lengths agree/i); - expect(screen.getByRole('link', { name: /decide on activity/i }).getAttribute('href')).toBe('/wanted/activity'); + expect(screen.getByRole('link', { name: /decide on activity/i }).getAttribute('href')).toBe('/collection/activity'); // A live download can't be searched for again. expect((screen.getByRole('button', { name: /search for the ballad of darren now/i }) as HTMLButtonElement).disabled).toBe(true); }); @@ -251,9 +238,9 @@ describe('WantList', () => { expect(sections[0].querySelectorAll('[data-testid="want-row"]').length).toBe(2); expect(sections[0].textContent).toMatch(/2 albums/); expect(sections[1].textContent).toMatch(/1 album/); - // The artist links into the library when it holds them. - expect(screen.getByRole('link', { name: 'Blur' }).getAttribute('href')).toBe('/artists/art-1'); - expect(screen.queryByRole('link', { name: 'Camel' })).toBeNull(); + // Each artist opens on their page in the catalogue. + expect(screen.getByRole('link', { name: 'Blur' }).getAttribute('href')).toBe('/collection/artist/blur'); + expect(screen.getByRole('link', { name: 'Camel' }).getAttribute('href')).toBe('/collection/artist/camel'); }); test('Tracks shows the album\'s tracks, with the edition they come from', async () => { @@ -291,4 +278,40 @@ describe('WantList', () => { }); await waitFor(() => expect(createWantsQuery).toHaveBeenLastCalledWith('wanted', 'park')); }); + + test("an artist's page lists every album of theirs under a header", () => { + vi.mocked(createWantsQuery).mockReturnValue( + mockInfiniteQuery({ + pages: [ + page([want('w1', { held_at: ago(DAY), album_id: 'al-1', title: 'Parklife' }), want('w2')], undefined, { + artist_mbid: 'blur', + artist_name: 'Blur', + sort_name: 'Blur', + artist_id: 'art-1', + image_url: '/art/blur', + albums: 2, + held: 1, + wanted: 1, + missing: 0, + downloading: 0, + following: true + }) + ] + }) as unknown as ReturnType + ); + vi.mocked(createCandidatesQuery).mockReturnValue( + mockQuery({ data: [] }) as unknown as ReturnType + ); + render(WantList, { artist: 'blur' }); + expect(createWantsQuery).toHaveBeenCalledWith('all', '', 'blur'); + const head = screen.getByTestId('catalogue-artist-head'); + expect(head.textContent).toMatch(/Blur/); + expect(screen.getByTestId('catalogue-artist-counts').textContent).toMatch(/1 of 2 albums held\s+·\s+1 wanted/); + expect(screen.getByRole('link', { name: /open in the library/i }).getAttribute('href')).toBe('/artists/art-1'); + expect(screen.getByRole('link', { name: /catalogue/i }).getAttribute('href')).toBe('/collection'); + // One artist: no filter, no artist headings, both albums. + expect(screen.queryByRole('searchbox')).toBeNull(); + expect(screen.getAllByTestId('want-row')).toHaveLength(2); + expect(screen.queryByRole('link', { name: 'Blur' })).toBeNull(); + }); }); diff --git a/web/src/routes/admin/missing-files/+page.svelte b/web/src/routes/admin/missing-files/+page.svelte index fb89e25d..656a3cb4 100644 --- a/web/src/routes/admin/missing-files/+page.svelte +++ b/web/src/routes/admin/missing-files/+page.svelte @@ -107,7 +107,7 @@ {#if r.held_at} Held again {relativeTime(r.held_at)}. The rows here clear on the next scan. {:else if r.wanted_at} - On the wanted list since {relativeTime(r.wanted_at)}. {/if} diff --git a/web/src/routes/admin/missing-files/missing-files.test.ts b/web/src/routes/admin/missing-files/missing-files.test.ts index cb804653..a9a113c9 100644 --- a/web/src/routes/admin/missing-files/missing-files.test.ts +++ b/web/src/routes/admin/missing-files/missing-files.test.ts @@ -140,7 +140,7 @@ describe('admin missing files', () => { expect( within(screen.getByTestId('reacquisition-state')).getByRole('link', { name: /wanted list/i }).getAttribute('href') ).toBe( - '/wanted/missing' + '/collection/missing' ); }); diff --git a/web/src/routes/admin/requests/+page.svelte b/web/src/routes/admin/requests/+page.svelte index b2952c24..325eca16 100644 --- a/web/src/routes/admin/requests/+page.svelte +++ b/web/src/routes/admin/requests/+page.svelte @@ -146,7 +146,7 @@

Requests

Approve or set aside what users have asked for. An approved album or track goes on the - wanted list; an approved + wanted list; an approved artist is followed, wanting what they've released and what they release next.

diff --git a/web/src/routes/collection/+layout.svelte b/web/src/routes/collection/+layout.svelte new file mode 100644 index 00000000..3da3864e --- /dev/null +++ b/web/src/routes/collection/+layout.svelte @@ -0,0 +1,58 @@ + + +
+
+

Collection

+

+ Every artist and album in your collection: what the library holds, and what Minstrel is + getting for it, from user requests, new releases of the artists it follows, and albums whose + files went missing. It searches your indexers on a schedule, checks each download against + MusicBrainz, and files what's right into the library. Searching and quality are set under + Integrations. +

+ {#if verdict} +

+ {verdict} + {#if counts?.attention} + See Activity + {/if} +

+ {/if} +
+ +
+
+
+ +
+
+
+ {@render children?.()} +
+
+
diff --git a/web/src/routes/wanted/+layout.ts b/web/src/routes/collection/+layout.ts similarity index 100% rename from web/src/routes/wanted/+layout.ts rename to web/src/routes/collection/+layout.ts diff --git a/web/src/routes/collection/+page.svelte b/web/src/routes/collection/+page.svelte new file mode 100644 index 00000000..5a07e2e2 --- /dev/null +++ b/web/src/routes/collection/+page.svelte @@ -0,0 +1,8 @@ + + +{pageTitle('Collection')} + + diff --git a/web/src/routes/wanted/activity/+page.svelte b/web/src/routes/collection/activity/+page.svelte similarity index 67% rename from web/src/routes/wanted/activity/+page.svelte rename to web/src/routes/collection/activity/+page.svelte index c98588ee..d36f1008 100644 --- a/web/src/routes/wanted/activity/+page.svelte +++ b/web/src/routes/collection/activity/+page.svelte @@ -3,6 +3,6 @@ import DownloadList from '#lib/components/wanted/DownloadList.svelte'; -{pageTitle('Wanted · Activity')} +{pageTitle('Collection · Activity')} diff --git a/web/src/routes/collection/artist/[mbid]/+page.svelte b/web/src/routes/collection/artist/[mbid]/+page.svelte new file mode 100644 index 00000000..9732e908 --- /dev/null +++ b/web/src/routes/collection/artist/[mbid]/+page.svelte @@ -0,0 +1,15 @@ + + +{pageTitle('Collection · Artist')} + +{#key mbid} + +{/key} diff --git a/web/src/routes/wanted/blocked/+page.svelte b/web/src/routes/collection/blocked/+page.svelte similarity index 66% rename from web/src/routes/wanted/blocked/+page.svelte rename to web/src/routes/collection/blocked/+page.svelte index 3c4ea0d0..07c7b02d 100644 --- a/web/src/routes/wanted/blocked/+page.svelte +++ b/web/src/routes/collection/blocked/+page.svelte @@ -3,6 +3,6 @@ import BlockList from '#lib/components/wanted/BlockList.svelte'; -{pageTitle('Wanted · Blocked')} +{pageTitle('Collection · Blocked')} diff --git a/web/src/routes/wanted/wanted.test.ts b/web/src/routes/collection/collection.test.ts similarity index 52% rename from web/src/routes/wanted/wanted.test.ts rename to web/src/routes/collection/collection.test.ts index 4b1bb38d..ce1aab53 100644 --- a/web/src/routes/wanted/wanted.test.ts +++ b/web/src/routes/collection/collection.test.ts @@ -3,23 +3,23 @@ import { render, screen } from '@testing-library/svelte'; import { createRawSnippet } from 'svelte'; import { mockInfiniteQuery } from '../../test-utils/query'; import { pageUrlModule } from '#test-utils/mocks/appState.js'; -import type { WantCounts, WantsPage } from '#lib/api/types.js'; +import type { WantArtistsPage, WantCounts } from '#lib/api/types.js'; -const state = vi.hoisted(() => ({ pageUrl: new URL('http://localhost/wanted') })); +const state = vi.hoisted(() => ({ pageUrl: new URL('http://localhost/collection') })); vi.mock('$app/state', () => pageUrlModule(state)); -vi.mock('#lib/api/wanted.js', () => ({ createWantsQuery: vi.fn() })); +vi.mock('#lib/api/wanted.js', () => ({ createWantArtistsQuery: vi.fn() })); import WantedLayout from './+layout.svelte'; -import { createWantsQuery } from '#lib/api/wanted.js'; +import { createWantArtistsQuery } from '#lib/api/wanted.js'; -const counts: WantCounts = { wanted: 5, missing: 1, held: 9, following: 2, activity: 3, attention: 0, blocked: 4 }; +const counts: WantCounts = { artists: 7, wanted: 5, missing: 1, held: 9, following: 2, activity: 3, attention: 0, blocked: 4 }; const children = createRawSnippet(() => ({ render: () => `

body

` })); afterEach(() => vi.clearAllMocks()); -function setup(c: WantCounts = counts, path = '/wanted') { +function setup(c: WantCounts = counts, path = '/collection') { state.pageUrl = new URL(`http://localhost${path}`); - const page: WantsPage = { + const page: WantArtistsPage = { items: [], total: 0, limit: 50, @@ -27,42 +27,47 @@ function setup(c: WantCounts = counts, path = '/wanted') { counts: c, library: { listed_at: null, unlisted: 0, unmatched: [] } }; - vi.mocked(createWantsQuery).mockReturnValue( - mockInfiniteQuery({ pages: [page] }) as unknown as ReturnType + vi.mocked(createWantArtistsQuery).mockReturnValue( + mockInfiniteQuery({ pages: [page] }) as unknown as ReturnType ); // eslint-disable-next-line @typescript-eslint/no-explicit-any return render(WantedLayout, { children } as any); } -describe('/wanted layout', () => { - test('tabs are links carrying their counts, the current one marked', () => { - setup(counts, '/wanted/activity'); - const nav = screen.getByRole('navigation', { name: /wanted sections/i }); +describe('/collection layout', () => { + test('the sections down the left carry their counts, the current one marked', () => { + setup(counts, '/collection/activity'); + const nav = screen.getByRole('navigation', { name: /collection sections/i }); const links = Array.from(nav.querySelectorAll('a')).map((a) => [a.getAttribute('href'), a.textContent?.replace(/\s+/g, ' ').trim()]); expect(links).toEqual([ - ['/wanted', 'Wanted 5'], - ['/wanted/activity', 'Activity 3'], - ['/wanted/missing', 'Missing files 1'], - ['/wanted/held', 'Held 9'], - ['/wanted/following', 'Following 2'], - ['/wanted/blocked', 'Blocked 4'] + ['/collection', 'Catalogue 7'], + ['/collection/wanted', 'Wanted 5'], + ['/collection/missing', 'Missing files 1'], + ['/collection/activity', 'Activity 3'], + ['/collection/following', 'Following 2'], + ['/collection/blocked', 'Blocked 4'] ]); expect(screen.getByRole('link', { name: /activity/i })).toHaveAttribute('aria-current', 'page'); - expect(screen.getByRole('link', { name: /^wanted/i })).not.toHaveAttribute('aria-current'); + expect(screen.getByRole('link', { name: /^catalogue/i })).not.toHaveAttribute('aria-current'); expect(screen.getByTestId('tab-body')).toBeTruthy(); }); test('when nothing needs a person the verdict says so', () => { setup(); - expect(screen.getByTestId('wanted-verdict').textContent).toMatch(/5 albums wanted, 3 downloading\. Nothing needs you\./); + expect(screen.getByTestId('collection-verdict').textContent).toMatch(/5 albums wanted, 3 downloading\. Nothing needs you\./); expect(screen.queryByTestId('tab-attention')).toBeNull(); }); test('downloads waiting on a person lead the page and mark Activity', () => { setup({ ...counts, attention: 2 }); - const verdict = screen.getByTestId('wanted-verdict'); + const verdict = screen.getByTestId('collection-verdict'); expect(verdict.textContent).toMatch(/2 downloads need you/); - expect(screen.getByRole('link', { name: /see activity/i }).getAttribute('href')).toBe('/wanted/activity'); + expect(screen.getByRole('link', { name: /see activity/i }).getAttribute('href')).toBe('/collection/activity'); expect(screen.getByTestId('tab-attention').textContent).toBe('2'); }); + + test("an artist's page sits under Catalogue", () => { + setup(counts, '/collection/artist/blur'); + expect(screen.getByRole('link', { name: /^catalogue/i })).toHaveAttribute('aria-current', 'page'); + }); }); diff --git a/web/src/routes/wanted/following/+page.svelte b/web/src/routes/collection/following/+page.svelte similarity index 78% rename from web/src/routes/wanted/following/+page.svelte rename to web/src/routes/collection/following/+page.svelte index 7303fe95..b891a091 100644 --- a/web/src/routes/wanted/following/+page.svelte +++ b/web/src/routes/collection/following/+page.svelte @@ -4,7 +4,7 @@ import ReleaseProfilesCard from '#lib/components/ReleaseProfilesCard.svelte'; -{pageTitle('Wanted · Following')} +{pageTitle('Collection · Following')}
diff --git a/web/src/routes/wanted/missing/+page.svelte b/web/src/routes/collection/missing/+page.svelte similarity index 66% rename from web/src/routes/wanted/missing/+page.svelte rename to web/src/routes/collection/missing/+page.svelte index 35163ed2..a06ded29 100644 --- a/web/src/routes/wanted/missing/+page.svelte +++ b/web/src/routes/collection/missing/+page.svelte @@ -3,6 +3,6 @@ import WantList from '#lib/components/wanted/WantList.svelte'; -{pageTitle('Wanted · Missing files')} +{pageTitle('Collection · Missing files')} diff --git a/web/src/routes/wanted/+page.svelte b/web/src/routes/collection/wanted/+page.svelte similarity index 68% rename from web/src/routes/wanted/+page.svelte rename to web/src/routes/collection/wanted/+page.svelte index 9d6f7d36..9655cd2c 100644 --- a/web/src/routes/wanted/+page.svelte +++ b/web/src/routes/collection/wanted/+page.svelte @@ -3,6 +3,6 @@ import WantList from '#lib/components/wanted/WantList.svelte'; -{pageTitle('Wanted')} +{pageTitle('Collection · Wanted')} diff --git a/web/src/routes/wanted/+layout.svelte b/web/src/routes/wanted/+layout.svelte deleted file mode 100644 index 0e2116bf..00000000 --- a/web/src/routes/wanted/+layout.svelte +++ /dev/null @@ -1,48 +0,0 @@ - - -
-
-

Wanted

-

- Albums Minstrel is getting: ones users requested, new releases from the artists it follows, - and albums whose files went missing. It searches your indexers on a schedule, checks each - download against MusicBrainz, and files what's right into the library. Searching and quality - are set under Integrations. -

- {#if verdict} -

- {verdict} - {#if counts?.attention} - See Activity - {/if} -

- {/if} -
- - - - {@render children?.()} -
diff --git a/web/src/routes/wanted/held/+page.svelte b/web/src/routes/wanted/held/+page.svelte deleted file mode 100644 index 652d8b20..00000000 --- a/web/src/routes/wanted/held/+page.svelte +++ /dev/null @@ -1,8 +0,0 @@ - - -{pageTitle('Wanted · Held')} - - -- 2.54.0 From 10be068dccb2ec2d002a0861fcd9d6b00360ce84 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Sat, 10 Oct 2026 22:12:29 -0400 Subject: [PATCH 027/117] fix: the player bar's heart is Lucide's, outlined then filled as on Android, and its title and artist read at the size of the rest of the page (#5580) Co-Authored-By: Claude Opus 5.5 --- web/src/lib/components/LikeButton.svelte | 17 ++++++++++++----- web/src/lib/components/PlayerBar.svelte | 16 ++++++++++------ 2 files changed, 22 insertions(+), 11 deletions(-) diff --git a/web/src/lib/components/LikeButton.svelte b/web/src/lib/components/LikeButton.svelte index 34fffcf7..b4b1df5c 100644 --- a/web/src/lib/components/LikeButton.svelte +++ b/web/src/lib/components/LikeButton.svelte @@ -1,4 +1,5 @@ {#if library && !library.listed_at} @@ -29,41 +27,6 @@ Your library's albums are being added to the catalogue. Albums whose tags don't name them are matched on MusicBrainz first, by their artist, title and tracks, about one a second.

-{:else if library && library.unlisted > 0} -
-

- {library.unlisted === 1 ? '1 album' : `${library.unlisted} albums`} in your library - {library.unlisted === 1 ? "isn't" : "aren't"} in the catalogue: no MusicBrainz album matched - {library.unlisted === 1 ? 'its' : 'their'} artist, title and tracks. Until - {library.unlisted === 1 ? "it's" : "they're"} matched, {library.unlisted === 1 ? "it isn't" : "they aren't"} - fetched back if {library.unlisted === 1 ? 'its' : 'their'} files go missing. -

- {#if library.unmatched.length > 0} - - {#if showUnmatched} -
    - {#each library.unmatched as u (u.album_id)} -
  • - {u.title} - · {u.artist} -
    {u.detail}
    -
    • - {/each} -
- {#if library.unmatched.length < library.unlisted} -

The first {library.unmatched.length} of {library.unlisted}.

- {/if} - {/if} - {/if} -
{/if} {#if !artists.isPending && (items.length > 0 || q)} diff --git a/web/src/lib/components/wanted/CatalogueGrid.test.ts b/web/src/lib/components/wanted/CatalogueGrid.test.ts index 3b4bc138..cceb1cdc 100644 --- a/web/src/lib/components/wanted/CatalogueGrid.test.ts +++ b/web/src/lib/components/wanted/CatalogueGrid.test.ts @@ -32,14 +32,14 @@ function setup( artist('blur', 'Blur', { following: true, downloading: 1 }), artist('beatles', 'The Beatles', { sort_name: 'Beatles, The', albums: 13, held: 13, wanted: 0 }) ], - library: LibraryListing = { listed_at: ago(DAY), unlisted: 0, unmatched: [] } + library: LibraryListing = { listed_at: ago(DAY), unlisted: 0 } ) { const page: WantArtistsPage = { items, total: items.length, limit: 96, offset: 0, - counts: { artists: items.length, wanted: 1, missing: 0, held: 16, following: 1, activity: 1, attention: 0, blocked: 0 }, + counts: { artists: items.length, wanted: 1, missing: 0, held: 16, following: 1, activity: 1, attention: 0, blocked: 0, unmatched: 0 }, library }; vi.mocked(createWantArtistsQuery).mockReturnValue( @@ -80,24 +80,7 @@ describe('CatalogueGrid', () => { }); test('says while the library is being listed', () => { - setup([], { listed_at: null, unlisted: 0, unmatched: [] }); + setup([], { listed_at: null, unlisted: 0 }); expect(screen.getByTestId('library-listing').textContent).toMatch(/being added to the catalogue/i); }); - - test('names the albums no MusicBrainz album matched, on request', async () => { - setup(undefined, { - listed_at: ago(DAY), - unlisted: 12, - unmatched: [ - { album_id: 'al-9', title: 'Demo Tape', artist: 'Blur', detail: 'MusicBrainz has no album called "Demo Tape" by Blur.' } - ] - }); - expect(screen.getByTestId('library-listing').textContent).toMatch(/12 albums in your library\s+aren't in the catalogue/i); - expect(screen.queryByTestId('unmatched-albums')).toBeNull(); - await fireEvent.click(screen.getByRole('button', { name: /show them/i })); - const list = screen.getByTestId('unmatched-albums'); - expect(list.textContent).toMatch(/Demo Tape/); - expect(screen.getByRole('link', { name: 'Demo Tape' }).getAttribute('href')).toBe('/albums/al-9'); - expect(screen.getByText(/the first 1 of 12/i)).toBeTruthy(); - }); }); diff --git a/web/src/lib/components/wanted/CollectionRail.svelte b/web/src/lib/components/wanted/CollectionRail.svelte index a2d2e52b..6f58c691 100644 --- a/web/src/lib/components/wanted/CollectionRail.svelte +++ b/web/src/lib/components/wanted/CollectionRail.svelte @@ -5,7 +5,9 @@ // The Collection's sections, down the left like Lidarr's (#5572): the // catalogue of artists first, then the sections that show state. On a // narrow screen the rail runs across the top. Activity carries a second, - // accented count when downloads are waiting on a person. + // accented count when downloads are waiting on a person. Unmatched, like + // Lidarr's Unmapped Files, holds the library albums no MusicBrainz album + // matched (#5579). let { counts }: { counts?: WantCounts } = $props(); @@ -17,7 +19,8 @@ { href: '/collection/missing', label: 'Missing files', count: counts?.missing }, { href: '/collection/activity', label: 'Activity', count: counts?.activity, attention: counts?.attention }, { href: '/collection/following', label: 'Following', count: counts?.following }, - { href: '/collection/blocked', label: 'Blocked', count: counts?.blocked } + { href: '/collection/blocked', label: 'Blocked', count: counts?.blocked }, + { href: '/collection/unmatched', label: 'Unmatched', count: counts?.unmatched } ]); // An artist's page belongs to the catalogue. diff --git a/web/src/lib/components/wanted/UnmatchedList.svelte b/web/src/lib/components/wanted/UnmatchedList.svelte new file mode 100644 index 00000000..85d6ce19 --- /dev/null +++ b/web/src/lib/components/wanted/UnmatchedList.svelte @@ -0,0 +1,150 @@ + + +{#if unmatched.isPending} +

Reading the unmatched albums…

+{:else if unmatched.isError} +

Couldn't load the unmatched albums.

+{:else} + {#if toMatch.length === 0} +
+ +

Every album in your library is matched.

+

+ An album no MusicBrainz album matches by its artist, title and tracks lands here, to be + matched by hand or left off. +

+
+ {:else} +

+ No MusicBrainz album matched {toMatch.length === 1 ? 'this album' : `these ${toMatch.length} albums`} + by artist, title and tracks, so {toMatch.length === 1 ? "it isn't" : "they aren't"} fetched back if + {toMatch.length === 1 ? 'its' : 'their'} files go missing. Match each to its MusicBrainz album, or + leave it off if MusicBrainz doesn't have it. +

+
    + {#each toMatch as u (u.album_id)} +
  • +
    +
    + {u.title} +
    {u.artist} · {tracks(u.tracks)}
    +

    {u.detail}

    +
    +
    + + +
    +
    + {#if open === u.album_id} + onMatch(u, ref)} /> + {/if} +
    • + {/each} +
+ {/if} + + {#if leftOff.length > 0} +
+

Left off

+

+ Kept in the library, never matched again, and not fetched back if their files go missing. +

+
    + {#each leftOff as u (u.album_id)} +
  • +
    + {u.title} +
    + {u.artist} · {tracks(u.tracks)} · left off {relativeTime(u.decided_at)} +
    +
    + +
    • + {/each} +
+
+ {/if} +{/if} diff --git a/web/src/lib/components/wanted/UnmatchedList.test.ts b/web/src/lib/components/wanted/UnmatchedList.test.ts new file mode 100644 index 00000000..2245246c --- /dev/null +++ b/web/src/lib/components/wanted/UnmatchedList.test.ts @@ -0,0 +1,109 @@ +import { afterEach, describe, expect, test, vi } from 'vitest'; +import { fireEvent, render, screen, waitFor } from '@testing-library/svelte'; +import type { CatalogueSearchResult, UnmatchedAlbum } from '#lib/api/types.js'; + +vi.mock('#lib/api/wanted.js', async () => { + const { mockQuery } = await import('../../../test-utils/query'); + return { + createUnmatchedQuery: vi.fn(() => mockQuery({ data: [] })), + matchUnmatched: vi.fn().mockResolvedValue({ + release_group_mbid: 'rg-1', + title: 'Songs for the Deaf', + artist: 'Queens of the Stone Age', + agree: 9, + tracks: 10 + }), + leaveUnmatchedOff: vi.fn().mockResolvedValue(undefined), + putUnmatchedBack: vi.fn().mockResolvedValue(undefined) + }; +}); +vi.mock('#lib/api/catalogue.js', () => ({ createCatalogueSearchQuery: vi.fn() })); +vi.mock('#lib/stores/toast.svelte.js', () => ({ pushToast: vi.fn() })); + +import UnmatchedList from './UnmatchedList.svelte'; +import { mockQuery } from '../../../test-utils/query'; +import { createUnmatchedQuery, leaveUnmatchedOff, matchUnmatched, putUnmatchedBack } from '#lib/api/wanted.js'; +import { createCatalogueSearchQuery } from '#lib/api/catalogue.js'; +import { pushToast } from '#lib/stores/toast.svelte.js'; + +const DAY = 24 * 3_600_000; + +function album(id: string, title: string, artist: string, extra: Partial = {}): UnmatchedAlbum { + return { + album_id: id, + title, + artist, + detail: `MusicBrainz has no album called "${title}" by ${artist}.`, + tracks: 10, + left_off: false, + decided_at: new Date(Date.now() - 2 * DAY).toISOString(), + ...extra + }; +} + +const result: CatalogueSearchResult = { + mbid: 'rg-1', + name: 'Songs for the Deaf', + secondary_text: 'Queens of the Stone Age · 2002 · Album', + image_url: '', + artist_mbid: 'qotsa', + album_mbid: '', + in_library: false, + requested: false +}; + +afterEach(() => vi.clearAllMocks()); + +function setup(items: UnmatchedAlbum[], results: CatalogueSearchResult[] = [result]) { + vi.mocked(createUnmatchedQuery).mockReturnValue( + mockQuery({ data: items }) as unknown as ReturnType + ); + vi.mocked(createCatalogueSearchQuery).mockReturnValue( + mockQuery({ data: results }) as unknown as ReturnType + ); + return render(UnmatchedList); +} + +describe('UnmatchedList', () => { + test('each unmatched album says why, with its tracks, and links to the album', () => { + setup([album('al-1', 'Dial Tone', 'Annadale', { tracks: 1 })]); + const row = screen.getByTestId('unmatched-row'); + expect(row.textContent).toMatch(/Annadale · 1 track/); + expect(row.textContent).toMatch(/no album called "Dial Tone"/); + expect(screen.getByRole('link', { name: 'Dial Tone' }).getAttribute('href')).toBe('/albums/al-1'); + }); + + test('Match searches MusicBrainz for the album by its own title and artist, and matches a result', async () => { + setup([album('al-2', 'Songs for the Deaf', 'Queens of the Stone Age')]); + await fireEvent.click(screen.getByRole('button', { name: /match…/i })); + expect(createCatalogueSearchQuery).toHaveBeenLastCalledWith('Songs for the Deaf Queens of the Stone Age', 'album'); + await fireEvent.click(screen.getByRole('button', { name: /match to songs for the deaf/i })); + await waitFor(() => expect(matchUnmatched).toHaveBeenCalledWith('al-2', 'rg-1')); + await waitFor(() => expect(pushToast).toHaveBeenCalledWith(expect.stringMatching(/9 of 10 tracks agree/))); + }); + + test('a pasted MusicBrainz link matches the album', async () => { + setup([album('al-3', '+++', 'DROELOE')]); + await fireEvent.click(screen.getByRole('button', { name: /match…/i })); + const link = 'https://musicbrainz.org/release/11111111-0000-4000-8000-000000000003'; + await fireEvent.input(screen.getByRole('textbox', { name: /musicbrainz album link/i }), { target: { value: link } }); + await fireEvent.click(screen.getByRole('button', { name: /^match$/i })); + await waitFor(() => expect(matchUnmatched).toHaveBeenCalledWith('al-3', link)); + }); + + test("a local band's album is left off, and one left off can be put back", async () => { + setup([album('al-1', 'Dial Tone', 'Annadale'), album('al-4', 'Demo', 'Annadale', { left_off: true })]); + await fireEvent.click(screen.getByRole('button', { name: /leave it off/i })); + await waitFor(() => expect(leaveUnmatchedOff).toHaveBeenCalledWith('al-1')); + const leftOff = screen.getByTestId('left-off-row'); + expect(leftOff.textContent).toMatch(/left off 2d ago/); + await fireEvent.click(screen.getByRole('button', { name: /put back/i })); + await waitFor(() => expect(putUnmatchedBack).toHaveBeenCalledWith('al-4')); + }); + + test('with every album matched it says so, and still lists those left off', () => { + setup([album('al-4', 'Demo', 'Annadale', { left_off: true })]); + expect(screen.getByText(/every album in your library is matched/i)).toBeTruthy(); + expect(screen.getByTestId('left-off')).toBeTruthy(); + }); +}); diff --git a/web/src/lib/components/wanted/UnmatchedMatch.svelte b/web/src/lib/components/wanted/UnmatchedMatch.svelte new file mode 100644 index 00000000..d0ac3942 --- /dev/null +++ b/web/src/lib/components/wanted/UnmatchedMatch.svelte @@ -0,0 +1,106 @@ + + +
+
{ + e.preventDefault(); + asked = text.trim(); + }} + > + + +
+ + {#if search.isPending && asked} +

Asking MusicBrainz…

+ {:else if search.isError} +

Couldn't reach MusicBrainz. A pasted link works the same.

+ {:else if asked && results.length === 0} +

MusicBrainz has no album like that.

+ {:else if results.length > 0} +
    + {#each results as r (r.mbid)} +
  • +
    +
    {r.name}
    +
    {r.secondary_text}
    +
    + MusicBrainz + +
    • + {/each} +
+ {/if} + +
{ + e.preventDefault(); + if (link.trim()) onMatch(link.trim()); + }} + > + + +
+
diff --git a/web/src/lib/components/wanted/WantList.test.ts b/web/src/lib/components/wanted/WantList.test.ts index 2ad923ea..cdf212ce 100644 --- a/web/src/lib/components/wanted/WantList.test.ts +++ b/web/src/lib/components/wanted/WantList.test.ts @@ -54,7 +54,7 @@ function want(id: string, extra: Partial = {}): Want { function page( items: Want[], - library: LibraryListing = { listed_at: ago(DAY), unlisted: 0, unmatched: [] }, + library: LibraryListing = { listed_at: ago(DAY), unlisted: 0 }, artist?: WantsPage['artist'] ): WantsPage { return { @@ -63,7 +63,7 @@ function page( total: items.length, limit: 50, offset: 0, - counts: { artists: 2, wanted: 2, missing: 1, held: 4, following: 1, activity: 1, attention: 0, blocked: 0 }, + counts: { artists: 2, wanted: 2, missing: 1, held: 4, following: 1, activity: 1, attention: 0, blocked: 0, unmatched: 0 }, library }; } diff --git a/web/src/routes/collection/+layout.svelte b/web/src/routes/collection/+layout.svelte index 3da3864e..136345e1 100644 --- a/web/src/routes/collection/+layout.svelte +++ b/web/src/routes/collection/+layout.svelte @@ -21,8 +21,14 @@ } const parts = [counts.wanted === 1 ? '1 album wanted' : `${counts.wanted} albums wanted`]; if (counts.activity) parts.push(`${counts.activity} downloading`); + // Library albums no MusicBrainz album matched wait on a person too (#5579). + if (counts.unmatched) { + const n = counts.unmatched === 1 ? '1 album in your library needs' : `${counts.unmatched} albums in your library need`; + return parts.join(', ') + `. ${n} matching.`; + } return parts.join(', ') + '. Nothing needs you.'; }); + const needsYou = $derived(!!counts?.attention || !!counts?.unmatched);
@@ -36,10 +42,12 @@ Integrations.

{#if verdict} -

+

{verdict} {#if counts?.attention} See Activity + {:else if counts?.unmatched} + See Unmatched {/if}

{/if} diff --git a/web/src/routes/collection/collection.test.ts b/web/src/routes/collection/collection.test.ts index ce1aab53..5fa70f2c 100644 --- a/web/src/routes/collection/collection.test.ts +++ b/web/src/routes/collection/collection.test.ts @@ -12,7 +12,7 @@ vi.mock('#lib/api/wanted.js', () => ({ createWantArtistsQuery: vi.fn() })); import WantedLayout from './+layout.svelte'; import { createWantArtistsQuery } from '#lib/api/wanted.js'; -const counts: WantCounts = { artists: 7, wanted: 5, missing: 1, held: 9, following: 2, activity: 3, attention: 0, blocked: 4 }; +const counts: WantCounts = { artists: 7, wanted: 5, missing: 1, held: 9, following: 2, activity: 3, attention: 0, blocked: 4, unmatched: 3 }; const children = createRawSnippet(() => ({ render: () => `

body

` })); afterEach(() => vi.clearAllMocks()); @@ -25,7 +25,7 @@ function setup(c: WantCounts = counts, path = '/collection') { limit: 50, offset: 0, counts: c, - library: { listed_at: null, unlisted: 0, unmatched: [] } + library: { listed_at: null, unlisted: 0 } }; vi.mocked(createWantArtistsQuery).mockReturnValue( mockInfiniteQuery({ pages: [page] }) as unknown as ReturnType @@ -45,7 +45,8 @@ describe('/collection layout', () => { ['/collection/missing', 'Missing files 1'], ['/collection/activity', 'Activity 3'], ['/collection/following', 'Following 2'], - ['/collection/blocked', 'Blocked 4'] + ['/collection/blocked', 'Blocked 4'], + ['/collection/unmatched', 'Unmatched 3'] ]); expect(screen.getByRole('link', { name: /activity/i })).toHaveAttribute('aria-current', 'page'); expect(screen.getByRole('link', { name: /^catalogue/i })).not.toHaveAttribute('aria-current'); @@ -53,7 +54,7 @@ describe('/collection layout', () => { }); test('when nothing needs a person the verdict says so', () => { - setup(); + setup({ ...counts, unmatched: 0 }); expect(screen.getByTestId('collection-verdict').textContent).toMatch(/5 albums wanted, 3 downloading\. Nothing needs you\./); expect(screen.queryByTestId('tab-attention')).toBeNull(); }); @@ -66,6 +67,14 @@ describe('/collection layout', () => { expect(screen.getByTestId('tab-attention').textContent).toBe('2'); }); + test('library albums waiting to be matched need a person, and link to Unmatched', () => { + setup(); + expect(screen.getByTestId('collection-verdict').textContent).toMatch( + /5 albums wanted, 3 downloading\. 3 albums in your library need matching\./ + ); + expect(screen.getByRole('link', { name: /see unmatched/i }).getAttribute('href')).toBe('/collection/unmatched'); + }); + test("an artist's page sits under Catalogue", () => { setup(counts, '/collection/artist/blur'); expect(screen.getByRole('link', { name: /^catalogue/i })).toHaveAttribute('aria-current', 'page'); diff --git a/web/src/routes/collection/unmatched/+page.svelte b/web/src/routes/collection/unmatched/+page.svelte new file mode 100644 index 00000000..44fa469c --- /dev/null +++ b/web/src/routes/collection/unmatched/+page.svelte @@ -0,0 +1,8 @@ + + +{pageTitle('Collection · Unmatched')} + + -- 2.54.0 From 2f2573dd07a5727331b0d27b319b2080ffc1eaa2 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Sat, 10 Oct 2026 22:23:16 -0400 Subject: [PATCH 029/117] fix: the re-tag test sets its clock one statement at a time, as a prepared statement takes only one (M502 #5579) Co-Authored-By: Claude Opus 5.5 --- internal/wanted/manual_test.go | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/internal/wanted/manual_test.go b/internal/wanted/manual_test.go index 21ea8d5e..7478fb8a 100644 --- a/internal/wanted/manual_test.go +++ b/internal/wanted/manual_test.go @@ -131,11 +131,16 @@ func TestListLibrary_MatchesARetaggedAlbum_Integration(t *testing.T) { AlbumID: untagged.ID, Outcome: "unmatched", Detail: "no match", Matcher: libraryMatcher}); err != nil { t.Fatal(err) } - if _, err := pool.Exec(ctx, ` - UPDATE albums SET created_at = now() - interval '2 hours' WHERE id = $1; - UPDATE tracks SET added_at = now() - interval '2 hours' WHERE album_id = $1; - UPDATE acquisition_settings SET library_listing_started_at = now() - interval '1 hour', - library_listed_at = now() - interval '1 hour', library_unlisted = 1`, untagged.ID); err != nil { + for _, stmt := range []string{ + `UPDATE albums SET created_at = now() - interval '2 hours' WHERE id = $1`, + `UPDATE tracks SET added_at = now() - interval '2 hours' WHERE album_id = $1`, + } { + if _, err := pool.Exec(ctx, stmt, untagged.ID); err != nil { + t.Fatal(err) + } + } + if _, err := pool.Exec(ctx, `UPDATE acquisition_settings SET library_listing_started_at = now() - interval '1 hour', + library_listed_at = now() - interval '1 hour', library_unlisted = 1`); err != nil { t.Fatal(err) } // The re-tag: the same tracks, under the album their new tags name. -- 2.54.0 From c0c0f8a71fde88a2298bb9cf049f89852294ec0f Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Sat, 10 Oct 2026 23:05:05 -0400 Subject: [PATCH 030/117] feat: Now Playing's cover grows to fill the screen, and the page takes the cover's dominant colour as Android does (#5592) The web colour helper now picks Palette's dominant swatch (5-bit quantise, Palette's default filter, median cut to 16, the most-populated box) instead of the 1x1 mean, so the web and Android agree; PlayerBar's accent strip shares it. Now Playing draws Android's gradient (55% at the top, 18% at 45%, then the page) as a masked background colour that fades over 600ms between tracks, holding the old colour until the new resolves. Co-Authored-By: Claude Opus 5.5 --- web/src/lib/media/dominantColor.test.ts | 39 ++++++ web/src/lib/media/dominantColor.ts | 153 +++++++++++++++++++++--- web/src/routes/now-playing/+page.svelte | 53 ++++++-- 3 files changed, 220 insertions(+), 25 deletions(-) create mode 100644 web/src/lib/media/dominantColor.test.ts diff --git a/web/src/lib/media/dominantColor.test.ts b/web/src/lib/media/dominantColor.test.ts new file mode 100644 index 00000000..e450d18a --- /dev/null +++ b/web/src/lib/media/dominantColor.test.ts @@ -0,0 +1,39 @@ +import { describe, expect, test } from 'vitest'; +import { dominantOf } from './dominantColor'; + +// Pixels as RGBA, n of each colour. +function pixels(...runs: [number, number, number, number][]): Uint8ClampedArray { + const out: number[] = []; + for (const [r, g, b, n] of runs) for (let i = 0; i < n; i++) out.push(r, g, b, 255); + return new Uint8ClampedArray(out); +} + +const near = (got: { r: number; g: number; b: number } | null, r: number, g: number, b: number) => + !!got && Math.abs(got.r - r) <= 8 && Math.abs(got.g - g) <= 8 && Math.abs(got.b - b) <= 8; + +describe('dominantOf', () => { + test('the colour covering the most of the cover wins, not a small bright accent', () => { + const got = dominantOf(pixels([40, 70, 140, 700], [250, 20, 60, 100], [30, 160, 60, 200])); + expect(near(got, 40, 70, 140)).toBe(true); + }); + + test('is not the mean: two halves of different colours give one of them', () => { + const got = dominantOf(pixels([200, 40, 40, 520], [40, 40, 200, 480])); + expect(near(got, 200, 40, 40)).toBe(true); + }); + + test('near black and near white are passed over, as Palette does', () => { + const got = dominantOf(pixels([0, 0, 0, 800], [255, 255, 255, 600], [60, 120, 90, 100])); + expect(near(got, 60, 120, 90)).toBe(true); + }); + + test('a cover of nothing but black and white has no colour', () => { + expect(dominantOf(pixels([0, 0, 0, 500], [255, 255, 255, 500]))).toBeNull(); + }); + + test('transparent pixels are not counted', () => { + const data = pixels([40, 70, 140, 100], [200, 40, 40, 300]); + for (let i = 100 * 4; i < data.length; i += 4) data[i + 3] = 0; + expect(near(dominantOf(data), 40, 70, 140)).toBe(true); + }); +}); diff --git a/web/src/lib/media/dominantColor.ts b/web/src/lib/media/dominantColor.ts index 16b76a13..72687480 100644 --- a/web/src/lib/media/dominantColor.ts +++ b/web/src/lib/media/dominantColor.ts @@ -1,19 +1,26 @@ -// Cheap dominant-color extractor: loads an image, draws it scaled -// to a 1x1 canvas, reads the resulting pixel. The browser's bilinear -// downsample produces an arithmetic-mean color, which approximates -// the "dominant" tone close enough for an ambient accent strip. -// Not aiming for ColorThief-grade clustering — that adds ~5KB and -// noticeable extraction latency. The user-visible feature is just -// "the page subtly takes on the current track's hue" and a mean -// color delivers that. +// The colour that covers the most of a cover, as Android takes it +// (player/ui/DominantColor.kt): androidx.palette's dominant swatch, not +// the mean and not the most vibrant. A mean turns a busy cover to mud; +// a vibrant pick lets a small bright accent tint the whole page. The +// dominant swatch matches what the eye sees most of (#5592). // -// Cover URLs are same-origin (/api/albums//cover), so no CORS -// dance is needed. Returns null on any failure (load error, decode -// error, opaque-canvas getImageData reject) so the caller can fall -// back to the static accent. +// Palette's method, kept close so the web and Android agree: shrink the +// image, quantise each pixel to 5 bits a channel, drop the colours its +// default filter drops (near black, near white, and the skin-tone band +// on the red "I line"), split what's left by median cut into at most 16 +// boxes, and take the box holding the most pixels. +// +// Cover URLs are same-origin (/api/albums//cover), so the canvas is +// readable. Returns null on any failure (load or decode error, no 2D +// context, a cover with nothing but black and white) and the caller +// keeps what it showed before. export type Rgb = { r: number; g: number; b: number }; +// Palette shrinks a bitmap to about 112×112 before quantising. +const SAMPLE = 112; +const MAX_COLORS = 16; + const cache = new Map>(); export function dominantColorFromUrl(url: string): Promise { @@ -32,13 +39,12 @@ function sample(url: string): Promise { img.onload = () => { try { const canvas = document.createElement('canvas'); - canvas.width = 1; - canvas.height = 1; + canvas.width = SAMPLE; + canvas.height = SAMPLE; const ctx = canvas.getContext('2d'); if (!ctx) return resolve(null); - ctx.drawImage(img, 0, 0, 1, 1); - const data = ctx.getImageData(0, 0, 1, 1).data; - resolve({ r: data[0], g: data[1], b: data[2] }); + ctx.drawImage(img, 0, 0, SAMPLE, SAMPLE); + resolve(dominantOf(ctx.getImageData(0, 0, SAMPLE, SAMPLE).data)); } catch { resolve(null); } @@ -48,6 +54,119 @@ function sample(url: string): Promise { }); } +type Color5 = { r: number; g: number; b: number; n: number }; + +// dominantOf picks the dominant colour of RGBA pixel data. +export function dominantOf(data: Uint8ClampedArray): Rgb | null { + const hist = new Map(); + for (let i = 0; i < data.length; i += 4) { + if (data[i + 3] < 128) continue; + const key = ((data[i] >> 3) << 10) | ((data[i + 1] >> 3) << 5) | (data[i + 2] >> 3); + hist.set(key, (hist.get(key) ?? 0) + 1); + } + const colors: Color5[] = []; + for (const [key, n] of hist) { + const c = { r: (key >> 10) & 31, g: (key >> 5) & 31, b: key & 31, n }; + if (!ignored(to8(c))) colors.push(c); + } + if (colors.length === 0) return null; + + let best: Rgb | null = null; + let bestN = 0; + for (const box of medianCut(colors, MAX_COLORS)) { + let n = 0, r = 0, g = 0, b = 0; + for (const c of box) { + n += c.n; + r += c.r * c.n; + g += c.g * c.n; + b += c.b * c.n; + } + const avg = to8({ r: r / n, g: g / n, b: b / n }); + if (n > bestN && !ignored(avg)) { + best = avg; + bestN = n; + } + } + return best; +} + +// medianCut splits the colours into at most max boxes, always splitting +// the box with the largest volume, at the population median of its +// longest side. +function medianCut(colors: Color5[], max: number): Color5[][] { + const boxes: Color5[][] = [colors]; + while (boxes.length < max) { + let pick = -1; + let pickVolume = 0; + boxes.forEach((box, i) => { + if (box.length < 2) return; + const v = volume(box); + if (v > pickVolume) { + pick = i; + pickVolume = v; + } + }); + if (pick < 0) break; + const box = boxes[pick]; + const side = longestSide(box); + box.sort((a, b) => a[side] - b[side]); + const total = box.reduce((s, c) => s + c.n, 0); + let run = 0; + let at = box.length - 1; + for (let i = 0; i < box.length - 1; i++) { + run += box[i].n; + if (run >= total / 2) { + at = i + 1; + break; + } + } + boxes.splice(pick, 1, box.slice(0, at), box.slice(at)); + } + return boxes; +} + +function range(box: Color5[], side: 'r' | 'g' | 'b'): number { + let lo = 31, hi = 0; + for (const c of box) { + lo = Math.min(lo, c[side]); + hi = Math.max(hi, c[side]); + } + return hi - lo + 1; +} + +function volume(box: Color5[]): number { + return range(box, 'r') * range(box, 'g') * range(box, 'b'); +} + +function longestSide(box: Color5[]): 'r' | 'g' | 'b' { + const r = range(box, 'r'), g = range(box, 'g'), b = range(box, 'b'); + return r >= g && r >= b ? 'r' : g >= b ? 'g' : 'b'; +} + +function to8(c: { r: number; g: number; b: number }): Rgb { + return { r: Math.round((c.r * 255) / 31), g: Math.round((c.g * 255) / 31), b: Math.round((c.b * 255) / 31) }; +} + +// Palette's default filter: near black, near white, and the band of +// hues around skin tones (10°–37°, not fully saturated). +function ignored({ r, g, b }: Rgb): boolean { + const [h, s, l] = hsl(r, g, b); + return l <= 0.05 || l >= 0.95 || (h >= 10 && h <= 37 && s <= 0.82); +} + +function hsl(r8: number, g8: number, b8: number): [number, number, number] { + const r = r8 / 255, g = g8 / 255, b = b8 / 255; + const max = Math.max(r, g, b), min = Math.min(r, g, b); + const l = (max + min) / 2; + const d = max - min; + if (d === 0) return [0, 0, l]; + const s = d / (1 - Math.abs(2 * l - 1)); + let h = max === r ? ((g - b) / d) % 6 : max === g ? (b - r) / d + 2 : (r - g) / d + 4; + h *= 60; + if (h < 0) h += 360; + return [h, s, l]; +} + export function rgbToCssString({ r, g, b }: Rgb): string { return `rgb(${r} ${g} ${b})`; } diff --git a/web/src/routes/now-playing/+page.svelte b/web/src/routes/now-playing/+page.svelte index e9c90166..4ef02562 100644 --- a/web/src/routes/now-playing/+page.svelte +++ b/web/src/routes/now-playing/+page.svelte @@ -14,6 +14,7 @@ } from '#lib/player/store.svelte.js'; import { formatDuration } from '#lib/media/duration.js'; import { FALLBACK_COVER, coverUrl } from '#lib/media/covers.js'; + import { dominantColorFromUrl, rgbToCssString } from '#lib/media/dominantColor.js'; import { useSmoothPosition } from '#lib/player/smoothPosition.svelte.js'; import LikeButton from '#lib/components/LikeButton.svelte'; import TrackMenu from '#lib/components/TrackMenu.svelte'; @@ -41,6 +42,22 @@ player.volume < 0.5 ? Volume1 : Volume2 ); + // The backdrop takes the cover's dominant colour, as Android's Now + // Playing does (#5592). The colour is held until the next cover's + // resolves, so a track change fades old to new rather than dipping to + // the plain background between them; a track with no cover clears it. + let tint = $state('transparent'); + $effect(() => { + const albumId = current?.album_id; + if (!albumId) { + tint = 'transparent'; + return; + } + dominantColorFromUrl(coverUrl(albumId)).then((rgb) => { + if (rgb && current?.album_id === albumId) tint = rgbToCssString(rgb); + }); + }); + function onSeekInput(e: Event) { const v = Number((e.currentTarget as HTMLInputElement).value); seekTo(v); @@ -66,7 +83,17 @@ {pageTitle(current ? `${current.title} · ${current.artist_name}` : 'Now playing')} -
+
+ +
+