From ebbc49b9024cf32b4e40ea2678b423c92fb797ac Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Sat, 10 Oct 2026 10:36:21 -0400 Subject: [PATCH] feat: import with verification, and a Wanted view of its own (M502 #5522) A finished download is checked against MusicBrainz before anything is filed: the files are paired with an edition's tracks by position or path, and AcoustID, tag ids, lengths and titles say whether it is the album. What is the album is copied into the library as Artist/Album (Year)/Artist - Album - NN - Title, tagged with Picard's MusicBrainz ids, and scanned; the download keeps seeding. What is not is refused: set aside, removed from the client, and the album searched for again. A download the check cannot settle waits for a person, who imports it anyway or refuses it. Tracks the library already holds are skipped. The wanted list leaves Admin for a top-level Wanted view, admin-only and web-only, with tabs for Wanted, Activity (downloads and their imports, the ones needing a person first), Missing files, Held, Following (with the release-type profiles, moved off Integrations) and Blocked. The search settings gain "File new music into" when the library has several folders. The search pass logs why each failed search failed. Co-Authored-By: Claude Opus 5.5 --- README.md | 12 +- cmd/minstrel/main.go | 15 +- internal/acquire/memory.go | 2 + internal/api/admin_import.go | 205 +++++ internal/api/admin_search.go | 21 +- internal/api/admin_wanted.go | 51 +- internal/api/api.go | 19 + internal/db/dbq/imports.sql.go | 719 ++++++++++++++++++ internal/db/dbq/models.go | 45 +- internal/db/dbq/search.sql.go | 43 +- internal/db/dbq/wanted.sql.go | 89 ++- internal/db/migrations/0083_import.down.sql | 24 + internal/db/migrations/0083_import.up.sql | 37 + internal/db/queries/imports.sql | 130 ++++ internal/db/queries/search.sql | 3 +- internal/db/queries/wanted.sql | 19 +- internal/dbtest/reset.go | 2 +- internal/importer/importer.go | 608 +++++++++++++++ internal/importer/importer_test.go | 268 +++++++ internal/importer/inspect.go | 195 +++++ internal/importer/name.go | 110 +++ internal/importer/place.go | 182 +++++ internal/importer/verify.go | 466 ++++++++++++ internal/importer/verify_test.go | 243 ++++++ internal/library/fingerprint.go | 13 + internal/library/scanner.go | 13 + internal/wanted/search.go | 1 + web/src/lib/api/queries.ts | 3 + web/src/lib/api/types.ts | 91 ++- web/src/lib/api/wanted.ts | 52 ++ web/src/lib/components/AdminTabs.svelte | 1 - web/src/lib/components/AdminTabs.test.ts | 4 +- .../ReacquisitionSettingsCard.svelte | 2 +- .../lib/components/ReleaseProfilesCard.svelte | 3 +- .../lib/components/SearchSettingsCard.svelte | 43 +- .../lib/components/SearchSettingsCard.test.ts | 24 +- web/src/lib/components/Shell.svelte | 12 +- web/src/lib/components/Shell.test.ts | 15 + web/src/lib/components/WantedTabs.svelte | 54 ++ .../lib/components/wanted/BlockList.svelte | 88 +++ .../lib/components/wanted/BlockList.test.ts | 50 ++ .../lib/components/wanted/DownloadList.svelte | 192 +++++ .../components/wanted/DownloadList.test.ts | 109 +++ .../lib/components/wanted/FollowList.svelte | 117 +++ .../lib/components/wanted/FollowList.test.ts | 73 ++ web/src/lib/components/wanted/WantList.svelte | 263 +++++++ .../components/wanted/WantList.test.ts} | 146 ++-- web/src/lib/styles/error-copy.json | 3 + .../routes/admin/integrations/+page.svelte | 3 - .../admin/integrations/integrations.test.ts | 21 +- .../routes/admin/missing-files/+page.svelte | 2 +- .../admin/missing-files/missing-files.test.ts | 2 +- web/src/routes/admin/requests/+page.svelte | 2 +- web/src/routes/admin/wanted/+page.svelte | 363 --------- web/src/routes/wanted/+layout.svelte | 48 ++ web/src/routes/wanted/+layout.ts | 13 + web/src/routes/wanted/+page.svelte | 8 + web/src/routes/wanted/activity/+page.svelte | 8 + web/src/routes/wanted/blocked/+page.svelte | 8 + web/src/routes/wanted/following/+page.svelte | 12 + web/src/routes/wanted/held/+page.svelte | 8 + web/src/routes/wanted/missing/+page.svelte | 8 + web/src/routes/wanted/wanted.test.ts | 61 ++ 63 files changed, 4892 insertions(+), 555 deletions(-) create mode 100644 internal/api/admin_import.go create mode 100644 internal/db/dbq/imports.sql.go create mode 100644 internal/db/migrations/0083_import.down.sql create mode 100644 internal/db/migrations/0083_import.up.sql create mode 100644 internal/db/queries/imports.sql create mode 100644 internal/importer/importer.go create mode 100644 internal/importer/importer_test.go create mode 100644 internal/importer/inspect.go create mode 100644 internal/importer/name.go create mode 100644 internal/importer/place.go create mode 100644 internal/importer/verify.go create mode 100644 internal/importer/verify_test.go create mode 100644 web/src/lib/components/WantedTabs.svelte create mode 100644 web/src/lib/components/wanted/BlockList.svelte create mode 100644 web/src/lib/components/wanted/BlockList.test.ts create mode 100644 web/src/lib/components/wanted/DownloadList.svelte create mode 100644 web/src/lib/components/wanted/DownloadList.test.ts create mode 100644 web/src/lib/components/wanted/FollowList.svelte create mode 100644 web/src/lib/components/wanted/FollowList.test.ts create mode 100644 web/src/lib/components/wanted/WantList.svelte rename web/src/{routes/admin/wanted/wanted.test.ts => lib/components/wanted/WantList.test.ts} (58%) delete mode 100644 web/src/routes/admin/wanted/+page.svelte create mode 100644 web/src/routes/wanted/+layout.svelte create mode 100644 web/src/routes/wanted/+layout.ts create mode 100644 web/src/routes/wanted/+page.svelte create mode 100644 web/src/routes/wanted/activity/+page.svelte create mode 100644 web/src/routes/wanted/blocked/+page.svelte create mode 100644 web/src/routes/wanted/following/+page.svelte create mode 100644 web/src/routes/wanted/held/+page.svelte create mode 100644 web/src/routes/wanted/missing/+page.svelte create mode 100644 web/src/routes/wanted/wanted.test.ts diff --git a/README.md b/README.md index 3437ba73..5e00f972 100644 --- a/README.md +++ b/README.md @@ -40,11 +40,13 @@ services: ports: ['4533:4533'] volumes: # Your music library. Point ./music at wherever your audio files - # live. Writable, because Minstrel deletes a file when an admin asks - # it to (for example, quarantine's "Delete file"). It never moves, - # renames or retags anything. The container runs as uid 1000, so that - # user needs write access to the folders. Mount it :ro to forbid even - # deletes: those actions then refuse, say why, and delete nothing. + # live. Writable, because Minstrel files the albums it downloads into + # it (as Artist/Album (Year)/..., tagged with MusicBrainz ids) and + # deletes a file when an admin asks it to (for example, quarantine's + # "Delete file"). Files already there are never moved, renamed or + # retagged. The container runs as uid 1000, so that user needs write + # access to the folders. Mount it :ro to forbid writes: those actions + # then refuse, say why, and change nothing. - ./music:/music # Generated data: playlist cover collages, artist art, caches. # The path must match MINSTREL_STORAGE_DATA_DIR, which the image diff --git a/cmd/minstrel/main.go b/cmd/minstrel/main.go index ef6a0e9d..028f6a82 100644 --- a/cmd/minstrel/main.go +++ b/cmd/minstrel/main.go @@ -22,6 +22,7 @@ import ( "git.fabledsword.com/bvandeusen/minstrel/internal/db" "git.fabledsword.com/bvandeusen/minstrel/internal/eventbus" "git.fabledsword.com/bvandeusen/minstrel/internal/gc" + "git.fabledsword.com/bvandeusen/minstrel/internal/importer" "git.fabledsword.com/bvandeusen/minstrel/internal/library" "git.fabledsword.com/bvandeusen/minstrel/internal/logging" "git.fabledsword.com/bvandeusen/minstrel/internal/mailer" @@ -148,8 +149,11 @@ func run() error { if acoustIDErr != nil { logger.Warn("acoustid settings: using defaults (lookup off)", "err", acoustIDErr) } + // One AcoustID client, so the lookup worker and the import's check share + // its pace (AcoustID allows three requests a second per key). + acoustIDClient := acoustid.New(acoustid.DefaultBaseURL) acoustIDLookup := library.NewAcoustIDLookupWorker(pool, logger.With("component", "acoustid_lookup"), - acoustIDSettings, acoustid.New(acoustid.DefaultBaseURL)) + acoustIDSettings, acoustIDClient) contact := cfg.Library.ContactEmail if contact == "" { @@ -328,6 +332,15 @@ func run() error { // when auto-grab is on. go wanted.NewSearcher(pool, cat, logger.With("component", "search")).Run(ctx) + // Import with verification (M502 #5522): every minute, finished + // downloads are checked against their album (AcoustID, lengths, tags) + // and filed into the library with MusicBrainz tags, or refused. + var printer importer.Printer + if acoustIDSettings != nil { + printer = &importer.AcoustIDPrinter{Settings: acoustIDSettings, Client: acoustIDClient} + } + go importer.New(pool, scanner, cat, printer, cfg.Library.ScanPaths, logger.With("component", "import")).Run(ctx) + // Duplicate resolver (M498, M502): classifies the sweep's groups and, with // the operator's auto-resolve on, merges the copies that are safe to merge. // Started after SetNotifier so its summaries reach admins. See diff --git a/internal/acquire/memory.go b/internal/acquire/memory.go index a9b0354a..bf8ab8f7 100644 --- a/internal/acquire/memory.go +++ b/internal/acquire/memory.go @@ -19,6 +19,8 @@ import ( const ( BlockOperator = "operator" BlockFiles = "files" + // BlockImport: the import found its files were not the album (#5522). + BlockImport = "import" ) // Grab origins, matching grabs_origin_check. diff --git a/internal/api/admin_import.go b/internal/api/admin_import.go new file mode 100644 index 00000000..71675013 --- /dev/null +++ b/internal/api/admin_import.go @@ -0,0 +1,205 @@ +package api + +import ( + "encoding/json" + "errors" + "net/http" + + "github.com/jackc/pgx/v5" + + "git.fabledsword.com/bvandeusen/minstrel/internal/apierror" + "git.fabledsword.com/bvandeusen/minstrel/internal/db/dbq" + "git.fabledsword.com/bvandeusen/minstrel/internal/importer" +) + +// Downloads and their imports (M502 #5522): the Wanted view's Activity tab, +// a person's "import anyway", "refuse" and "retry", and the Blocked tab. + +// downloadView is one grab and what its import made of it. +type downloadView struct { + ID string `json:"id"` + WantID *string `json:"want_id"` + WantTitle string `json:"want_title"` + WantArtist string `json:"want_artist"` + Name string `json:"name"` + Indexer string `json:"indexer"` + Client string `json:"client"` + Quality string `json:"quality"` + Origin string `json:"origin"` + GrabbedAt string `json:"grabbed_at"` + EndedAt *string `json:"ended_at"` + EndReason string `json:"end_reason"` + Progress float32 `json:"progress"` + ImportState string `json:"import_state"` + ImportReason string `json:"import_reason"` + ImportedAt *string `json:"imported_at"` + ImportedFiles int32 `json:"imported_files"` + Destination string `json:"destination"` + Edition string `json:"edition,omitempty"` + Evidence *importer.Evidence `json:"evidence,omitempty"` + Files []string `json:"files"` + Skipped int `json:"skipped"` + RehomedFrom string `json:"rehomed_from,omitempty"` +} + +func downloadFrom(g dbq.ListGrabsRow) downloadView { + v := downloadView{ + ID: uuidToString(g.ID), Name: g.Name, Indexer: g.Indexer, Client: g.Client, Quality: g.Quality, + Origin: g.Origin, GrabbedAt: formatTimestamp(g.GrabbedAt), EndReason: g.EndReason, + Progress: g.Progress, ImportState: g.ImportState, ImportReason: g.ImportReason, + ImportedFiles: g.ImportedFiles, Destination: g.Destination, Files: []string{}, + } + if g.WantID.Valid { + id := uuidToString(g.WantID) + v.WantID = &id + } + if g.WantTitle != nil { + v.WantTitle = *g.WantTitle + } + if g.WantArtist != nil { + v.WantArtist = *g.WantArtist + } + if g.EndedAt.Valid { + at := formatTimestamp(g.EndedAt) + v.EndedAt = &at + } + if g.ImportedAt.Valid { + at := formatTimestamp(g.ImportedAt) + v.ImportedAt = &at + } + var det importer.Detail + if len(g.ImportDetail) > 0 && json.Unmarshal(g.ImportDetail, &det) == nil && det.Verdict.Outcome != "" { + ev := det.Verdict.Evidence + v.Evidence, v.Edition, v.Skipped, v.RehomedFrom = &ev, det.Edition, det.Skipped, det.RehomedFrom + if det.Files != nil { + v.Files = det.Files + } + } + return v +} + +type downloadsPage struct { + Items []downloadView `json:"items"` + Limit int `json:"limit"` + Offset int `json:"offset"` +} + +// handleListDownloads implements GET /api/admin/downloads?offset=: running +// downloads first, then the rest newest first. +func (h *handlers) handleListDownloads(w http.ResponseWriter, r *http.Request) { + limit, offset, err := parsePaging(r.URL.Query()) + if err != nil { + writeErr(w, apierror.BadRequest("invalid_paging", "limit and offset must be numbers")) + return + } + rows, err := dbq.New(h.pool).ListGrabs(r.Context(), dbq.ListGrabsParams{MaxRows: int32(limit), Skip: int32(offset)}) + if err != nil { + writeErrWithLog(w, h.logger, "admin downloads: list", apierror.Internal(err)) + return + } + out := downloadsPage{Items: make([]downloadView, 0, len(rows)), Limit: limit, Offset: offset} + for _, g := range rows { + out.Items = append(out.Items, downloadFrom(g)) + } + writeJSON(w, http.StatusOK, out) +} + +// handleImportAnyway implements POST /api/admin/downloads/{id}/import: a +// download the check held for a person (or one that failed) is imported on +// the next pass whatever the check says. +func (h *handlers) handleImportAnyway(w http.ResponseWriter, r *http.Request) { + id, ok := requireURLUUID(w, r, "id") + if !ok { + return + } + if _, err := dbq.New(h.pool).ForceGrabImport(r.Context(), id); err != nil { + h.writeImportErr(w, "import anyway", err) + return + } + w.WriteHeader(http.StatusNoContent) +} + +// handleRetryImport implements POST /api/admin/downloads/{id}/retry. +func (h *handlers) handleRetryImport(w http.ResponseWriter, r *http.Request) { + id, ok := requireURLUUID(w, r, "id") + if !ok { + return + } + if _, err := dbq.New(h.pool).RetryGrabImport(r.Context(), id); err != nil { + h.writeImportErr(w, "retry", err) + return + } + w.WriteHeader(http.StatusNoContent) +} + +// handleRefuseDownload implements POST /api/admin/downloads/{id}/refuse: the +// release is blocked, the download removed and the album searched again. +func (h *handlers) handleRefuseDownload(w http.ResponseWriter, r *http.Request) { + admin, ok := requireUser(w, r) + if !ok { + return + } + id, ok := requireURLUUID(w, r, "id") + if !ok { + return + } + if _, err := h.importer.Refuse(r.Context(), id, admin.ID); err != nil { + h.writeImportErr(w, "refuse", err) + return + } + w.WriteHeader(http.StatusNoContent) +} + +func (h *handlers) writeImportErr(w http.ResponseWriter, what string, err error) { + switch { + case errors.Is(err, pgx.ErrNoRows), errors.Is(err, importer.ErrNotDecidable): + writeErr(w, apierror.Conflict("not_decidable", "that download is not waiting on a decision")) + default: + writeErrWithLog(w, h.logger, "admin downloads: "+what, apierror.Internal(err)) + } +} + +type releaseBlockView struct { + InfoHash string `json:"info_hash"` + Name string `json:"name"` + Reason string `json:"reason"` + Detail string `json:"detail"` + BlockedAt string `json:"blocked_at"` + WantTitle string `json:"want_title"` + WantArtist string `json:"want_artist"` +} + +type releaseBlocksPage struct { + Items []releaseBlockView `json:"items"` + Limit int `json:"limit"` + Offset int `json:"offset"` +} + +// handleListReleaseBlocks implements GET /api/admin/release-blocks?offset=: +// every release set aside, by a person or by a check, newest first. +func (h *handlers) handleListReleaseBlocks(w http.ResponseWriter, r *http.Request) { + limit, offset, err := parsePaging(r.URL.Query()) + if err != nil { + writeErr(w, apierror.BadRequest("invalid_paging", "limit and offset must be numbers")) + return + } + rows, err := dbq.New(h.pool).ListReleaseBlocksPage(r.Context(), dbq.ListReleaseBlocksPageParams{ + MaxRows: int32(limit), Skip: int32(offset)}) + if err != nil { + writeErrWithLog(w, h.logger, "admin release blocks: list", apierror.Internal(err)) + return + } + out := releaseBlocksPage{Items: make([]releaseBlockView, 0, len(rows)), Limit: limit, Offset: offset} + for _, b := range rows { + v := releaseBlockView{InfoHash: b.InfoHash, Name: b.Name, Reason: b.Reason, Detail: b.Detail, + BlockedAt: formatTimestamp(b.BlockedAt)} + if b.WantTitle != nil { + v.WantTitle = *b.WantTitle + } + if b.WantArtist != nil { + v.WantArtist = *b.WantArtist + } + out.Items = append(out.Items, v) + } + writeJSON(w, http.StatusOK, out) +} diff --git a/internal/api/admin_search.go b/internal/api/admin_search.go index 53b7dbe0..84a17101 100644 --- a/internal/api/admin_search.go +++ b/internal/api/admin_search.go @@ -5,6 +5,7 @@ import ( "encoding/json" "errors" "net/http" + "slices" "time" "github.com/go-chi/chi/v5" @@ -32,12 +33,20 @@ type searchSettingsView struct { Tiers []release.TierInfo `json:"tiers"` LastPassAt *string `json:"last_pass_at"` LastPass json.RawMessage `json:"last_pass"` + // ImportRoot is the library folder new music is filed into (#5522); "" + // is the first of ScanPaths, the library's folders. + ImportRoot string `json:"import_root"` + ScanPaths []string `json:"scan_paths"` } -func searchSettings(s dbq.AcquisitionSetting) searchSettingsView { +func searchSettings(s dbq.AcquisitionSetting, roots []string) searchSettingsView { v := searchSettingsView{ AutoGrab: s.AutoGrab, SearchIntervalHours: s.SearchIntervalHours, AllowedQualities: s.AllowedQualities, Tiers: release.Tiers, LastPass: s.LastPass, + ImportRoot: s.ImportRoot, ScanPaths: roots, + } + if v.ScanPaths == nil { + v.ScanPaths = []string{} } if v.AllowedQualities == nil { v.AllowedQualities = []string{} @@ -59,13 +68,14 @@ func (h *handlers) handleGetSearchSettings(w http.ResponseWriter, r *http.Reques writeErrWithLog(w, h.logger, "admin search settings: get", apierror.Internal(err)) return } - writeJSON(w, http.StatusOK, searchSettings(s)) + writeJSON(w, http.StatusOK, searchSettings(s, h.importer.Roots)) } type putSearchSettingsBody struct { AutoGrab bool `json:"auto_grab"` SearchIntervalHours int32 `json:"search_interval_hours"` AllowedQualities []string `json:"allowed_qualities"` + ImportRoot string `json:"import_root"` } // handlePutSearchSettings implements PUT /api/admin/acquisition/settings. @@ -88,14 +98,19 @@ func (h *handlers) handlePutSearchSettings(w http.ResponseWriter, r *http.Reques return } } + if body.ImportRoot != "" && !slices.Contains(h.importer.Roots, body.ImportRoot) { + writeErr(w, apierror.BadRequest("bad_import_root", "file new music into one of the library's folders")) + return + } s, err := dbq.New(h.pool).UpdateAcquisitionSettings(r.Context(), dbq.UpdateAcquisitionSettingsParams{ AutoGrab: body.AutoGrab, SearchIntervalHours: body.SearchIntervalHours, AllowedQualities: body.AllowedQualities, + ImportRoot: body.ImportRoot, }) if err != nil { writeErrWithLog(w, h.logger, "admin search settings: put", apierror.Internal(err)) return } - writeJSON(w, http.StatusOK, searchSettings(s)) + writeJSON(w, http.StatusOK, searchSettings(s, h.importer.Roots)) } // candidateView is one release a want's last search found. diff --git a/internal/api/admin_wanted.go b/internal/api/admin_wanted.go index 6c740ab6..d5cc061f 100644 --- a/internal/api/admin_wanted.go +++ b/internal/api/admin_wanted.go @@ -49,11 +49,21 @@ type wantSearchView struct { Error string `json:"error"` } +// wantGrabView is the want's latest download, running or ended, and what +// its import made of it (#5522). type wantGrabView struct { - Name string `json:"name"` - GrabbedAt string `json:"grabbed_at"` - Origin string `json:"origin"` - Quality string `json:"quality"` + ID string `json:"id"` + Name string `json:"name"` + GrabbedAt string `json:"grabbed_at"` + Origin string `json:"origin"` + Quality string `json:"quality"` + Live bool `json:"live"` + EndReason string `json:"end_reason"` + Progress float32 `json:"progress"` + ImportState string `json:"import_state"` + ImportReason string `json:"import_reason"` + ImportedAt *string `json:"imported_at"` + ImportedFiles int32 `json:"imported_files"` } // wantCounts sizes each tab of the Wanted page. @@ -62,6 +72,11 @@ type wantCounts struct { Missing int64 `json:"missing"` Held int64 `json:"held"` Following int64 `json:"following"` + // Activity: downloads running; Attention: of those, the ones waiting on + // a person; Blocked: releases set aside (#5522). + Activity int64 `json:"activity"` + Attention int64 `json:"attention"` + Blocked int64 `json:"blocked"` } type wantsPage struct { @@ -102,7 +117,8 @@ func (h *handlers) handleListWants(w http.ResponseWriter, r *http.Request) { } out := wantsPage{ Items: make([]wantView, 0, len(rows)), Limit: wantsPageSize, Offset: offset, - Counts: wantCounts{Wanted: counts.Wanted, Missing: counts.Missing, Held: counts.Held, Following: counts.Following}, + Counts: wantCounts{Wanted: counts.Wanted, Missing: counts.Missing, Held: counts.Held, Following: counts.Following, + Activity: counts.Activity, Attention: counts.Attention, Blocked: counts.Blocked}, } switch state { case "missing": @@ -145,12 +161,19 @@ func (h *handlers) handleListWants(w http.ResponseWriter, r *http.Request) { } } if row.GrabName != nil { - v.Grab = &wantGrabView{Name: *row.GrabName, GrabbedAt: formatTimestamp(row.GrabbedAt)} - if row.GrabOrigin != nil { - v.Grab.Origin = *row.GrabOrigin + v.Grab = &wantGrabView{ID: uuidToString(row.GrabID), Name: *row.GrabName, + GrabbedAt: formatTimestamp(row.GrabbedAt), Live: !row.GrabEndedAt.Valid, + Origin: deref(row.GrabOrigin), Quality: deref(row.GrabQuality), EndReason: deref(row.GrabEndReason), + ImportState: deref(row.GrabImportState), ImportReason: deref(row.GrabImportReason)} + if row.GrabProgress != nil { + v.Grab.Progress = *row.GrabProgress } - if row.GrabQuality != nil { - v.Grab.Quality = *row.GrabQuality + if row.GrabImportedFiles != nil { + v.Grab.ImportedFiles = *row.GrabImportedFiles + } + if row.GrabImportedAt.Valid { + at := formatTimestamp(row.GrabImportedAt) + v.Grab.ImportedAt = &at } } out.Items = append(out.Items, v) @@ -425,3 +448,11 @@ func (h *handlers) writeProfileErr(w http.ResponseWriter, err error) { writeErrWithLog(w, h.logger, "admin release profiles", apierror.Internal(err)) } } + +// deref is a nullable column's text, or "". +func deref(s *string) string { + if s == nil { + return "" + } + return *s +} diff --git a/internal/api/api.go b/internal/api/api.go index 08d2d231..fc4d6133 100644 --- a/internal/api/api.go +++ b/internal/api/api.go @@ -19,6 +19,7 @@ import ( "git.fabledsword.com/bvandeusen/minstrel/internal/config" "git.fabledsword.com/bvandeusen/minstrel/internal/coverart" "git.fabledsword.com/bvandeusen/minstrel/internal/eventbus" + "git.fabledsword.com/bvandeusen/minstrel/internal/importer" "git.fabledsword.com/bvandeusen/minstrel/internal/library" "git.fabledsword.com/bvandeusen/minstrel/internal/lidarrconfig" "git.fabledsword.com/bvandeusen/minstrel/internal/lidarrquarantine" @@ -78,6 +79,7 @@ func Mount(r chi.Router, pool *pgxpool.Pool, logger *slog.Logger, events *playev catalogue: cat, wanted: wanted.New(pool, cat, logger), searcher: wanted.NewSearcher(pool, cat, logger), + importer: importer.New(pool, scanner, cat, nil, scannerRoots(scanner), logger), catalogueSearches: newSearchCache(), catalogueArt: catalogue.NewArt(dataDir, artSources(coverSettings)), leveled: newLeveledRenderer(dataDir, loudSettings, logger), @@ -257,6 +259,11 @@ func Mount(r chi.Router, pool *pgxpool.Pool, logger *slog.Logger, events *playev admin.Post("/wanted/{id}/grab", h.handleGrabCandidate) admin.Post("/wanted/{id}/block", h.handleBlockCandidate) admin.Delete("/release-blocks/{hash}", h.handleUnblockRelease) + admin.Get("/release-blocks", h.handleListReleaseBlocks) + admin.Get("/downloads", h.handleListDownloads) + admin.Post("/downloads/{id}/import", h.handleImportAnyway) + admin.Post("/downloads/{id}/retry", h.handleRetryImport) + admin.Post("/downloads/{id}/refuse", h.handleRefuseDownload) admin.Get("/follows", h.handleListFollows) admin.Get("/follows/{mbid}", h.handleGetFollow) admin.Put("/follows/{mbid}", h.handlePutFollow) @@ -427,6 +434,9 @@ type handlers struct { wanted *wanted.Service // searcher searches for wants and grabs releases for them (#5521). searcher *wanted.Searcher + // importer refuses a download at a person's word (#5522). The import + // loop itself runs in main; this one only acts on a request. + importer *importer.Importer // catalogueSearches reuses recent MusicBrainz answers; nil asks every // time, which is what handler tests get. catalogueSearches *searchCache @@ -482,3 +492,12 @@ func artSources(s *coverart.SettingsService) catalogue.ArtSources { } return s } + +// scannerRoots are the library folders, or none when the API runs without a +// scanner (tests). +func scannerRoots(s *library.Scanner) []string { + if s == nil { + return nil + } + return s.Roots() +} diff --git a/internal/db/dbq/imports.sql.go b/internal/db/dbq/imports.sql.go new file mode 100644 index 00000000..df96ed4d --- /dev/null +++ b/internal/db/dbq/imports.sql.go @@ -0,0 +1,719 @@ +// Code generated by sqlc. DO NOT EDIT. +// versions: +// sqlc v1.31.1 +// source: imports.sql + +package dbq + +import ( + "context" + + "github.com/jackc/pgx/v5/pgtype" +) + +const claimGrabImport = `-- name: ClaimGrabImport :one +UPDATE grabs + SET import_state = 'importing', import_attempts = import_attempts + 1, + import_updated_at = now(), progress = 1 + WHERE id = $1 AND ended_at IS NULL AND import_state IN ('', 'downloading', 'waiting') +RETURNING id, want_id, info_hash, name, key, indexer, source, client, save_path, quality, origin, forced, grabbed_by, grabbed_at, ended_at, end_reason, end_detail, progress, import_state, import_reason, import_detail, import_forced, import_attempts, import_updated_at, imported_at, imported_files, destination +` + +// Takes a finished download for import. The update is the lock: a second +// pass finds nothing to claim. +func (q *Queries) ClaimGrabImport(ctx context.Context, id pgtype.UUID) (Grab, error) { + row := q.db.QueryRow(ctx, claimGrabImport, id) + var i Grab + err := row.Scan( + &i.ID, + &i.WantID, + &i.InfoHash, + &i.Name, + &i.Key, + &i.Indexer, + &i.Source, + &i.Client, + &i.SavePath, + &i.Quality, + &i.Origin, + &i.Forced, + &i.GrabbedBy, + &i.GrabbedAt, + &i.EndedAt, + &i.EndReason, + &i.EndDetail, + &i.Progress, + &i.ImportState, + &i.ImportReason, + &i.ImportDetail, + &i.ImportForced, + &i.ImportAttempts, + &i.ImportUpdatedAt, + &i.ImportedAt, + &i.ImportedFiles, + &i.Destination, + ) + return i, err +} + +const forceGrabImport = `-- name: ForceGrabImport :one +UPDATE grabs + SET import_state = '', import_forced = true, import_reason = '', import_updated_at = now() + WHERE id = $1 AND ended_at IS NULL AND import_state IN ('review', 'failed') +RETURNING id, want_id, info_hash, name, key, indexer, source, client, save_path, quality, origin, forced, grabbed_by, grabbed_at, ended_at, end_reason, end_detail, progress, import_state, import_reason, import_detail, import_forced, import_attempts, import_updated_at, imported_at, imported_files, destination +` + +// "Import anyway": the next pass imports the download whatever the check said. +func (q *Queries) ForceGrabImport(ctx context.Context, id pgtype.UUID) (Grab, error) { + row := q.db.QueryRow(ctx, forceGrabImport, id) + var i Grab + err := row.Scan( + &i.ID, + &i.WantID, + &i.InfoHash, + &i.Name, + &i.Key, + &i.Indexer, + &i.Source, + &i.Client, + &i.SavePath, + &i.Quality, + &i.Origin, + &i.Forced, + &i.GrabbedBy, + &i.GrabbedAt, + &i.EndedAt, + &i.EndReason, + &i.EndDetail, + &i.Progress, + &i.ImportState, + &i.ImportReason, + &i.ImportDetail, + &i.ImportForced, + &i.ImportAttempts, + &i.ImportUpdatedAt, + &i.ImportedAt, + &i.ImportedFiles, + &i.Destination, + ) + return i, err +} + +const getGrab = `-- name: GetGrab :one +SELECT id, want_id, info_hash, name, key, indexer, source, client, save_path, quality, origin, forced, grabbed_by, grabbed_at, ended_at, end_reason, end_detail, progress, import_state, import_reason, import_detail, import_forced, import_attempts, import_updated_at, imported_at, imported_files, destination FROM grabs WHERE id = $1 +` + +func (q *Queries) GetGrab(ctx context.Context, id pgtype.UUID) (Grab, error) { + row := q.db.QueryRow(ctx, getGrab, id) + var i Grab + err := row.Scan( + &i.ID, + &i.WantID, + &i.InfoHash, + &i.Name, + &i.Key, + &i.Indexer, + &i.Source, + &i.Client, + &i.SavePath, + &i.Quality, + &i.Origin, + &i.Forced, + &i.GrabbedBy, + &i.GrabbedAt, + &i.EndedAt, + &i.EndReason, + &i.EndDetail, + &i.Progress, + &i.ImportState, + &i.ImportReason, + &i.ImportDetail, + &i.ImportForced, + &i.ImportAttempts, + &i.ImportUpdatedAt, + &i.ImportedAt, + &i.ImportedFiles, + &i.Destination, + ) + return i, err +} + +const heldPositions = `-- name: HeldPositions :many +SELECT DISTINCT COALESCE(t.disc_number, 1)::int AS disc, t.track_number::int AS track + FROM tracks t + JOIN albums a ON a.id = t.album_id + WHERE a.release_group_mbid = $1 + AND t.missing_since IS NULL AND t.track_number IS NOT NULL +` + +type HeldPositionsRow struct { + Disc int32 + Track int32 +} + +// The disc and track positions the library holds for an album of this group. +func (q *Queries) HeldPositions(ctx context.Context, releaseGroupMbid *string) ([]HeldPositionsRow, error) { + rows, err := q.db.Query(ctx, heldPositions, releaseGroupMbid) + if err != nil { + return nil, err + } + defer rows.Close() + var items []HeldPositionsRow + for rows.Next() { + var i HeldPositionsRow + if err := rows.Scan(&i.Disc, &i.Track); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const heldRecordings = `-- name: HeldRecordings :many +SELECT DISTINCT mbid::text AS mbid FROM tracks + WHERE mbid = ANY($1::text[]) AND missing_since IS NULL +` + +// Which of these recordings the library holds a file for. +func (q *Queries) HeldRecordings(ctx context.Context, recordings []string) ([]string, error) { + rows, err := q.db.Query(ctx, heldRecordings, recordings) + if err != nil { + return nil, err + } + defer rows.Close() + var items []string + for rows.Next() { + var mbid string + if err := rows.Scan(&mbid); err != nil { + return nil, err + } + items = append(items, mbid) + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const listGrabs = `-- name: ListGrabs :many +SELECT g.id, g.want_id, g.name, g.indexer, g.client, g.quality, g.origin, g.forced, + g.grabbed_at, g.ended_at, g.end_reason, g.end_detail, g.progress, g.import_state, + g.import_reason, g.import_detail, g.import_updated_at, g.imported_at, g.imported_files, + g.destination, w.title AS want_title, w.artist_name AS want_artist + FROM grabs g + LEFT JOIN wants w ON w.id = g.want_id + ORDER BY (g.ended_at IS NULL AND g.import_state IN ('review', 'failed')) DESC, + g.ended_at IS NULL DESC, g.grabbed_at DESC, g.id + LIMIT $2 OFFSET $1 +` + +type ListGrabsParams struct { + Skip int32 + MaxRows int32 +} + +type ListGrabsRow struct { + ID pgtype.UUID + WantID pgtype.UUID + Name string + Indexer string + Client string + Quality string + Origin string + Forced bool + GrabbedAt pgtype.Timestamptz + EndedAt pgtype.Timestamptz + EndReason string + EndDetail string + Progress float32 + ImportState string + ImportReason string + ImportDetail []byte + ImportUpdatedAt pgtype.Timestamptz + ImportedAt pgtype.Timestamptz + ImportedFiles int32 + Destination string + WantTitle *string + WantArtist *string +} + +// The Wanted view's Activity tab: downloads waiting on a person first, then +// the rest still running, then the ended ones newest first. +func (q *Queries) ListGrabs(ctx context.Context, arg ListGrabsParams) ([]ListGrabsRow, error) { + rows, err := q.db.Query(ctx, listGrabs, arg.Skip, arg.MaxRows) + if err != nil { + return nil, err + } + defer rows.Close() + var items []ListGrabsRow + for rows.Next() { + var i ListGrabsRow + if err := rows.Scan( + &i.ID, + &i.WantID, + &i.Name, + &i.Indexer, + &i.Client, + &i.Quality, + &i.Origin, + &i.Forced, + &i.GrabbedAt, + &i.EndedAt, + &i.EndReason, + &i.EndDetail, + &i.Progress, + &i.ImportState, + &i.ImportReason, + &i.ImportDetail, + &i.ImportUpdatedAt, + &i.ImportedAt, + &i.ImportedFiles, + &i.Destination, + &i.WantTitle, + &i.WantArtist, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const listLiveGrabs = `-- name: ListLiveGrabs :many + +SELECT g.id, g.want_id, g.info_hash, g.name, g.key, g.indexer, g.source, g.client, g.save_path, g.quality, g.origin, g.forced, g.grabbed_by, g.grabbed_at, g.ended_at, g.end_reason, g.end_detail, g.progress, g.import_state, g.import_reason, g.import_detail, g.import_forced, g.import_attempts, g.import_updated_at, g.imported_at, g.imported_files, g.destination, w.release_group_mbid, w.artist_mbid, w.artist_name, w.title AS want_title, + w.reason AS want_reason, w.album_id AS want_album_id + FROM grabs g + LEFT JOIN wants w ON w.id = g.want_id + WHERE g.ended_at IS NULL + ORDER BY g.grabbed_at, g.id +` + +type ListLiveGrabsRow struct { + ID pgtype.UUID + WantID pgtype.UUID + InfoHash string + Name string + Key string + Indexer string + Source string + Client string + SavePath string + Quality string + Origin string + Forced bool + GrabbedBy pgtype.UUID + GrabbedAt pgtype.Timestamptz + EndedAt pgtype.Timestamptz + EndReason string + EndDetail string + Progress float32 + ImportState string + ImportReason string + ImportDetail []byte + ImportForced bool + ImportAttempts int32 + ImportUpdatedAt pgtype.Timestamptz + ImportedAt pgtype.Timestamptz + ImportedFiles int32 + Destination string + ReleaseGroupMbid *string + ArtistMbid *string + ArtistName *string + WantTitle *string + WantReason *string + WantAlbumID pgtype.UUID +} + +// Import with verification (M502 #5522). +// Every download still running or waiting on its import, oldest first. +func (q *Queries) ListLiveGrabs(ctx context.Context) ([]ListLiveGrabsRow, error) { + rows, err := q.db.Query(ctx, listLiveGrabs) + if err != nil { + return nil, err + } + defer rows.Close() + var items []ListLiveGrabsRow + for rows.Next() { + var i ListLiveGrabsRow + if err := rows.Scan( + &i.ID, + &i.WantID, + &i.InfoHash, + &i.Name, + &i.Key, + &i.Indexer, + &i.Source, + &i.Client, + &i.SavePath, + &i.Quality, + &i.Origin, + &i.Forced, + &i.GrabbedBy, + &i.GrabbedAt, + &i.EndedAt, + &i.EndReason, + &i.EndDetail, + &i.Progress, + &i.ImportState, + &i.ImportReason, + &i.ImportDetail, + &i.ImportForced, + &i.ImportAttempts, + &i.ImportUpdatedAt, + &i.ImportedAt, + &i.ImportedFiles, + &i.Destination, + &i.ReleaseGroupMbid, + &i.ArtistMbid, + &i.ArtistName, + &i.WantTitle, + &i.WantReason, + &i.WantAlbumID, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const listReleaseBlocksPage = `-- name: ListReleaseBlocksPage :many +SELECT b.info_hash, b.name, b.reason, b.detail, b.blocked_at, w.title AS want_title, + w.artist_name AS want_artist + FROM release_blocks b + LEFT JOIN wants w ON w.id = b.want_id + ORDER BY b.blocked_at DESC, b.info_hash + LIMIT $2 OFFSET $1 +` + +type ListReleaseBlocksPageParams struct { + Skip int32 + MaxRows int32 +} + +type ListReleaseBlocksPageRow struct { + InfoHash string + Name string + Reason string + Detail string + BlockedAt pgtype.Timestamptz + WantTitle *string + WantArtist *string +} + +func (q *Queries) ListReleaseBlocksPage(ctx context.Context, arg ListReleaseBlocksPageParams) ([]ListReleaseBlocksPageRow, error) { + rows, err := q.db.Query(ctx, listReleaseBlocksPage, arg.Skip, arg.MaxRows) + if err != nil { + return nil, err + } + defer rows.Close() + var items []ListReleaseBlocksPageRow + for rows.Next() { + var i ListReleaseBlocksPageRow + if err := rows.Scan( + &i.InfoHash, + &i.Name, + &i.Reason, + &i.Detail, + &i.BlockedAt, + &i.WantTitle, + &i.WantArtist, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const markWantSearched = `-- name: MarkWantSearched :exec +UPDATE wants SET searched_at = now() WHERE id = $1 +` + +// Imported: nothing to search for until the wanted pass settles the want. +func (q *Queries) MarkWantSearched(ctx context.Context, id pgtype.UUID) error { + _, err := q.db.Exec(ctx, markWantSearched, id) + return err +} + +const openWantsForGroups = `-- name: OpenWantsForGroups :many +SELECT w.id, w.release_group_mbid, w.artist_mbid, w.artist_name, w.title, w.primary_type, w.first_release_date, w.reason, w.album_id, w.created_by, w.wanted_at, w.fulfilled_at, w.created_at, w.updated_at, w.searched_at, w.search_outcome, w.search_considered, w.best_release, w.best_verdict, w.search_error FROM wants w + WHERE w.fulfilled_at IS NULL AND w.release_group_mbid = ANY($1::text[]) + AND NOT EXISTS (SELECT 1 FROM grabs g WHERE g.want_id = w.id AND g.ended_at IS NULL) +` + +// Open wants for these release groups with no download running. +func (q *Queries) OpenWantsForGroups(ctx context.Context, groups []string) ([]Want, error) { + rows, err := q.db.Query(ctx, openWantsForGroups, groups) + if err != nil { + return nil, err + } + defer rows.Close() + var items []Want + for rows.Next() { + var i Want + if err := rows.Scan( + &i.ID, + &i.ReleaseGroupMbid, + &i.ArtistMbid, + &i.ArtistName, + &i.Title, + &i.PrimaryType, + &i.FirstReleaseDate, + &i.Reason, + &i.AlbumID, + &i.CreatedBy, + &i.WantedAt, + &i.FulfilledAt, + &i.CreatedAt, + &i.UpdatedAt, + &i.SearchedAt, + &i.SearchOutcome, + &i.SearchConsidered, + &i.BestRelease, + &i.BestVerdict, + &i.SearchError, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const reclaimStaleImports = `-- name: ReclaimStaleImports :execrows +UPDATE grabs + SET import_state = 'waiting', + import_reason = 'The last import was cut short, so it is being tried again.' + WHERE ended_at IS NULL AND import_state = 'importing' + AND import_updated_at < now() - make_interval(mins => $1::int) +` + +// An import cut short (a restart mid-copy) is tried again. +func (q *Queries) ReclaimStaleImports(ctx context.Context, minutes int32) (int64, error) { + result, err := q.db.Exec(ctx, reclaimStaleImports, minutes) + if err != nil { + return 0, err + } + return result.RowsAffected(), nil +} + +const recordImport = `-- name: RecordImport :one +UPDATE grabs + SET import_state = $1, + import_reason = $2, + import_detail = $3, + import_updated_at = now(), + import_forced = false, + imported_at = CASE WHEN $1::text = 'imported' THEN now() ELSE imported_at END, + imported_files = $4, + destination = $5, + ended_at = CASE WHEN $6::text <> '' THEN now() ELSE ended_at END, + end_reason = $6, + end_detail = CASE WHEN $6::text <> '' THEN $2::text ELSE end_detail END + WHERE id = $7 +RETURNING id, want_id, info_hash, name, key, indexer, source, client, save_path, quality, origin, forced, grabbed_by, grabbed_at, ended_at, end_reason, end_detail, progress, import_state, import_reason, import_detail, import_forced, import_attempts, import_updated_at, imported_at, imported_files, destination +` + +type RecordImportParams struct { + ImportState string + ImportReason string + ImportDetail []byte + ImportedFiles int32 + Destination string + EndReason string + ID pgtype.UUID +} + +// The import's outcome. An end reason ends the grab: imported, refused or +// held. Without one the grab stays live: review and failed wait for a person. +func (q *Queries) RecordImport(ctx context.Context, arg RecordImportParams) (Grab, error) { + row := q.db.QueryRow(ctx, recordImport, + arg.ImportState, + arg.ImportReason, + arg.ImportDetail, + arg.ImportedFiles, + arg.Destination, + arg.EndReason, + arg.ID, + ) + var i Grab + err := row.Scan( + &i.ID, + &i.WantID, + &i.InfoHash, + &i.Name, + &i.Key, + &i.Indexer, + &i.Source, + &i.Client, + &i.SavePath, + &i.Quality, + &i.Origin, + &i.Forced, + &i.GrabbedBy, + &i.GrabbedAt, + &i.EndedAt, + &i.EndReason, + &i.EndDetail, + &i.Progress, + &i.ImportState, + &i.ImportReason, + &i.ImportDetail, + &i.ImportForced, + &i.ImportAttempts, + &i.ImportUpdatedAt, + &i.ImportedAt, + &i.ImportedFiles, + &i.Destination, + ) + return i, err +} + +const rehomeGrab = `-- name: RehomeGrab :one +UPDATE grabs g + SET want_id = $1 + WHERE g.id = $2 AND g.ended_at IS NULL + AND NOT EXISTS (SELECT 1 FROM grabs o WHERE o.want_id = $1 AND o.ended_at IS NULL) +RETURNING id, want_id, info_hash, name, key, indexer, source, client, save_path, quality, origin, forced, grabbed_by, grabbed_at, ended_at, end_reason, end_detail, progress, import_state, import_reason, import_detail, import_forced, import_attempts, import_updated_at, imported_at, imported_files, destination +` + +type RehomeGrabParams struct { + WantID pgtype.UUID + ID pgtype.UUID +} + +// A download that turned out to be another wanted album is that want's grab, +// when the want has none running. +func (q *Queries) RehomeGrab(ctx context.Context, arg RehomeGrabParams) (Grab, error) { + row := q.db.QueryRow(ctx, rehomeGrab, arg.WantID, arg.ID) + var i Grab + err := row.Scan( + &i.ID, + &i.WantID, + &i.InfoHash, + &i.Name, + &i.Key, + &i.Indexer, + &i.Source, + &i.Client, + &i.SavePath, + &i.Quality, + &i.Origin, + &i.Forced, + &i.GrabbedBy, + &i.GrabbedAt, + &i.EndedAt, + &i.EndReason, + &i.EndDetail, + &i.Progress, + &i.ImportState, + &i.ImportReason, + &i.ImportDetail, + &i.ImportForced, + &i.ImportAttempts, + &i.ImportUpdatedAt, + &i.ImportedAt, + &i.ImportedFiles, + &i.Destination, + ) + return i, err +} + +const retryGrabImport = `-- name: RetryGrabImport :one +UPDATE grabs + SET import_state = '', import_reason = '', import_updated_at = now() + WHERE id = $1 AND ended_at IS NULL AND import_state = 'failed' +RETURNING id, want_id, info_hash, name, key, indexer, source, client, save_path, quality, origin, forced, grabbed_by, grabbed_at, ended_at, end_reason, end_detail, progress, import_state, import_reason, import_detail, import_forced, import_attempts, import_updated_at, imported_at, imported_files, destination +` + +func (q *Queries) RetryGrabImport(ctx context.Context, id pgtype.UUID) (Grab, error) { + row := q.db.QueryRow(ctx, retryGrabImport, id) + var i Grab + err := row.Scan( + &i.ID, + &i.WantID, + &i.InfoHash, + &i.Name, + &i.Key, + &i.Indexer, + &i.Source, + &i.Client, + &i.SavePath, + &i.Quality, + &i.Origin, + &i.Forced, + &i.GrabbedBy, + &i.GrabbedAt, + &i.EndedAt, + &i.EndReason, + &i.EndDetail, + &i.Progress, + &i.ImportState, + &i.ImportReason, + &i.ImportDetail, + &i.ImportForced, + &i.ImportAttempts, + &i.ImportUpdatedAt, + &i.ImportedAt, + &i.ImportedFiles, + &i.Destination, + ) + return i, err +} + +const searchWantAgain = `-- name: SearchWantAgain :exec +UPDATE wants SET searched_at = NULL, search_outcome = '' WHERE id = $1 +` + +// Refused: search again on the next pass. +func (q *Queries) SearchWantAgain(ctx context.Context, id pgtype.UUID) error { + _, err := q.db.Exec(ctx, searchWantAgain, id) + return err +} + +const setGrabProgress = `-- name: SetGrabProgress :exec +UPDATE grabs + SET progress = $1, + import_state = $2, + import_reason = $3, + import_updated_at = now() + WHERE id = $4 AND ended_at IS NULL + AND import_state IN ('', 'downloading', 'waiting') +` + +type SetGrabProgressParams struct { + Progress float32 + ImportState string + ImportReason string + ID pgtype.UUID +} + +// What the download client says of a download not yet finished, or finished +// and waiting for what the import needs. Leaves a claimed, decided or failed +// import alone. +func (q *Queries) SetGrabProgress(ctx context.Context, arg SetGrabProgressParams) error { + _, err := q.db.Exec(ctx, setGrabProgress, + arg.Progress, + arg.ImportState, + arg.ImportReason, + arg.ID, + ) + return err +} diff --git a/internal/db/dbq/models.go b/internal/db/dbq/models.go index 0d4214dd..c42a4152 100644 --- a/internal/db/dbq/models.go +++ b/internal/db/dbq/models.go @@ -225,6 +225,7 @@ type AcquisitionSetting struct { LastPassAt pgtype.Timestamptz LastPass []byte UpdatedAt pgtype.Timestamptz + ImportRoot string } type Album struct { @@ -431,23 +432,33 @@ type GeneralLikesArtist struct { } type Grab struct { - ID pgtype.UUID - WantID pgtype.UUID - InfoHash string - Name string - Key string - Indexer string - Source string - Client string - SavePath string - Quality string - Origin string - Forced bool - GrabbedBy pgtype.UUID - GrabbedAt pgtype.Timestamptz - EndedAt pgtype.Timestamptz - EndReason string - EndDetail string + ID pgtype.UUID + WantID pgtype.UUID + InfoHash string + Name string + Key string + Indexer string + Source string + Client string + SavePath string + Quality string + Origin string + Forced bool + GrabbedBy pgtype.UUID + GrabbedAt pgtype.Timestamptz + EndedAt pgtype.Timestamptz + EndReason string + EndDetail string + Progress float32 + ImportState string + ImportReason string + ImportDetail []byte + ImportForced bool + ImportAttempts int32 + ImportUpdatedAt pgtype.Timestamptz + ImportedAt pgtype.Timestamptz + ImportedFiles int32 + Destination string } type LibraryChange struct { diff --git a/internal/db/dbq/search.sql.go b/internal/db/dbq/search.sql.go index b1043de2..9ecde58f 100644 --- a/internal/db/dbq/search.sql.go +++ b/internal/db/dbq/search.sql.go @@ -89,7 +89,7 @@ func (q *Queries) EndGrab(ctx context.Context, arg EndGrabParams) error { const getAcquisitionSettings = `-- name: GetAcquisitionSettings :one -SELECT id, auto_grab, search_interval_hours, allowed_qualities, last_pass_at, last_pass, updated_at FROM acquisition_settings WHERE id = true +SELECT id, auto_grab, search_interval_hours, allowed_qualities, last_pass_at, last_pass, updated_at, import_root FROM acquisition_settings WHERE id = true ` // Searching for wants and grabbing releases (M502 #5521). @@ -104,12 +104,13 @@ func (q *Queries) GetAcquisitionSettings(ctx context.Context) (AcquisitionSettin &i.LastPassAt, &i.LastPass, &i.UpdatedAt, + &i.ImportRoot, ) return i, err } const getLiveGrab = `-- name: GetLiveGrab :one -SELECT id, want_id, info_hash, name, key, indexer, source, client, save_path, quality, origin, forced, grabbed_by, grabbed_at, ended_at, end_reason, end_detail FROM grabs WHERE want_id = $1 AND ended_at IS NULL +SELECT id, want_id, info_hash, name, key, indexer, source, client, save_path, quality, origin, forced, grabbed_by, grabbed_at, ended_at, end_reason, end_detail, progress, import_state, import_reason, import_detail, import_forced, import_attempts, import_updated_at, imported_at, imported_files, destination FROM grabs WHERE want_id = $1 AND ended_at IS NULL ` func (q *Queries) GetLiveGrab(ctx context.Context, wantID pgtype.UUID) (Grab, error) { @@ -133,6 +134,16 @@ func (q *Queries) GetLiveGrab(ctx context.Context, wantID pgtype.UUID) (Grab, er &i.EndedAt, &i.EndReason, &i.EndDetail, + &i.Progress, + &i.ImportState, + &i.ImportReason, + &i.ImportDetail, + &i.ImportForced, + &i.ImportAttempts, + &i.ImportUpdatedAt, + &i.ImportedAt, + &i.ImportedFiles, + &i.Destination, ) return i, err } @@ -173,7 +184,7 @@ func (q *Queries) GetWantCandidate(ctx context.Context, arg GetWantCandidatePara } const getWantForSearch = `-- name: GetWantForSearch :one -SELECT w.id, w.release_group_mbid, w.artist_name, w.title, w.reason, w.fulfilled_at, +SELECT w.id, w.release_group_mbid, w.artist_mbid, w.artist_name, w.title, w.reason, w.fulfilled_at, COALESCE(NULLIF(w.primary_type, ''), g.primary_type, '')::text AS primary_type, COALESCE(g.secondary_types, '{}')::text[] AS secondary_types, COALESCE(NULLIF(w.first_release_date, ''), g.first_release_date, '')::text AS first_release_date @@ -185,6 +196,7 @@ SELECT w.id, w.release_group_mbid, w.artist_name, w.title, w.reason, w.fulfilled type GetWantForSearchRow struct { ID pgtype.UUID ReleaseGroupMbid string + ArtistMbid string ArtistName string Title string Reason string @@ -200,6 +212,7 @@ func (q *Queries) GetWantForSearch(ctx context.Context, id pgtype.UUID) (GetWant err := row.Scan( &i.ID, &i.ReleaseGroupMbid, + &i.ArtistMbid, &i.ArtistName, &i.Title, &i.Reason, @@ -217,7 +230,7 @@ INSERT INTO grabs (want_id, info_hash, name, key, indexer, source, client, save_ VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12) -RETURNING id, want_id, info_hash, name, key, indexer, source, client, save_path, quality, origin, forced, grabbed_by, grabbed_at, ended_at, end_reason, end_detail +RETURNING id, want_id, info_hash, name, key, indexer, source, client, save_path, quality, origin, forced, grabbed_by, grabbed_at, ended_at, end_reason, end_detail, progress, import_state, import_reason, import_detail, import_forced, import_attempts, import_updated_at, imported_at, imported_files, destination ` type InsertGrabParams struct { @@ -269,6 +282,16 @@ func (q *Queries) InsertGrab(ctx context.Context, arg InsertGrabParams) (Grab, e &i.EndedAt, &i.EndReason, &i.EndDetail, + &i.Progress, + &i.ImportState, + &i.ImportReason, + &i.ImportDetail, + &i.ImportForced, + &i.ImportAttempts, + &i.ImportUpdatedAt, + &i.ImportedAt, + &i.ImportedFiles, + &i.Destination, ) return i, err } @@ -558,19 +581,26 @@ UPDATE acquisition_settings SET auto_grab = $1, search_interval_hours = $2, allowed_qualities = $3, + import_root = $4, updated_at = now() WHERE id = true -RETURNING id, auto_grab, search_interval_hours, allowed_qualities, last_pass_at, last_pass, updated_at +RETURNING id, auto_grab, search_interval_hours, allowed_qualities, last_pass_at, last_pass, updated_at, import_root ` type UpdateAcquisitionSettingsParams struct { AutoGrab bool SearchIntervalHours int32 AllowedQualities []string + ImportRoot string } func (q *Queries) UpdateAcquisitionSettings(ctx context.Context, arg UpdateAcquisitionSettingsParams) (AcquisitionSetting, error) { - row := q.db.QueryRow(ctx, updateAcquisitionSettings, arg.AutoGrab, arg.SearchIntervalHours, arg.AllowedQualities) + row := q.db.QueryRow(ctx, updateAcquisitionSettings, + arg.AutoGrab, + arg.SearchIntervalHours, + arg.AllowedQualities, + arg.ImportRoot, + ) var i AcquisitionSetting err := row.Scan( &i.ID, @@ -580,6 +610,7 @@ func (q *Queries) UpdateAcquisitionSettings(ctx context.Context, arg UpdateAcqui &i.LastPassAt, &i.LastPass, &i.UpdatedAt, + &i.ImportRoot, ) return i, err } diff --git a/internal/db/dbq/wanted.sql.go b/internal/db/dbq/wanted.sql.go index 1374189c..3f467f48 100644 --- a/internal/db/dbq/wanted.sql.go +++ b/internal/db/dbq/wanted.sql.go @@ -24,7 +24,13 @@ const countWants = `-- name: CountWants :one SELECT COUNT(*) FILTER (WHERE fulfilled_at IS NULL AND reason <> 'missing')::bigint AS wanted, COUNT(*) FILTER (WHERE fulfilled_at IS NULL AND reason = 'missing')::bigint AS missing, COUNT(*) FILTER (WHERE fulfilled_at IS NOT NULL)::bigint AS held, - (SELECT COUNT(*) FROM follows)::bigint AS following + (SELECT COUNT(*) FROM follows)::bigint AS following, + -- The Wanted view's other tabs (#5522): downloads running, those + -- waiting on a person, and releases set aside. + (SELECT COUNT(*) FROM grabs WHERE ended_at IS NULL)::bigint AS activity, + (SELECT COUNT(*) FROM grabs + WHERE ended_at IS NULL AND import_state IN ('review', 'failed'))::bigint AS attention, + (SELECT COUNT(*) FROM release_blocks)::bigint AS blocked FROM wants ` @@ -33,6 +39,9 @@ type CountWantsRow struct { Missing int64 Held int64 Following int64 + Activity int64 + Attention int64 + Blocked int64 } func (q *Queries) CountWants(ctx context.Context) (CountWantsRow, error) { @@ -43,6 +52,9 @@ func (q *Queries) CountWants(ctx context.Context) (CountWantsRow, error) { &i.Missing, &i.Held, &i.Following, + &i.Activity, + &i.Attention, + &i.Blocked, ) return i, err } @@ -520,11 +532,18 @@ SELECT w.id, w.release_group_mbid, w.artist_mbid, w.artist_name, w.title, w.reas (SELECT ar.id FROM artists ar WHERE ar.mbid = w.artist_mbid) AS library_artist_id, -- The search schedule's last look (#5521), and the download running. w.searched_at, w.search_outcome, w.search_considered, w.best_release, w.best_verdict, w.search_error, - gr.name AS grab_name, gr.grabbed_at, gr.origin AS grab_origin, gr.quality AS grab_quality + -- Its latest download, running or ended, and what the import made of + -- it (#5522). + gr.id AS grab_id, gr.name AS grab_name, gr.grabbed_at, gr.origin AS grab_origin, + gr.quality AS grab_quality, gr.ended_at AS grab_ended_at, gr.end_reason AS grab_end_reason, + gr.progress AS grab_progress, gr.import_state AS grab_import_state, + gr.import_reason AS grab_import_reason, gr.imported_at AS grab_imported_at, + gr.imported_files AS grab_imported_files FROM wants w LEFT JOIN users u ON u.id = w.created_by LEFT JOIN mb_release_groups g ON g.mbid = w.release_group_mbid - LEFT JOIN grabs gr ON gr.want_id = w.id AND gr.ended_at IS NULL + LEFT JOIN grabs gr ON gr.id = (SELECT x.id FROM grabs x WHERE x.want_id = w.id + ORDER BY x.ended_at IS NULL DESC, x.grabbed_at DESC LIMIT 1) WHERE CASE $1::text WHEN 'held' THEN w.fulfilled_at IS NOT NULL WHEN 'missing' THEN w.fulfilled_at IS NULL AND w.reason = 'missing' @@ -541,30 +560,38 @@ type ListWantsParams struct { } type ListWantsRow struct { - ID pgtype.UUID - ReleaseGroupMbid string - ArtistMbid string - ArtistName string - Title string - Reason string - AlbumID pgtype.UUID - WantedAt pgtype.Timestamptz - FulfilledAt pgtype.Timestamptz - PrimaryType string - SecondaryTypes []string - FirstReleaseDate string - RequestedBy *string - LibraryArtistID pgtype.UUID - SearchedAt pgtype.Timestamptz - SearchOutcome string - SearchConsidered int32 - BestRelease string - BestVerdict string - SearchError string - GrabName *string - GrabbedAt pgtype.Timestamptz - GrabOrigin *string - GrabQuality *string + ID pgtype.UUID + ReleaseGroupMbid string + ArtistMbid string + ArtistName string + Title string + Reason string + AlbumID pgtype.UUID + WantedAt pgtype.Timestamptz + FulfilledAt pgtype.Timestamptz + PrimaryType string + SecondaryTypes []string + FirstReleaseDate string + RequestedBy *string + LibraryArtistID pgtype.UUID + SearchedAt pgtype.Timestamptz + SearchOutcome string + SearchConsidered int32 + BestRelease string + BestVerdict string + SearchError string + GrabID pgtype.UUID + GrabName *string + GrabbedAt pgtype.Timestamptz + GrabOrigin *string + GrabQuality *string + GrabEndedAt pgtype.Timestamptz + GrabEndReason *string + GrabProgress *float32 + GrabImportState *string + GrabImportReason *string + GrabImportedAt pgtype.Timestamptz + GrabImportedFiles *int32 } // The Wanted page: open wants (all but missing ones, or only missing ones), @@ -599,10 +626,18 @@ func (q *Queries) ListWants(ctx context.Context, arg ListWantsParams) ([]ListWan &i.BestRelease, &i.BestVerdict, &i.SearchError, + &i.GrabID, &i.GrabName, &i.GrabbedAt, &i.GrabOrigin, &i.GrabQuality, + &i.GrabEndedAt, + &i.GrabEndReason, + &i.GrabProgress, + &i.GrabImportState, + &i.GrabImportReason, + &i.GrabImportedAt, + &i.GrabImportedFiles, ); err != nil { return nil, err } diff --git a/internal/db/migrations/0083_import.down.sql b/internal/db/migrations/0083_import.down.sql new file mode 100644 index 00000000..a7677dcb --- /dev/null +++ b/internal/db/migrations/0083_import.down.sql @@ -0,0 +1,24 @@ +ALTER TABLE acquisition_settings DROP COLUMN import_root; + +DELETE FROM release_blocks WHERE reason = 'import'; +ALTER TABLE release_blocks DROP CONSTRAINT release_blocks_reason_check; +ALTER TABLE release_blocks ADD CONSTRAINT release_blocks_reason_check + CHECK (reason IN ('operator', 'files')); + +DROP INDEX grabs_live_idx; + +UPDATE grabs SET end_reason = 'superseded' WHERE end_reason IN ('imported', 'refused', 'held'); +ALTER TABLE grabs DROP CONSTRAINT grabs_end_reason_check; +ALTER TABLE grabs ADD CONSTRAINT grabs_end_reason_check CHECK (end_reason IN ('', 'superseded')); + +ALTER TABLE grabs + DROP COLUMN progress, + DROP COLUMN import_state, + DROP COLUMN import_reason, + DROP COLUMN import_detail, + DROP COLUMN import_forced, + DROP COLUMN import_attempts, + DROP COLUMN import_updated_at, + DROP COLUMN imported_at, + DROP COLUMN imported_files, + DROP COLUMN destination; diff --git a/internal/db/migrations/0083_import.up.sql b/internal/db/migrations/0083_import.up.sql new file mode 100644 index 00000000..eb30d143 --- /dev/null +++ b/internal/db/migrations/0083_import.up.sql @@ -0,0 +1,37 @@ +-- Import with verification (M502 #5522): a finished download is checked +-- against the album it was grabbed for before anything reaches the library. +-- +-- grabs (columns) how far the download has got, and what the import +-- made of it: the verdict, its reason and evidence, +-- where the files went. +-- grabs.end_reason a grab also ends by being imported, refused, or +-- found to be an album already held. +-- release_blocks.reason import: the files were not the album. +-- acquisition_settings which library folder new music is filed into +-- ('' = the first scan path). +ALTER TABLE grabs + ADD COLUMN progress real NOT NULL DEFAULT 0, + ADD COLUMN import_state text NOT NULL DEFAULT '' + CONSTRAINT grabs_import_state_check CHECK (import_state IN + ('', 'downloading', 'waiting', 'importing', 'imported', 'refused', 'review', 'held', 'failed')), + ADD COLUMN import_reason text NOT NULL DEFAULT '', + ADD COLUMN import_detail jsonb NOT NULL DEFAULT '{}', + -- A person said "import anyway" to a download the check held for them. + ADD COLUMN import_forced boolean NOT NULL DEFAULT false, + ADD COLUMN import_attempts integer NOT NULL DEFAULT 0, + ADD COLUMN import_updated_at timestamptz, + ADD COLUMN imported_at timestamptz, + ADD COLUMN imported_files integer NOT NULL DEFAULT 0, + ADD COLUMN destination text NOT NULL DEFAULT ''; + +ALTER TABLE grabs DROP CONSTRAINT grabs_end_reason_check; +ALTER TABLE grabs ADD CONSTRAINT grabs_end_reason_check + CHECK (end_reason IN ('', 'superseded', 'imported', 'refused', 'held')); + +CREATE INDEX grabs_live_idx ON grabs (grabbed_at) WHERE ended_at IS NULL; + +ALTER TABLE release_blocks DROP CONSTRAINT release_blocks_reason_check; +ALTER TABLE release_blocks ADD CONSTRAINT release_blocks_reason_check + CHECK (reason IN ('operator', 'files', 'import')); + +ALTER TABLE acquisition_settings ADD COLUMN import_root text NOT NULL DEFAULT ''; diff --git a/internal/db/queries/imports.sql b/internal/db/queries/imports.sql new file mode 100644 index 00000000..28bf0ef0 --- /dev/null +++ b/internal/db/queries/imports.sql @@ -0,0 +1,130 @@ +-- Import with verification (M502 #5522). + +-- name: ListLiveGrabs :many +-- Every download still running or waiting on its import, oldest first. +SELECT g.*, w.release_group_mbid, w.artist_mbid, w.artist_name, w.title AS want_title, + w.reason AS want_reason, w.album_id AS want_album_id + FROM grabs g + LEFT JOIN wants w ON w.id = g.want_id + WHERE g.ended_at IS NULL + ORDER BY g.grabbed_at, g.id; + +-- name: GetGrab :one +SELECT * FROM grabs WHERE id = $1; + +-- name: SetGrabProgress :exec +-- What the download client says of a download not yet finished, or finished +-- and waiting for what the import needs. Leaves a claimed, decided or failed +-- import alone. +UPDATE grabs + SET progress = sqlc.arg(progress), + import_state = sqlc.arg(import_state), + import_reason = sqlc.arg(import_reason), + import_updated_at = now() + WHERE id = sqlc.arg(id) AND ended_at IS NULL + AND import_state IN ('', 'downloading', 'waiting'); + +-- name: ClaimGrabImport :one +-- Takes a finished download for import. The update is the lock: a second +-- pass finds nothing to claim. +UPDATE grabs + SET import_state = 'importing', import_attempts = import_attempts + 1, + import_updated_at = now(), progress = 1 + WHERE id = $1 AND ended_at IS NULL AND import_state IN ('', 'downloading', 'waiting') +RETURNING *; + +-- name: ReclaimStaleImports :execrows +-- An import cut short (a restart mid-copy) is tried again. +UPDATE grabs + SET import_state = 'waiting', + import_reason = 'The last import was cut short, so it is being tried again.' + WHERE ended_at IS NULL AND import_state = 'importing' + AND import_updated_at < now() - make_interval(mins => sqlc.arg(minutes)::int); + +-- name: RecordImport :one +-- The import's outcome. An end reason ends the grab: imported, refused or +-- held. Without one the grab stays live: review and failed wait for a person. +UPDATE grabs + SET import_state = sqlc.arg(import_state), + import_reason = sqlc.arg(import_reason), + import_detail = sqlc.arg(import_detail), + import_updated_at = now(), + import_forced = false, + imported_at = CASE WHEN sqlc.arg(import_state)::text = 'imported' THEN now() ELSE imported_at END, + imported_files = sqlc.arg(imported_files), + destination = sqlc.arg(destination), + ended_at = CASE WHEN sqlc.arg(end_reason)::text <> '' THEN now() ELSE ended_at END, + end_reason = sqlc.arg(end_reason), + end_detail = CASE WHEN sqlc.arg(end_reason)::text <> '' THEN sqlc.arg(import_reason)::text ELSE end_detail END + WHERE id = sqlc.arg(id) +RETURNING *; + +-- name: ForceGrabImport :one +-- "Import anyway": the next pass imports the download whatever the check said. +UPDATE grabs + SET import_state = '', import_forced = true, import_reason = '', import_updated_at = now() + WHERE id = $1 AND ended_at IS NULL AND import_state IN ('review', 'failed') +RETURNING *; + +-- name: RetryGrabImport :one +UPDATE grabs + SET import_state = '', import_reason = '', import_updated_at = now() + WHERE id = $1 AND ended_at IS NULL AND import_state = 'failed' +RETURNING *; + +-- name: RehomeGrab :one +-- A download that turned out to be another wanted album is that want's grab, +-- when the want has none running. +UPDATE grabs g + SET want_id = sqlc.arg(want_id) + WHERE g.id = sqlc.arg(id) AND g.ended_at IS NULL + AND NOT EXISTS (SELECT 1 FROM grabs o WHERE o.want_id = sqlc.arg(want_id) AND o.ended_at IS NULL) +RETURNING *; + +-- name: OpenWantsForGroups :many +-- Open wants for these release groups with no download running. +SELECT w.* FROM wants w + WHERE w.fulfilled_at IS NULL AND w.release_group_mbid = ANY(sqlc.arg(groups)::text[]) + AND NOT EXISTS (SELECT 1 FROM grabs g WHERE g.want_id = w.id AND g.ended_at IS NULL); + +-- name: MarkWantSearched :exec +-- Imported: nothing to search for until the wanted pass settles the want. +UPDATE wants SET searched_at = now() WHERE id = $1; + +-- name: SearchWantAgain :exec +-- Refused: search again on the next pass. +UPDATE wants SET searched_at = NULL, search_outcome = '' WHERE id = $1; + +-- name: HeldRecordings :many +-- Which of these recordings the library holds a file for. +SELECT DISTINCT mbid::text AS mbid FROM tracks + WHERE mbid = ANY(sqlc.arg(recordings)::text[]) AND missing_since IS NULL; + +-- name: HeldPositions :many +-- The disc and track positions the library holds for an album of this group. +SELECT DISTINCT COALESCE(t.disc_number, 1)::int AS disc, t.track_number::int AS track + FROM tracks t + JOIN albums a ON a.id = t.album_id + WHERE a.release_group_mbid = sqlc.arg(release_group_mbid) + AND t.missing_since IS NULL AND t.track_number IS NOT NULL; + +-- name: ListGrabs :many +-- The Wanted view's Activity tab: downloads waiting on a person first, then +-- the rest still running, then the ended ones newest first. +SELECT g.id, g.want_id, g.name, g.indexer, g.client, g.quality, g.origin, g.forced, + g.grabbed_at, g.ended_at, g.end_reason, g.end_detail, g.progress, g.import_state, + g.import_reason, g.import_detail, g.import_updated_at, g.imported_at, g.imported_files, + g.destination, w.title AS want_title, w.artist_name AS want_artist + FROM grabs g + LEFT JOIN wants w ON w.id = g.want_id + ORDER BY (g.ended_at IS NULL AND g.import_state IN ('review', 'failed')) DESC, + g.ended_at IS NULL DESC, g.grabbed_at DESC, g.id + LIMIT sqlc.arg(max_rows) OFFSET sqlc.arg(skip); + +-- name: ListReleaseBlocksPage :many +SELECT b.info_hash, b.name, b.reason, b.detail, b.blocked_at, w.title AS want_title, + w.artist_name AS want_artist + FROM release_blocks b + LEFT JOIN wants w ON w.id = b.want_id + ORDER BY b.blocked_at DESC, b.info_hash + LIMIT sqlc.arg(max_rows) OFFSET sqlc.arg(skip); diff --git a/internal/db/queries/search.sql b/internal/db/queries/search.sql index 832e35f2..9dc88be8 100644 --- a/internal/db/queries/search.sql +++ b/internal/db/queries/search.sql @@ -8,6 +8,7 @@ UPDATE acquisition_settings SET auto_grab = sqlc.arg(auto_grab), search_interval_hours = sqlc.arg(search_interval_hours), allowed_qualities = sqlc.arg(allowed_qualities), + import_root = sqlc.arg(import_root), updated_at = now() WHERE id = true RETURNING *; @@ -34,7 +35,7 @@ SELECT w.id, w.release_group_mbid, w.artist_name, w.title, w.reason, LIMIT sqlc.arg(max_rows); -- name: GetWantForSearch :one -SELECT w.id, w.release_group_mbid, w.artist_name, w.title, w.reason, w.fulfilled_at, +SELECT w.id, w.release_group_mbid, w.artist_mbid, w.artist_name, w.title, w.reason, w.fulfilled_at, COALESCE(NULLIF(w.primary_type, ''), g.primary_type, '')::text AS primary_type, COALESCE(g.secondary_types, '{}')::text[] AS secondary_types, COALESCE(NULLIF(w.first_release_date, ''), g.first_release_date, '')::text AS first_release_date diff --git a/internal/db/queries/wanted.sql b/internal/db/queries/wanted.sql index bde3219d..d52d47a6 100644 --- a/internal/db/queries/wanted.sql +++ b/internal/db/queries/wanted.sql @@ -189,11 +189,18 @@ SELECT w.id, w.release_group_mbid, w.artist_mbid, w.artist_name, w.title, w.reas (SELECT ar.id FROM artists ar WHERE ar.mbid = w.artist_mbid) AS library_artist_id, -- The search schedule's last look (#5521), and the download running. w.searched_at, w.search_outcome, w.search_considered, w.best_release, w.best_verdict, w.search_error, - gr.name AS grab_name, gr.grabbed_at, gr.origin AS grab_origin, gr.quality AS grab_quality + -- Its latest download, running or ended, and what the import made of + -- it (#5522). + gr.id AS grab_id, gr.name AS grab_name, gr.grabbed_at, gr.origin AS grab_origin, + gr.quality AS grab_quality, gr.ended_at AS grab_ended_at, gr.end_reason AS grab_end_reason, + gr.progress AS grab_progress, gr.import_state AS grab_import_state, + gr.import_reason AS grab_import_reason, gr.imported_at AS grab_imported_at, + gr.imported_files AS grab_imported_files FROM wants w LEFT JOIN users u ON u.id = w.created_by LEFT JOIN mb_release_groups g ON g.mbid = w.release_group_mbid - LEFT JOIN grabs gr ON gr.want_id = w.id AND gr.ended_at IS NULL + LEFT JOIN grabs gr ON gr.id = (SELECT x.id FROM grabs x WHERE x.want_id = w.id + ORDER BY x.ended_at IS NULL DESC, x.grabbed_at DESC LIMIT 1) WHERE CASE sqlc.arg(state)::text WHEN 'held' THEN w.fulfilled_at IS NOT NULL WHEN 'missing' THEN w.fulfilled_at IS NULL AND w.reason = 'missing' @@ -206,5 +213,11 @@ SELECT w.id, w.release_group_mbid, w.artist_mbid, w.artist_name, w.title, w.reas SELECT COUNT(*) FILTER (WHERE fulfilled_at IS NULL AND reason <> 'missing')::bigint AS wanted, COUNT(*) FILTER (WHERE fulfilled_at IS NULL AND reason = 'missing')::bigint AS missing, COUNT(*) FILTER (WHERE fulfilled_at IS NOT NULL)::bigint AS held, - (SELECT COUNT(*) FROM follows)::bigint AS following + (SELECT COUNT(*) FROM follows)::bigint AS following, + -- The Wanted view's other tabs (#5522): downloads running, those + -- waiting on a person, and releases set aside. + (SELECT COUNT(*) FROM grabs WHERE ended_at IS NULL)::bigint AS activity, + (SELECT COUNT(*) FROM grabs + WHERE ended_at IS NULL AND import_state IN ('review', 'failed'))::bigint AS attention, + (SELECT COUNT(*) FROM release_blocks)::bigint AS blocked FROM wants; diff --git a/internal/dbtest/reset.go b/internal/dbtest/reset.go index fd1b7865..8b138e24 100644 --- a/internal/dbtest/reset.go +++ b/internal/dbtest/reset.go @@ -186,7 +186,7 @@ func ResetDB(t *testing.T, pool *pgxpool.Pool) { // Search and grab settings (M502 #5521), a singleton reset the same way. if _, err := pool.Exec(ctx, ` UPDATE acquisition_settings - SET auto_grab = DEFAULT, search_interval_hours = DEFAULT, allowed_qualities = DEFAULT, + SET auto_grab = DEFAULT, search_interval_hours = DEFAULT, allowed_qualities = DEFAULT, import_root = DEFAULT, last_pass_at = NULL, last_pass = DEFAULT, updated_at = DEFAULT`, ); err != nil { t.Fatalf("dbtest.ResetDB reset acquisition settings: %v", err) diff --git a/internal/importer/importer.go b/internal/importer/importer.go new file mode 100644 index 00000000..5811f03d --- /dev/null +++ b/internal/importer/importer.go @@ -0,0 +1,608 @@ +package importer + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "log/slog" + "os" + "path/filepath" + "slices" + "strconv" + "sync" + "time" + + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgtype" + "github.com/jackc/pgx/v5/pgxpool" + + "git.fabledsword.com/bvandeusen/minstrel/internal/acquire" + "git.fabledsword.com/bvandeusen/minstrel/internal/catalogue" + "git.fabledsword.com/bvandeusen/minstrel/internal/db/dbq" + "git.fabledsword.com/bvandeusen/minstrel/internal/downloader" + "git.fabledsword.com/bvandeusen/minstrel/internal/indexer" +) + +// Import states, as grabs.import_state holds them. +const ( + StateDownloading = "downloading" + StateWaiting = "waiting" + StateImporting = "importing" + StateImported = "imported" + StateRefused = "refused" + StateReview = "review" + StateHeld = "held" + StateFailed = "failed" +) + +const ( + // interval: a finished download is noticed within a minute. One + // Torrents call per download client, so the client barely notices. + interval = time.Minute + // passTimeout bounds a pass: fingerprinting a long album against + // AcoustID's pace takes a minute or two, so half an hour is a hang. + passTimeout = 30 * time.Minute + // staleMinutes: an import still "importing" after this was cut short + // (a restart mid-copy) and is tried again. + staleMinutes = 30 +) + +// Scanner reads new files into the library. +type Scanner interface { + ScanFiles(ctx context.Context, paths []string) ([]pgtype.UUID, error) +} + +// Catalogue lists a release group's editions. +type Catalogue interface { + Releases(ctx context.Context, groupMBID string) (catalogue.Editions, error) +} + +// Importer watches every live grab's download and imports it once finished. +type Importer struct { + pool *pgxpool.Pool + Scanner Scanner + Catalogue Catalogue + Printer Printer + // Roots are the library's folders (the scan paths). New music is filed + // under the one the settings name, or the first. + Roots []string + // ClientFor builds the download client a grab went to. Defaults to the + // acquisition_services row of that code. + ClientFor func(ctx context.Context, code string) (downloader.Driver, error) + logger *slog.Logger + wake chan struct{} + mu sync.Mutex // one pass at a time +} + +// New returns an Importer. printer may be nil: the check then rests on +// lengths and tags. +func New(pool *pgxpool.Pool, scanner Scanner, cat Catalogue, printer Printer, roots []string, logger *slog.Logger) *Importer { + if logger == nil { + logger = slog.Default() + } + im := &Importer{pool: pool, Scanner: scanner, Catalogue: cat, Printer: printer, Roots: roots, + logger: logger, wake: make(chan struct{}, 1)} + im.ClientFor = im.clientFor + return im +} + +func (im *Importer) clientFor(ctx context.Context, code string) (downloader.Driver, error) { + row, err := dbq.New(im.pool).GetAcquisitionService(ctx, code) + if err != nil { + if errors.Is(err, pgx.ErrNoRows) { + return nil, fmt.Errorf("the download client %q is no longer set up", code) + } + return nil, err + } + return acquire.DownloadClient(row) +} + +// Wake asks for a pass now: after a grab, or a person's decision. +func (im *Importer) Wake() { + select { + case im.wake <- struct{}{}: + default: + } +} + +// Run passes every minute, and when woken, until ctx ends. +func (im *Importer) Run(ctx context.Context) { + t := time.NewTicker(interval) + defer t.Stop() + for { + select { + case <-ctx.Done(): + return + case <-t.C: + case <-im.wake: + } + rep := im.Pass(ctx) + if rep.Imported+rep.Refused+rep.Review+rep.Held+rep.Failed > 0 || rep.Error != "" { + im.logger.Info("import: pass", "imported", rep.Imported, "refused", rep.Refused, + "review", rep.Review, "held", rep.Held, "failed", rep.Failed, + "downloading", rep.Downloading, "error", rep.Error) + } + } +} + +// Report is one pass. +type Report struct { + Downloading, Imported, Refused, Review, Held, Failed, Waiting int + Error string +} + +// Pass looks at every live grab once. +func (im *Importer) Pass(ctx context.Context) Report { + im.mu.Lock() + defer im.mu.Unlock() + ctx, cancel := context.WithTimeout(ctx, passTimeout) + defer cancel() + var rep Report + q := dbq.New(im.pool) + if _, err := q.ReclaimStaleImports(ctx, staleMinutes); err != nil { + rep.Error = err.Error() + return rep + } + grabs, err := q.ListLiveGrabs(ctx) + if err != nil { + rep.Error = err.Error() + return rep + } + byClient := map[string][]dbq.ListLiveGrabsRow{} + var order []string + for _, g := range grabs { + if _, ok := byClient[g.Client]; !ok { + order = append(order, g.Client) + } + byClient[g.Client] = append(byClient[g.Client], g) + } + for _, code := range order { + im.passClient(ctx, q, code, byClient[code], &rep) + } + return rep +} + +func (im *Importer) passClient(ctx context.Context, q *dbq.Queries, code string, gs []dbq.ListLiveGrabsRow, rep *Report) { + wait := func(g dbq.ListLiveGrabsRow, why string) { + rep.Waiting++ + _ = q.SetGrabProgress(ctx, dbq.SetGrabProgressParams{ + ID: g.ID, Progress: g.Progress, ImportState: StateWaiting, ImportReason: why}) + } + d, err := im.ClientFor(ctx, code) + if err != nil { + for _, g := range gs { + wait(g, fmt.Sprintf("Can't reach its download client: %v.", err)) + } + return + } + hashes := make([]string, 0, len(gs)) + for _, g := range gs { + hashes = append(hashes, g.InfoHash) + } + ts, err := d.Torrents(ctx, hashes...) + if err != nil { + for _, g := range gs { + wait(g, fmt.Sprintf("Can't reach its download client %s: %s", code, indexer.FirstLine(err.Error()))) + } + return + } + byHash := map[string]downloader.Torrent{} + for _, t := range ts { + byHash[indexer.NormalizeHash(t.Hash)] = t + } + for _, g := range gs { + if ctx.Err() != nil { + return + } + switch g.ImportState { + case StateReview, StateFailed, StateImporting: + continue + } + t, ok := byHash[indexer.NormalizeHash(g.InfoHash)] + if !ok { + continue // gone from the client: the download lifecycle's (#5523) + } + if !g.WantID.Valid { + wait(g, "Its album is no longer wanted.") + continue + } + if !finished(ctx, d, t) { + rep.Downloading++ + _ = q.SetGrabProgress(ctx, dbq.SetGrabProgressParams{ + ID: g.ID, Progress: float32(min(t.Progress, 0.999)), ImportState: StateDownloading}) + continue + } + claimed, err := q.ClaimGrabImport(ctx, g.ID) + if err != nil { + continue // claimed by a pass racing this one, or ended + } + switch out := im.importOne(ctx, q, claimed, d, t); out { + case StateImported: + rep.Imported++ + case StateRefused: + rep.Refused++ + case StateReview: + rep.Review++ + case StateHeld: + rep.Held++ + case StateFailed: + rep.Failed++ + case StateWaiting: + rep.Waiting++ + } + } +} + +// upStates are qBittorrent's states for a torrent that has every byte and +// is seeding, paused or queued to seed. Exchange's states arrive in the same +// vocabulary. +var upStates = map[string]bool{ + "uploading": true, "stalledUP": true, "pausedUP": true, "stoppedUP": true, + "queuedUP": true, "forcedUP": true, "checkingUP": true, +} + +// finished: whole, in a seeding state, and no wanted file short of bytes. +func finished(ctx context.Context, d downloader.Driver, t downloader.Torrent) bool { + if !t.Done() || !upStates[t.State] { + return false + } + if fp, ok := d.(downloader.FileProgress); ok { + if states, err := fp.FileStates(ctx, t.Hash); err == nil { + for _, f := range states { + if f.Unfinished() { + return false + } + } + } + } + return true +} + +// Detail is what grabs.import_detail keeps: the verdict and what it rested +// on, the edition chosen, and what happened to the files. +type Detail struct { + Verdict Verdict `json:"verdict"` + Edition string `json:"edition,omitempty"` + EditionMBID string `json:"edition_mbid,omitempty"` + Files []string `json:"files,omitempty"` + Placed int `json:"placed,omitempty"` + Skipped int `json:"skipped,omitempty"` + Untagged int `json:"untagged,omitempty"` + // RehomedFrom: the album it was grabbed for, when it turned out to be + // another wanted album. + RehomedFrom string `json:"rehomed_from,omitempty"` +} + +// importOne checks and files one finished download and records the outcome. +func (im *Importer) importOne(ctx context.Context, q *dbq.Queries, g dbq.Grab, d downloader.Driver, t downloader.Torrent) string { + record := func(state, reason, end string, det Detail, files int, dest string) string { + body, _ := json.Marshal(det) + // The outcome is written even when the pass's context is ending: + // a claimed grab left "importing" waits half an hour to be retried. + wctx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 15*time.Second) + defer cancel() + if _, err := q.RecordImport(wctx, dbq.RecordImportParams{ + ID: g.ID, ImportState: state, ImportReason: reason, ImportDetail: body, + ImportedFiles: int32(files), Destination: dest, EndReason: end, + }); err != nil { + im.logger.Warn("import: record outcome", "grab", g.Name, "state", state, "err", err) + } + im.logger.Info("import: "+state, "grab", g.Name, "reason", reason) + return state + } + + w, err := im.want(ctx, q, g.WantID) + if err != nil { + return record(StateWaiting, "Its album could not be read: "+err.Error(), "", Detail{}, 0, "") + } + editions, err := im.editions(ctx, w.GroupMBID) + if err != nil { + return record(StateWaiting, "MusicBrainz could not be reached to check it against the album; trying again shortly.", + "", Detail{}, 0, "") + } + src := t.ContentPath + if src == "" { + src = filepath.Join(g.SavePath, t.Name) + } + files, printed, err := inspect(ctx, src, im.Printer) + if err != nil { + return record(StateFailed, fmt.Sprintf("Couldn't read the download at %s: %v", src, err), "", Detail{}, 0, "") + } + v := Verify(w, editions, files, printed, g.ImportForced) + det := Detail{Verdict: v} + for _, f := range files { + det.Files = append(det.Files, f.Rel) + } + if v.Edition >= 0 { + det.Edition, det.EditionMBID = describe(editions[v.Edition]), editions[v.Edition].MBID + } + + switch v.Outcome { + case Review: + return record(StateReview, v.Reason, "", det, 0, "") + case Refuse: + if v.Code == CodeWrongAlbum { + if out, ok := im.rehome(ctx, q, g, d, w, files, printed, record); ok { + return out + } + } + im.refuse(ctx, q, g, d, w, v.Reason, pgtype.UUID{}) + return record(StateRefused, v.Reason, StateRefused, det, 0, "") + } + return im.place(ctx, q, g, d, w, editions, files, v, det, record) +} + +type recordFunc func(state, reason, end string, det Detail, files int, dest string) string + +// place files an accepted download and records it imported, or held when +// the library already had every track. +func (im *Importer) place(ctx context.Context, q *dbq.Queries, g dbq.Grab, d downloader.Driver, w Want, + editions []Edition, files []File, v Verdict, det Detail, record recordFunc) string { + root, err := im.root(ctx, q) + if err != nil { + return record(StateFailed, "Not imported: "+err.Error()+".", "", det, 0, "") + } + var e *Edition + if v.Edition >= 0 { + e = &editions[v.Edition] + } + plan := Plan(root, w, e, files, v.Pairs) + heldRec, heldPos, err := im.held(ctx, q, w, plan) + if err != nil { + return record(StateWaiting, "Couldn't read what the library holds: "+err.Error(), "", det, 0, "") + } + + var placed []string + for _, pl := range plan { + if (pl.Recording != "" && heldRec[pl.Recording]) || heldPos[[2]int{pl.Disc, pl.Track}] { + det.Skipped++ + continue + } + tags := Tags{Title: pl.Title, Artist: w.Artist, Album: w.Title, AlbumArtist: w.Artist, Year: w.Year, + Disc: pl.Disc, Discs: pl.Discs, Track: pl.Track, Tracks: pl.Tracks, + GroupMBID: w.GroupMBID, RecordingMBID: pl.Recording, ArtistMBID: w.ArtistMBID} + if e != nil { + tags.ReleaseMBID = e.MBID + } + tagged, err := Place(ctx, files[pl.File].Path, pl.Dest, tags) + if errors.Is(err, ErrOccupied) { + det.Skipped++ + continue + } + if err != nil { + for _, p := range placed { + _ = os.Remove(p) + } + return record(StateFailed, fmt.Sprintf("Copying into the library failed, and nothing was left behind: %v", err), + "", det, 0, "") + } + placed = append(placed, pl.Dest) + if !tagged { + det.Untagged++ + } + } + det.Placed = len(placed) + if len(placed) == 0 { + im.removeFromClient(ctx, d, g) + return record(StateHeld, fmt.Sprintf("The library already holds every track of %s, so nothing was imported.", w.Title), + StateHeld, det, 0, "") + } + + if _, err := im.Scanner.ScanFiles(ctx, placed); err != nil { + // The watcher and the next scan find the files anyway. + im.logger.Warn("import: scan the new files", "grab", g.Name, "err", err) + } + if err := q.MarkWantSearched(ctx, g.WantID); err != nil { + im.logger.Warn("import: mark the want searched", "grab", g.Name, "err", err) + } + dest := filepath.Dir(placed[0]) + if e != nil && e.Media() > 1 { + dest = filepath.Dir(dest) + } + reason := fmt.Sprintf("Imported %s into %s. %s", tracks(len(placed)), dest, v.Reason) + if det.Skipped > 0 { + reason += fmt.Sprintf(" %s already in the library left as they were.", tracks(det.Skipped)) + } + if det.Untagged > 0 { + reason += fmt.Sprintf(" %s copied with their own tags: ffmpeg could not rewrite them.", tracks(det.Untagged)) + } + return record(StateImported, reason, StateImported, det, len(placed), dest) +} + +// rehome imports a download that turned out to be another wanted album, +// named by its tags, for that album, when its own grab is not running. +func (im *Importer) rehome(ctx context.Context, q *dbq.Queries, g dbq.Grab, d downloader.Driver, + from Want, files []File, printed bool, record recordFunc) (string, bool) { + group := majority(files, func(f File) string { return f.GroupMBID }) + if group == "" { + return "", false + } + others, err := q.OpenWantsForGroups(ctx, []string{group}) + if err != nil || len(others) == 0 { + return "", false + } + other := others[0] + w, err := im.want(ctx, q, other.ID) + if err != nil { + return "", false + } + editions, err := im.editions(ctx, w.GroupMBID) + if err != nil { + return "", false + } + v := Verify(w, editions, files, printed, false) + if v.Outcome != Accept { + return "", false + } + moved, err := q.RehomeGrab(ctx, dbq.RehomeGrabParams{ID: g.ID, WantID: other.ID}) + if err != nil { + return "", false + } + if err := q.SearchWantAgain(ctx, g.WantID); err != nil { + im.logger.Warn("import: search the first album again", "want", from.Title, "err", err) + } + det := Detail{Verdict: v, RehomedFrom: from.Title} + for _, f := range files { + det.Files = append(det.Files, f.Rel) + } + if v.Edition >= 0 { + det.Edition, det.EditionMBID = describe(editions[v.Edition]), editions[v.Edition].MBID + } + v.Reason = fmt.Sprintf("Grabbed for %s, it is %s, which is wanted too. %s", from.Title, w.Title, v.Reason) + det.Verdict = v + return im.place(ctx, q, moved, d, w, editions, files, v, det, record), true +} + +// refuse blocks the release, asks for the album to be searched again and +// removes the download with its files: it is not the album. +func (im *Importer) refuse(ctx context.Context, q *dbq.Queries, g dbq.Grab, d downloader.Driver, w Want, why string, by pgtype.UUID) { + if err := acquire.NewMemory(im.pool).Block(ctx, dbq.BlockReleaseParams{ + InfoHash: g.InfoHash, Name: g.Name, Reason: acquire.BlockImport, Detail: why, WantID: g.WantID, BlockedBy: by, + }); err != nil { + im.logger.Warn("import: block the release", "grab", g.Name, "err", err) + } + if err := q.SearchWantAgain(ctx, g.WantID); err != nil { + im.logger.Warn("import: search again", "want", w.Title, "err", err) + } + im.removeFromClient(ctx, d, g) +} + +func (im *Importer) removeFromClient(ctx context.Context, d downloader.Driver, g dbq.Grab) { + if d == nil || g.InfoHash == "" { + return + } + if err := d.Delete(ctx, true, g.InfoHash); err != nil { + im.logger.Warn("import: remove the download", "grab", g.Name, "err", err) + } +} + +// Refuse is a person's "refuse" on a download the check held for them, or +// one that failed: it is refused as the check would have. +func (im *Importer) Refuse(ctx context.Context, grabID, by pgtype.UUID) (dbq.Grab, error) { + q := dbq.New(im.pool) + g, err := q.GetGrab(ctx, grabID) + if err != nil { + return dbq.Grab{}, err + } + if g.EndedAt.Valid || (g.ImportState != StateReview && g.ImportState != StateFailed) { + return dbq.Grab{}, ErrNotDecidable + } + var w Want + if g.WantID.Valid { + w, _ = im.want(ctx, q, g.WantID) + } + d, err := im.ClientFor(ctx, g.Client) + if err != nil { + d = nil + } + why := "A person refused it: it is not the album." + im.refuse(ctx, q, g, d, w, why, by) + out, err := q.RecordImport(ctx, dbq.RecordImportParams{ + ID: g.ID, ImportState: StateRefused, ImportReason: why, ImportDetail: g.ImportDetail, + EndReason: StateRefused, + }) + im.Wake() + return out, err +} + +// ErrNotDecidable: the grab is not waiting on a person. +var ErrNotDecidable = errors.New("importer: that download is not waiting on a decision") + +func (im *Importer) want(ctx context.Context, q *dbq.Queries, id pgtype.UUID) (Want, error) { + row, err := q.GetWantForSearch(ctx, id) + if err != nil { + return Want{}, err + } + return Want{ + GroupMBID: row.ReleaseGroupMbid, ArtistMBID: row.ArtistMbid, Artist: row.ArtistName, Title: row.Title, + Year: yearOf(row.FirstReleaseDate), PrimaryType: row.PrimaryType, SecondaryTypes: row.SecondaryTypes, + }, nil +} + +func (im *Importer) editions(ctx context.Context, group string) ([]Edition, error) { + if im.Catalogue == nil { + return nil, errors.New("no catalogue") + } + eds, err := im.Catalogue.Releases(ctx, group) + if err != nil { + return nil, err + } + out := make([]Edition, 0, len(eds.Editions)) + for _, e := range eds.Editions { + ed := Edition{MBID: e.Release.Mbid, Title: e.Release.Title, Status: e.Release.Status, + Date: e.Release.ReleaseDate, Country: e.Release.Country} + for _, t := range e.Tracks { + ed.Tracks = append(ed.Tracks, Track{Medium: int(t.MediumPosition), Format: t.MediumFormat, + Position: int(t.Position), Title: t.Title, RecordingMBID: t.RecordingMbid, LengthMs: int(t.LengthMs)}) + } + slices.SortStableFunc(ed.Tracks, func(a, b Track) int { + if a.Medium != b.Medium { + return a.Medium - b.Medium + } + return a.Position - b.Position + }) + out = append(out, ed) + } + return out, nil +} + +// root is the library folder new music is filed into. +func (im *Importer) root(ctx context.Context, q *dbq.Queries) (string, error) { + if len(im.Roots) == 0 { + return "", errors.New("there is no library folder to file new music into (MINSTREL_LIBRARY_SCAN_PATHS)") + } + s, err := q.GetAcquisitionSettings(ctx) + if err == nil && s.ImportRoot != "" && slices.Contains(im.Roots, s.ImportRoot) { + return s.ImportRoot, nil + } + return im.Roots[0], nil +} + +// held reads which of the planned tracks the library already has: by +// recording, and by disc and track of an album of the same group. +func (im *Importer) held(ctx context.Context, q *dbq.Queries, w Want, plan []Placement) (map[string]bool, map[[2]int]bool, error) { + var recs []string + for _, pl := range plan { + if pl.Recording != "" { + recs = append(recs, pl.Recording) + } + } + rec := map[string]bool{} + if len(recs) > 0 { + got, err := q.HeldRecordings(ctx, recs) + if err != nil { + return nil, nil, err + } + for _, r := range got { + rec[r] = true + } + } + pos := map[[2]int]bool{} + got, err := q.HeldPositions(ctx, &w.GroupMBID) + if err != nil { + return nil, nil, err + } + for _, p := range got { + pos[[2]int{int(p.Disc), int(p.Track)}] = true + } + return rec, pos, nil +} + +func yearOf(date string) int { + if len(date) < 4 { + return 0 + } + y, err := strconv.Atoi(date[:4]) + if err != nil { + return 0 + } + return y +} + +func tracks(n int) string { + if n == 1 { + return "1 track" + } + return strconv.Itoa(n) + " tracks" +} diff --git a/internal/importer/importer_test.go b/internal/importer/importer_test.go new file mode 100644 index 00000000..36766b81 --- /dev/null +++ b/internal/importer/importer_test.go @@ -0,0 +1,268 @@ +package importer + +import ( + "context" + "io" + "log/slog" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/jackc/pgx/v5/pgtype" + "github.com/jackc/pgx/v5/pgxpool" + + "git.fabledsword.com/bvandeusen/minstrel/internal/acquire" + "git.fabledsword.com/bvandeusen/minstrel/internal/catalogue" + "git.fabledsword.com/bvandeusen/minstrel/internal/db" + "git.fabledsword.com/bvandeusen/minstrel/internal/db/dbq" + "git.fabledsword.com/bvandeusen/minstrel/internal/dbtest" + "git.fabledsword.com/bvandeusen/minstrel/internal/downloader" + "git.fabledsword.com/bvandeusen/minstrel/internal/wanted" +) + +func newPool(t *testing.T) *pgxpool.Pool { + t.Helper() + if testing.Short() { + t.Skip("skipping integration test in -short mode") + } + dsn := os.Getenv("MINSTREL_TEST_DATABASE_URL") + if dsn == "" { + t.Skip("MINSTREL_TEST_DATABASE_URL not set") + } + if err := db.Migrate(dsn, slog.New(slog.NewTextHandler(io.Discard, nil))); err != nil { + t.Fatalf("migrate: %v", err) + } + pool, err := pgxpool.New(context.Background(), dsn) + if err != nil { + t.Fatalf("pool: %v", err) + } + t.Cleanup(pool.Close) + dbtest.ResetDB(t, pool) + return pool +} + +const hash = "0123456789abcdef0123456789abcdef01234567" + +// fakeClient is a download client holding one finished torrent. +type fakeClient struct { + downloader.Driver + content string + deleted []string +} + +func (c *fakeClient) Torrents(_ context.Context, _ ...string) ([]downloader.Torrent, error) { + return []downloader.Torrent{{Hash: hash, Name: "Daft Punk - RAM [FLAC]", Progress: 1, State: "uploading", + ContentPath: c.content}}, nil +} + +func (c *fakeClient) Delete(_ context.Context, _ bool, hashes ...string) error { + c.deleted = append(c.deleted, hashes...) + return nil +} + +// fakeCatalogue knows RAM as one three-track edition. +type fakeCatalogue struct{} + +func (fakeCatalogue) Releases(_ context.Context, _ string) (catalogue.Editions, error) { + return catalogue.Editions{Editions: []catalogue.Edition{{ + Release: dbq.MbRelease{Mbid: "rel-ram", Title: "Random Access Memories", Status: "Official", ReleaseDate: "2013-05-17"}, + Tracks: []dbq.MbReleaseTrack{ + {MediumPosition: 1, MediumFormat: "CD", Position: 1, Title: "Give Life Back to Music", RecordingMbid: "rec-01"}, + {MediumPosition: 1, MediumFormat: "CD", Position: 2, Title: "The Game of Love", RecordingMbid: "rec-02"}, + {MediumPosition: 1, MediumFormat: "CD", Position: 3, Title: "Giorgio by Moroder", RecordingMbid: "rec-03"}, + }, + }}}, nil +} + +// fakePrinter names each file's recording by its file name: 01.flac is +// rec-01, unless wrong is set. +type fakePrinter struct{ wrong bool } + +func (p fakePrinter) Recordings(_ context.Context, path string) ([]string, error) { + if p.wrong { + return []string{"rec-other"}, nil + } + return []string{"rec-" + strings.TrimSuffix(filepath.Base(path), ".flac")}, nil +} + +type fakeScanner struct{ scanned []string } + +func (s *fakeScanner) ScanFiles(_ context.Context, paths []string) ([]pgtype.UUID, error) { + s.scanned = append(s.scanned, paths...) + return nil, nil +} + +type stack struct { + pool *pgxpool.Pool + q *dbq.Queries + im *Importer + client *fakeClient + scanner *fakeScanner + library string + want dbq.Want + grab dbq.Grab +} + +// newStack wants RAM, has a finished download of it waiting in a fake +// client, and an importer filing into a temp library. +func newStack(t *testing.T, printer Printer) *stack { + t.Helper() + ctx := context.Background() + pool := newPool(t) + q := dbq.New(pool) + content := t.TempDir() + for _, n := range []string{"01", "02", "03"} { + if err := os.WriteFile(filepath.Join(content, n+".flac"), []byte("not really flac "+n), 0o644); err != nil { + t.Fatal(err) + } + } + w, err := wanted.WantWith(ctx, q, wanted.WantParams{GroupMBID: "rg-ram", ArtistMBID: "mb-dp", ArtistName: "Daft Punk", + Title: "Random Access Memories", PrimaryType: "Album", FirstReleaseDate: "2013-05-17", Reason: wanted.ReasonRequest}) + if err != nil { + t.Fatal(err) + } + g, err := acquire.NewMemory(pool).Grabbed(ctx, dbq.InsertGrabParams{WantID: w.ID, InfoHash: hash, + Name: "Daft Punk - RAM [FLAC]", Key: hash, Client: "qbittorrent", SavePath: content, Origin: acquire.OriginSchedule}, + pgtype.UUID{}, "") + if err != nil { + t.Fatal(err) + } + s := &stack{pool: pool, q: q, client: &fakeClient{content: content}, scanner: &fakeScanner{}, + library: t.TempDir(), want: w, grab: g} + s.im = New(pool, s.scanner, fakeCatalogue{}, printer, []string{s.library}, slog.New(slog.NewTextHandler(io.Discard, nil))) + s.im.ClientFor = func(context.Context, string) (downloader.Driver, error) { return s.client, nil } + return s +} + +func (s *stack) grabNow(t *testing.T) dbq.Grab { + t.Helper() + g, err := s.q.GetGrab(context.Background(), s.grab.ID) + if err != nil { + t.Fatal(err) + } + return g +} + +func TestImport_AFinishedDownloadOfTheAlbumIsFiled_Integration(t *testing.T) { + s := newStack(t, fakePrinter{}) + rep := s.im.Pass(context.Background()) + if rep.Imported != 1 { + t.Fatalf("pass = %+v", rep) + } + g := s.grabNow(t) + if g.ImportState != StateImported || g.EndReason != StateImported || !g.EndedAt.Valid || g.ImportedFiles != 3 { + t.Fatalf("grab = %s %s ended=%v files=%d: %s", g.ImportState, g.EndReason, g.EndedAt.Valid, g.ImportedFiles, g.ImportReason) + } + dir := filepath.Join(s.library, "Daft Punk", "Random Access Memories (2013)") + if g.Destination != dir { + t.Errorf("destination %q", g.Destination) + } + got, err := os.ReadFile(filepath.Join(dir, "Daft Punk - Random Access Memories - 02 - The Game of Love.flac")) + if err != nil || string(got) != "not really flac 02" { + t.Errorf("filed copy: %q %v", got, err) + } + if _, err := os.Stat(filepath.Join(s.client.content, "02.flac")); err != nil { + t.Errorf("the download's own file went: %v", err) + } + if len(s.scanner.scanned) != 3 || len(s.client.deleted) != 0 { + t.Errorf("scanned %v, deleted %v", s.scanner.scanned, s.client.deleted) + } + w, _ := s.q.GetWantByReleaseGroup(context.Background(), "rg-ram") + if !w.SearchedAt.Valid { + t.Error("an imported want is due for search again") + } + if s.im.Pass(context.Background()).Imported != 0 { + t.Error("imported twice") + } +} + +func TestImport_TheWrongAlbumIsRefusedBlockedAndSearchedAgain_Integration(t *testing.T) { + s := newStack(t, fakePrinter{wrong: true}) + ctx := context.Background() + if err := s.q.MarkWantSearched(ctx, s.want.ID); err != nil { + t.Fatal(err) + } + if rep := s.im.Pass(ctx); rep.Refused != 1 { + t.Fatalf("pass = %+v", rep) + } + g := s.grabNow(t) + if g.ImportState != StateRefused || g.EndReason != StateRefused || !strings.Contains(g.ImportReason, "recordings") { + t.Fatalf("grab = %s %s: %s", g.ImportState, g.EndReason, g.ImportReason) + } + blocks, err := s.q.ListReleaseBlocks(ctx) + if err != nil || len(blocks) != 1 || blocks[0].InfoHash != hash || blocks[0].Reason != acquire.BlockImport { + t.Errorf("blocks = %+v %v", blocks, err) + } + if len(s.client.deleted) != 1 { + t.Errorf("the refused download was not removed: %v", s.client.deleted) + } + w, _ := s.q.GetWantByReleaseGroup(ctx, "rg-ram") + if w.SearchedAt.Valid { + t.Error("a refused want is not due for search") + } + entries, _ := os.ReadDir(s.library) + if len(entries) != 0 { + t.Errorf("the library got %d entries", len(entries)) + } +} + +func TestImport_ADoubtfulDownloadWaitsForAPersonThenImportsAnyway_Integration(t *testing.T) { + s := newStack(t, nil) // no AcoustID; untagged files: nothing but lengths + ctx := context.Background() + if rep := s.im.Pass(ctx); rep.Review != 1 { + t.Fatalf("pass = %+v", rep) + } + g := s.grabNow(t) + if g.ImportState != StateReview || g.EndedAt.Valid || !strings.Contains(g.ImportReason, "A person decides") { + t.Fatalf("grab = %s ended=%v: %s", g.ImportState, g.EndedAt.Valid, g.ImportReason) + } + if rep := s.im.Pass(ctx); rep.Review+rep.Imported != 0 { + t.Errorf("a held download was looked at again: %+v", rep) + } + if _, err := s.q.ForceGrabImport(ctx, g.ID); err != nil { + t.Fatal(err) + } + if rep := s.im.Pass(ctx); rep.Imported != 1 { + t.Fatalf("after import anyway: %+v", rep) + } + if g := s.grabNow(t); g.ImportState != StateImported || !strings.Contains(g.ImportReason, "anyway") { + t.Errorf("grab = %s: %s", g.ImportState, g.ImportReason) + } +} + +func TestImport_APersonRefusesADoubtfulDownload_Integration(t *testing.T) { + s := newStack(t, nil) + ctx := context.Background() + s.im.Pass(ctx) + g, err := s.im.Refuse(ctx, s.grab.ID, pgtype.UUID{}) + if err != nil { + t.Fatal(err) + } + if g.ImportState != StateRefused || !g.EndedAt.Valid || len(s.client.deleted) != 1 { + t.Errorf("grab = %s ended=%v deleted=%v", g.ImportState, g.EndedAt.Valid, s.client.deleted) + } + if _, err := s.im.Refuse(ctx, s.grab.ID, pgtype.UUID{}); err == nil { + t.Error("refused twice") + } +} + +func TestImport_AnUnfinishedDownloadIsOnlyWatched_Integration(t *testing.T) { + s := newStack(t, fakePrinter{}) + s.im.ClientFor = func(context.Context, string) (downloader.Driver, error) { return &halfClient{s.client}, nil } + if rep := s.im.Pass(context.Background()); rep.Downloading != 1 { + t.Fatalf("pass = %+v", rep) + } + g := s.grabNow(t) + if g.ImportState != StateDownloading || g.Progress < 0.49 || g.Progress > 0.51 { + t.Errorf("grab = %s %.2f", g.ImportState, g.Progress) + } +} + +type halfClient struct{ *fakeClient } + +func (c *halfClient) Torrents(ctx context.Context, h ...string) ([]downloader.Torrent, error) { + ts, _ := c.fakeClient.Torrents(ctx, h...) + ts[0].Progress, ts[0].State = 0.5, "downloading" + return ts, nil +} diff --git a/internal/importer/inspect.go b/internal/importer/inspect.go new file mode 100644 index 00000000..a2a5f863 --- /dev/null +++ b/internal/importer/inspect.go @@ -0,0 +1,195 @@ +package importer + +import ( + "context" + "errors" + "io/fs" + "os" + "path/filepath" + "sort" + "strings" + + "github.com/dhowden/tag" + "github.com/dhowden/tag/mbz" + + "git.fabledsword.com/bvandeusen/minstrel/internal/acoustid" + "git.fabledsword.com/bvandeusen/minstrel/internal/library" +) + +// Printer says which MusicBrainz recordings a file's audio is. ErrNoPrints: +// it cannot say for any file (AcoustID is not set up, or not answering), so +// the check rests on lengths and tags. +type Printer interface { + Recordings(ctx context.Context, path string) ([]string, error) +} + +// ErrNoPrints: fingerprints are not available for this download. +var ErrNoPrints = errors.New("importer: fingerprints are not available") + +// AcoustIDPrinter fingerprints with fpcalc and asks AcoustID, under the +// operator's settings (key, minimum score). Each call reads the settings, so +// a change takes effect on the next file. +type AcoustIDPrinter struct { + Settings interface { + Get() library.AcoustIDSettings + } + Client *acoustid.Client +} + +// Recordings is every recording AcoustID gives at or above the minimum score: +// one audio can be several recordings (the album track and the single). +func (p *AcoustIDPrinter) Recordings(ctx context.Context, path string) ([]string, error) { + s := p.Settings.Get() + if !s.Ready() || p.Client == nil { + return nil, ErrNoPrints + } + fp, dur, err := library.LookupFingerprint(ctx, path) + if err != nil { + if library.IsInconclusive(err) { + return nil, ErrNoPrints + } + return nil, nil // fpcalc rejected the file: no evidence either way + } + cands, err := p.Client.Lookup(ctx, s.APIKey, fp, dur) + switch { + case errors.Is(err, acoustid.ErrInvalidKey), errors.Is(err, acoustid.ErrUnavailable): + return nil, ErrNoPrints + case err != nil: + return nil, nil + } + var out []string + for _, c := range cands { + if c.Score >= float64(s.MinScore) && c.ID != "" { + out = append(out, c.ID) + } + } + return out, nil +} + +// formatRank picks which format's files are the album when a download holds +// more than one (a FLAC folder and an MP3 folder): the most files, then the +// better format. +var formatRank = map[string]int{".flac": 6, ".wav": 5, ".m4a": 4, ".opus": 3, ".ogg": 2, ".mp3": 1, ".aac": 0} + +// audioFiles lists the audio files under root (a folder, or one file), in +// the one format that is the album. Dot-named files and folders are skipped: +// they are partial downloads, OS litter or Minstrel's own temp files. +func audioFiles(root string) ([]string, error) { + info, err := os.Stat(root) + if err != nil { + return nil, err + } + if !info.IsDir() { + if library.IsAudioFile(root) { + return []string{root}, nil + } + return nil, nil + } + byExt := map[string][]string{} + err = filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + if strings.HasPrefix(d.Name(), ".") && path != root { + if d.IsDir() { + return filepath.SkipDir + } + return nil + } + if !d.IsDir() && library.IsAudioFile(path) { + ext := strings.ToLower(filepath.Ext(path)) + byExt[ext] = append(byExt[ext], path) + } + return nil + }) + if err != nil { + return nil, err + } + best := "" + for ext, paths := range byExt { + if best == "" || len(paths) > len(byExt[best]) || + (len(paths) == len(byExt[best]) && formatRank[ext] > formatRank[best]) { + best = ext + } + } + out := byExt[best] + sort.Strings(out) + return out, nil +} + +// readFile reads what one file's tags and length say. A file whose tags +// cannot be read still counts, by its length and name. +func readFile(ctx context.Context, root, path string) File { + f := File{Path: path, Rel: rel(root, path), Video: len(library.SourceMarkersFor(path)) > 0} + if ms, err := library.ProbeDurationMs(ctx, path); err == nil { + f.LengthMs = int(ms) + } + h, err := os.Open(path) + if err != nil { + return f + } + defer func() { _ = h.Close() }() + m, err := tag.ReadFrom(h) + if err != nil { + return f + } + f.Title, f.Artist, f.AlbumArtist, f.Album = m.Title(), m.Artist(), m.AlbumArtist(), m.Album() + f.Track, _ = m.Track() + f.Disc, _ = m.Disc() + info := mbz.Extract(m) + f.GroupMBID = clean(info.Get(mbz.ReleaseGroup)) + f.ReleaseMBID = clean(info.Get(mbz.Album)) + f.RecordingMBID = clean(info.Get(mbz.Recording)) + if f.RecordingMBID == "" && m.Format() == tag.VORBIS { + // Picard's Vorbis key for the recording is MUSICBRAINZ_TRACKID, which + // mbz files under its release-track name. + f.RecordingMBID = clean(info.Get(mbz.Track)) + } + return f +} + +func rel(root, path string) string { + if r, err := filepath.Rel(root, path); err == nil && r != "." { + return r + } + return filepath.Base(path) +} + +// clean takes the first of NUL-separated values: ID3v2.4 multi-values. +func clean(s string) string { + for _, part := range strings.Split(s, "\x00") { + if t := strings.TrimSpace(part); t != "" { + return t + } + } + return "" +} + +// inspect reads every audio file under root and asks the printer about each. +// fingerprinted reports whether the printer answered at all. +func inspect(ctx context.Context, root string, p Printer) (files []File, fingerprinted bool, err error) { + paths, err := audioFiles(root) + if err != nil { + return nil, false, err + } + fingerprinted = p != nil + for _, path := range paths { + if ctx.Err() != nil { + return nil, false, ctx.Err() + } + f := readFile(ctx, root, path) + if fingerprinted && len(paths) <= maxFiles { + recs, err := p.Recordings(ctx, path) + if errors.Is(err, ErrNoPrints) { + fingerprinted = false + for i := range files { + files[i].Recordings = nil + } + } else { + f.Recordings = recs + } + } + files = append(files, f) + } + return files, fingerprinted, nil +} diff --git a/internal/importer/name.go b/internal/importer/name.go new file mode 100644 index 00000000..c9342fef --- /dev/null +++ b/internal/importer/name.go @@ -0,0 +1,110 @@ +package importer + +import ( + "fmt" + "path/filepath" + "strings" + "unicode/utf8" +) + +// Placement is where one file goes and the names written into its tags. +type Placement struct { + File int // index into the files + Dest string // absolute + Title string + Disc int + Discs int + Track int + Tracks int // on its disc + // Recording is the MusicBrainz recording the file was matched to. + Recording string +} + +// Plan names every file's destination under root the way Lidarr named the +// library Minstrel took over (note #5527), so new arrivals sit beside what +// is there: +// +// {Artist}/{Album} ({Year})/{Artist} - {Album} - {track:00} - {Title}.ext +// +// with a "{Medium Format} {medium:00}" folder between when the edition has +// more than one medium. A file the check paired to a track takes the track's +// name and place; one it did not (an "import anyway" with more files than +// tracks) keeps its own tags. +func Plan(root string, w Want, e *Edition, files []File, pairs []Pair) []Placement { + album := w.Title + if w.Year > 0 { + album = fmt.Sprintf("%s (%d)", w.Title, w.Year) + } + dir := filepath.Join(root, Segment(w.Artist), Segment(album)) + media := 1 + if e != nil { + media = max(e.Media(), 1) + } + perDisc := map[int]int{} + if e != nil { + for _, t := range e.Tracks { + perDisc[t.Medium]++ + } + } + paired := map[int]Pair{} + for _, p := range pairs { + paired[p.File] = p + } + + out := make([]Placement, 0, len(files)) + for i, f := range files { + pl := Placement{File: i, Title: f.Title, Disc: max(f.Disc, 1), Track: f.Track, Discs: media} + format := "" + if p, ok := paired[i]; ok && e != nil { + t := e.Tracks[p.Track] + pl.Title, pl.Disc, pl.Track, pl.Recording, format = t.Title, max(t.Medium, 1), t.Position, t.RecordingMBID, t.Format + pl.Tracks = perDisc[t.Medium] + } + if pl.Title == "" { + pl.Title = strings.TrimSuffix(filepath.Base(f.Path), filepath.Ext(f.Path)) + } + d := dir + if media > 1 { + if format == "" { + format = "Disc" + } + d = filepath.Join(dir, Segment(fmt.Sprintf("%s %02d", format, pl.Disc))) + } + name := fmt.Sprintf("%s - %s - %02d - %s", w.Artist, w.Title, pl.Track, pl.Title) + pl.Dest = filepath.Join(d, Segment(name)+strings.ToLower(filepath.Ext(f.Path))) + out = append(out, pl) + } + return out +} + +// segmentMax keeps a folder or file name inside the 255 bytes most +// filesystems allow, with room for an extension and a temp prefix. +const segmentMax = 200 + +var illegal = strings.NewReplacer( + `\`, "+", "/", "+", "<", "", ">", "", "?", "!", "*", "-", "|", "", `"`, "", + // Lidarr's default "smart" colon: "Title: Subtitle" reads "Title - Subtitle". + ": ", " - ", ":", "-", +) + +// Segment makes a name safe as one path segment, the way Lidarr and Sonarr +// clean file names: their illegal-character mapping, no leading or trailing +// spaces or dots, at most segmentMax bytes, never "" or "..". +func Segment(s string) string { + s = illegal.Replace(s) + s = strings.Map(func(r rune) rune { + if r < 0x20 || r == 0x7f { + return -1 + } + return r + }, s) + s = strings.Trim(s, " .") + for len(s) > segmentMax { + _, size := utf8.DecodeLastRuneInString(s) + s = strings.TrimRight(s[:len(s)-size], " .") + } + if s == "" { + return "_" + } + return s +} diff --git a/internal/importer/place.go b/internal/importer/place.go new file mode 100644 index 00000000..5687ca4b --- /dev/null +++ b/internal/importer/place.go @@ -0,0 +1,182 @@ +package importer + +import ( + "context" + "crypto/rand" + "encoding/hex" + "errors" + "fmt" + "io" + "os" + "os/exec" + "path/filepath" + "strconv" + "strings" + "time" +) + +// ErrOccupied: the destination already has a file. Nothing is overwritten. +var ErrOccupied = errors.New("importer: destination already holds a file") + +// tagTimeout bounds one ffmpeg run. A remux copies the audio untouched, so a +// whole album track takes well under a second on local disk; two minutes is +// for a slow network mount, not for ffmpeg. +const tagTimeout = 2 * time.Minute + +// Tags are the names written into an imported file. Empty values are left +// out; the file's own tags stay for everything not named here. +type Tags struct { + Title, Artist, Album, AlbumArtist string + Year int + Disc, Discs, Track, Tracks int + // MusicBrainz ids, written the way Picard writes them. + ReleaseMBID, GroupMBID, RecordingMBID, ArtistMBID string +} + +// args turns the tags into ffmpeg -metadata pairs for a file of this +// extension. Vorbis comments (FLAC, Ogg, Opus) take Picard's key names; +// ID3 takes Picard's TXXX descriptions, which ffmpeg writes for any key it +// has no frame for. MP4's freeform atoms ffmpeg cannot write, so an M4A +// gets the plain names only. +func (t Tags) args(ext string) []string { + kv := [][2]string{ + {"title", t.Title}, {"artist", t.Artist}, {"album", t.Album}, {"album_artist", t.AlbumArtist}, + {"track", ofN(t.Track, t.Tracks)}, {"disc", ofN(t.Disc, t.Discs)}, + } + if t.Year > 0 { + kv = append(kv, [2]string{"date", strconv.Itoa(t.Year)}) + } + switch ext { + case ".flac", ".ogg", ".opus": + kv = append(kv, + [2]string{"MUSICBRAINZ_ALBUMID", t.ReleaseMBID}, + [2]string{"MUSICBRAINZ_RELEASEGROUPID", t.GroupMBID}, + [2]string{"MUSICBRAINZ_TRACKID", t.RecordingMBID}, + [2]string{"MUSICBRAINZ_ARTISTID", t.ArtistMBID}, + [2]string{"MUSICBRAINZ_ALBUMARTISTID", t.ArtistMBID}) + case ".mp3": + kv = append(kv, + [2]string{"MusicBrainz Album Id", t.ReleaseMBID}, + [2]string{"MusicBrainz Release Group Id", t.GroupMBID}, + [2]string{"MusicBrainz Track Id", t.RecordingMBID}, + [2]string{"MusicBrainz Artist Id", t.ArtistMBID}, + [2]string{"MusicBrainz Album Artist Id", t.ArtistMBID}) + } + var out []string + for _, p := range kv { + if p[1] != "" { + out = append(out, "-metadata", p[0]+"="+p[1]) + } + } + return out +} + +func ofN(n, of int) string { + switch { + case n <= 0: + return "" + case of > 0: + return fmt.Sprintf("%d/%d", n, of) + } + return strconv.Itoa(n) +} + +// taggable: the formats ffmpeg can rewrite tags in by remuxing. +var taggable = map[string]bool{".flac": true, ".ogg": true, ".opus": true, ".mp3": true, ".m4a": true} + +// Place copies src to dst with tags written, leaving src as it was (the +// download keeps seeding). The copy is written to a dot-named temp file +// beside dst, which the scanner skips, and renamed into place, so the +// library never sees half a file. Tags are written by an ffmpeg remux that +// copies the audio untouched; when ffmpeg cannot (it is missing, or the +// container will not take the file's streams) the file is copied as it is +// and tagged reports false. An existing dst is never overwritten. +func Place(ctx context.Context, src, dst string, t Tags) (tagged bool, err error) { + if _, err := os.Lstat(dst); err == nil { + return false, fmt.Errorf("%w: %s", ErrOccupied, dst) + } + dir := filepath.Dir(dst) + if err := os.MkdirAll(dir, 0o755); err != nil { + return false, fmt.Errorf("make %s: %w", dir, err) + } + ext := strings.ToLower(filepath.Ext(dst)) + tmp := filepath.Join(dir, ".minstrel-import-"+randomHex()+ext) + defer func() { _ = os.Remove(tmp) }() // gone already once renamed + + if taggable[ext] { + tagged = remux(ctx, src, tmp, t.args(ext)) == nil + } + if !tagged { + if err := copyFile(src, tmp); err != nil { + return false, err + } + } + if err := os.Chmod(tmp, 0o644); err != nil { + return false, fmt.Errorf("chmod %s: %w", tmp, err) + } + if _, err := os.Lstat(dst); err == nil { + return false, fmt.Errorf("%w: %s", ErrOccupied, dst) + } + if err := os.Rename(tmp, dst); err != nil { + return false, fmt.Errorf("move into place %s: %w", dst, err) + } + return tagged, nil +} + +// remux writes src to dst through ffmpeg with every stream copied as it is +// (the cover art too) and the tags set. Its output is checked against the +// source's size: a remux shifts container bytes, never the audio, so a file +// much smaller than its source lost something on the way. +func remux(ctx context.Context, src, dst string, meta []string) error { + ctx, cancel := context.WithTimeout(ctx, tagTimeout) + defer cancel() + args := append([]string{"-nostdin", "-v", "error", "-y", "-i", src, + "-map", "0", "-c", "copy", "-map_metadata", "0"}, meta...) + args = append(args, dst) + cmd := exec.CommandContext(ctx, "ffmpeg", args...) + if out, err := cmd.CombinedOutput(); err != nil { + _ = os.Remove(dst) + return fmt.Errorf("ffmpeg: %w: %s", err, strings.TrimSpace(string(out))) + } + si, err := os.Stat(src) + if err != nil { + return err + } + di, err := os.Stat(dst) + if err != nil { + return err + } + if di.Size() < si.Size()/2 { + _ = os.Remove(dst) + return fmt.Errorf("ffmpeg wrote %d bytes from %d", di.Size(), si.Size()) + } + return nil +} + +// copyFile copies src to dst byte for byte and syncs it. +func copyFile(src, dst string) error { + in, err := os.Open(src) + if err != nil { + return err + } + defer func() { _ = in.Close() }() + out, err := os.OpenFile(dst, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0o644) + if err != nil { + return err + } + if _, err := io.Copy(out, in); err != nil { + _ = out.Close() + return fmt.Errorf("copy %s: %w", src, err) + } + if err := out.Sync(); err != nil { + _ = out.Close() + return err + } + return out.Close() +} + +func randomHex() string { + b := make([]byte, 6) + _, _ = rand.Read(b) + return hex.EncodeToString(b) +} diff --git a/internal/importer/verify.go b/internal/importer/verify.go new file mode 100644 index 00000000..e5721fc2 --- /dev/null +++ b/internal/importer/verify.go @@ -0,0 +1,466 @@ +// Package importer checks a finished download against the album it was +// grabbed for and files it into the library (M502 #5522). Nothing reaches +// the library before the check says the files are the album: a wrong album, +// another version of it, a video rip or an incomplete copy is refused, its +// release is blocked and the album is searched for again. +// +// Ported from Fabled Librarian's importer and relocate packages, with the +// check rewritten for music: the files are matched to one of the album's +// MusicBrainz editions by AcoustID fingerprints, track lengths, titles and +// the tags they carry. +package importer + +import ( + "fmt" + "slices" + "sort" + "strings" + + "git.fabledsword.com/bvandeusen/minstrel/internal/release" +) + +// Outcome is what the check decided. +type Outcome string + +const ( + Accept Outcome = "accept" + // Review: the files may be the album and the evidence cannot say. A + // person decides. + Review Outcome = "review" + Refuse Outcome = "refuse" +) + +// Codes say why, beside the sentence in Verdict.Reason. +const ( + CodeMatched = "matched" + CodeForced = "forced" + CodeNoAudio = "no_audio" + CodeVideo = "video" + CodeWrongAlbum = "wrong_album" + CodeOtherVersion = release.VetoVersion + CodeWrongTitle = release.VetoTitle + CodeIncomplete = "incomplete" + CodeNoEdition = "no_edition" + CodeWeak = "weak" + CodeTooMany = "too_many" +) + +// maxFiles is the most audio files one album is checked for. More is a box +// set or a discography, which is a person's call, not a fingerprint queue. +const maxFiles = 60 + +// File is what one audio file in a download says about itself. +type File struct { + Path string // absolute, as Minstrel sees it + Rel string // under the download, for the person reading the verdict + // Disc and Track are the tags' positions; 0 when untagged. + Disc, Track int + Title string + Artist string + AlbumArtist string + Album string + // MusicBrainz ids from the tags, when a tagger wrote them. + GroupMBID string + ReleaseMBID string + RecordingMBID string + LengthMs int + // Recordings are the MusicBrainz recordings AcoustID says the audio is, + // at or above the operator's minimum score. + Recordings []string + // Video: the file name says it is a music video, a lyric video or the + // like (library.SourceMarkersFor). + Video bool +} + +// Track is one track of an edition. +type Track struct { + Medium int + Format string // the medium's: CD, Digital Media, Vinyl + Position int + Title string + RecordingMBID string + LengthMs int +} + +// Edition is one MusicBrainz release of the wanted release group, its +// tracks in medium and position order. +type Edition struct { + MBID string + Title string + Status string + Date string + Country string + Tracks []Track +} + +// Media is how many media the edition has. +func (e Edition) Media() int { + n := 0 + for _, t := range e.Tracks { + n = max(n, t.Medium) + } + return n +} + +// Want is the album the download was grabbed for. +type Want struct { + GroupMBID string + ArtistMBID string + Artist string + Title string + Year int + PrimaryType string + SecondaryTypes []string +} + +// Pair is a file matched to an edition's track, and what agreed. +type Pair struct { + File int `json:"file"` + Track int `json:"track"` + Recording bool `json:"recording"` + Length bool `json:"length"` + Title bool `json:"title"` +} + +// Evidence is what the check counted, kept with the verdict so a person can +// see what it rested on. +type Evidence struct { + Files int `json:"files"` + Tracks int `json:"tracks"` + // Identified: files with a recording from AcoustID or their tags. + Identified int `json:"identified"` + Recordings int `json:"recordings"` + Lengths int `json:"lengths"` + Titles int `json:"titles"` + // GroupTag: "same" when the tags name this release group, "other" when + // another, "" when they name none. + GroupTag string `json:"group_tag,omitempty"` + AlbumTag string `json:"album_tag,omitempty"` + // Fingerprinted: AcoustID was asked. False when it is not set up or + // could not be reached; the check then rests on lengths and tags. + Fingerprinted bool `json:"fingerprinted"` +} + +// Verdict is the check's answer. +type Verdict struct { + Outcome Outcome `json:"outcome"` + Code string `json:"code"` + Reason string `json:"reason"` + // Edition indexes the editions passed in; -1 when none was chosen. + Edition int `json:"-"` + Pairs []Pair `json:"pairs,omitempty"` + Evidence Evidence `json:"evidence"` +} + +// Verify decides whether files are the wanted album and, if so, which +// edition and which track each file is. force is a person's "import anyway": +// the best edition is taken whatever the evidence, but a download with no +// audio is still refused. +func Verify(w Want, editions []Edition, files []File, fingerprinted, force bool) Verdict { + v := Verdict{Edition: -1, Evidence: Evidence{Files: len(files), Fingerprinted: fingerprinted}} + if len(files) == 0 { + return v.set(Refuse, CodeNoAudio, "The download holds no audio files.") + } + if !force && len(files) > maxFiles { + return v.set(Review, CodeTooMany, fmt.Sprintf( + "The download holds %d audio files, more than one album; a person decides.", len(files))) + } + if video := countIf(files, func(f File) bool { return f.Video }); !force && video*2 > len(files) { + return v.set(Refuse, CodeVideo, fmt.Sprintf( + "%d of its %d audio files are named as music or lyric videos: the sound of a video, not the album.", video, len(files))) + } + + group := majority(files, func(f File) string { return f.GroupMBID }) + switch { + case group == "": + case strings.EqualFold(group, w.GroupMBID): + v.Evidence.GroupTag = "same" + default: + v.Evidence.GroupTag = "other" + } + album := majority(files, func(f File) string { return f.Album }) + v.Evidence.AlbumTag = album + var tagVeto release.Match + albumAgrees := false + if album != "" { + // The want's own artist goes in front: the tags' artist spelling is + // not what this asks. Only the album title and its version words. + m := release.Judge(release.Want{Artist: w.Artist, Title: w.Title, PrimaryType: w.PrimaryType, + SecondaryTypes: w.SecondaryTypes}, release.Parse(w.Artist+" - "+album)) + switch m.Veto { + case release.VetoVersion, release.VetoTitle: + tagVeto = m + case "": + albumAgrees = true + } + } + if !force && v.Evidence.GroupTag == "other" { + return v.set(Refuse, CodeWrongAlbum, fmt.Sprintf( + "Its tags name a different MusicBrainz release group (%s), not %s.", group, w.Title)) + } + + if len(editions) == 0 { + if force { + v.Code, v.Outcome, v.Reason = CodeForced, Accept, "Imported anyway at a person's word; MusicBrainz lists no edition to match it to." + return v + } + return v.set(Review, CodeNoEdition, fmt.Sprintf( + "MusicBrainz lists no edition of %s to check the files against; a person decides.", w.Title)) + } + + best, bestScore := -1, -1 + shortest, longest := len(editions[0].Tracks), 0 + for i, e := range editions { + shortest, longest = min(shortest, len(e.Tracks)), max(longest, len(e.Tracks)) + if len(e.Tracks) != len(files) && !force { + continue + } + pairs := pairUp(files, e.Tracks) + if s := score(pairs); s > bestScore || (s == bestScore && better(e, editions[best])) { + best, bestScore = i, s + } + } + if best < 0 { + if len(files) < shortest { + return v.set(Refuse, CodeIncomplete, fmt.Sprintf( + "It holds %d audio files; the shortest edition of %s has %d tracks.", len(files), w.Title, shortest)) + } + return v.set(Review, CodeNoEdition, fmt.Sprintf( + "It holds %d audio files; no edition of %s has %d tracks (they have %s). A person decides.", + len(files), w.Title, len(files), span(shortest, longest))) + } + + e := editions[best] + v.Edition = best + v.Pairs = pairUp(files, e.Tracks) + n := len(e.Tracks) + ev := &v.Evidence + ev.Tracks = n + for _, p := range v.Pairs { + f := files[p.File] + if f.RecordingMBID != "" || len(f.Recordings) > 0 { + ev.Identified++ + } + ev.Recordings += b2i(p.Recording) + ev.Lengths += b2i(p.Length) + ev.Titles += b2i(p.Title) + } + if force { + v.Outcome, v.Code = Accept, CodeForced + v.Reason = fmt.Sprintf("Imported anyway at a person's word, as %s. %s", describe(e), ev.sentence()) + return v + } + + half := (n + 1) / 2 + strong := ev.GroupTag == "same" || (ev.Identified >= half && ev.Recordings*10 >= ev.Identified*7) + switch { + case ev.Identified >= half && ev.Recordings*10 < ev.Identified*3 && ev.GroupTag != "same": + return v.set(Refuse, CodeWrongAlbum, fmt.Sprintf( + "Only %d of the %d tracks AcoustID knows are recordings on %s.", ev.Recordings, ev.Identified, w.Title)) + case tagVeto.Veto != "" && !strong: + return v.set(Refuse, tagVeto.Veto, fmt.Sprintf("Its tags call it %q. %s", album, tagVeto.Reason)) + case !strong && ev.Identified == 0 && ev.Lengths*2 < n: + return v.set(Refuse, CodeWrongAlbum, fmt.Sprintf( + "Only %d of its %d tracks are the length of %s's, and nothing else says it is the album.", ev.Lengths, n, w.Title)) + case strong && ev.Lengths*10 >= n*6, + !strong && ev.Lengths*10 >= n*8 && (ev.Titles*10 >= n*6 || albumAgrees): + v.Outcome, v.Code = Accept, CodeMatched + v.Reason = fmt.Sprintf("It is %s. %s", describe(e), ev.sentence()) + return v + } + return v.set(Review, CodeWeak, fmt.Sprintf( + "It may be %s, but the evidence is thin: %s A person decides.", describe(e), ev.sentence())) +} + +func (v Verdict) set(o Outcome, code, reason string) Verdict { + v.Outcome, v.Code, v.Reason = o, code, reason + return v +} + +// sentence says what agreed, for the verdict's reason. +func (ev Evidence) sentence() string { + var parts []string + if ev.GroupTag == "same" { + parts = append(parts, "its tags name this album") + } + if ev.Identified > 0 { + parts = append(parts, fmt.Sprintf("%d of %d identified tracks are its recordings", ev.Recordings, ev.Identified)) + } + parts = append(parts, fmt.Sprintf("%d of %d lengths fit", ev.Lengths, ev.Tracks), + fmt.Sprintf("%d of %d titles agree", ev.Titles, ev.Tracks)) + s := strings.Join(parts, ", ") + s = strings.ToUpper(s[:1]) + s[1:] + "." + if !ev.Fingerprinted { + s += " AcoustID was not asked (not set up or unreachable), so lengths and tags decided." + } + return s +} + +// describe names an edition for a person: "the 2013 CD release (13 tracks)". +func describe(e Edition) string { + var b strings.Builder + b.WriteString("the ") + if len(e.Date) >= 4 { + b.WriteString(e.Date[:4] + " ") + } + if f := e.format(); f != "" { + b.WriteString(f + " ") + } + fmt.Fprintf(&b, "edition (%d tracks)", len(e.Tracks)) + return b.String() +} + +func (e Edition) format() string { + if len(e.Tracks) == 0 { + return "" + } + f := e.Tracks[0].Format + for _, t := range e.Tracks { + if t.Format != f { + return "" + } + } + return f +} + +// pairUp matches files to tracks in order: by the tags' disc and track +// numbers when every file has them and no two share a place, by file path +// otherwise. With a person's "import anyway" the counts may differ; pairs +// run to the shorter. +func pairUp(files []File, tracks []Track) []Pair { + order := make([]int, len(files)) + for i := range order { + order[i] = i + } + if numbered(files) { + sort.SliceStable(order, func(a, b int) bool { + fa, fb := files[order[a]], files[order[b]] + if da, db := max(fa.Disc, 1), max(fb.Disc, 1); da != db { + return da < db + } + return fa.Track < fb.Track + }) + } else { + sort.SliceStable(order, func(a, b int) bool { return files[order[a]].Rel < files[order[b]].Rel }) + } + n := min(len(files), len(tracks)) + pairs := make([]Pair, n) + for i := range n { + f, t := files[order[i]], tracks[i] + pairs[i] = Pair{ + File: order[i], + Track: i, + Recording: t.RecordingMBID != "" && (f.RecordingMBID == t.RecordingMBID || slices.Contains(f.Recordings, t.RecordingMBID)), + Length: lengthFits(f.LengthMs, t.LengthMs), + Title: titleAgrees(f.Title, t.Title), + } + } + return pairs +} + +// numbered: every file carries a track number and no two share a disc and +// track. +func numbered(files []File) bool { + seen := map[[2]int]bool{} + for _, f := range files { + k := [2]int{max(f.Disc, 1), f.Track} + if f.Track <= 0 || seen[k] { + return false + } + seen[k] = true + } + return true +} + +// lengthFits allows five seconds or 3%, whichever is more: remasters and +// rips differ by a second or two, a different recording by far more. A +// track MusicBrainz has no length for says nothing against the file. +func lengthFits(fileMs, trackMs int) bool { + if trackMs <= 0 { + return true + } + if fileMs <= 0 { + return false + } + tol := max(5000, trackMs*3/100) + d := fileMs - trackMs + return d <= tol && -d <= tol +} + +// titleAgrees: the same words, or one title holding all of the other's +// ("Get Lucky (Radio Edit)" holds "Get Lucky"). +func titleAgrees(a, b string) bool { + ta, tb := release.Tokens(a), release.Tokens(b) + if len(ta) == 0 || len(tb) == 0 { + return false + } + return subset(ta, tb) || subset(tb, ta) +} + +func subset(small, big []string) bool { + for _, w := range small { + if !slices.Contains(big, w) { + return false + } + } + return true +} + +// score weighs recordings above lengths above titles: a fingerprint is the +// audio itself, a length is the audio's shape, a title is only a name. +func score(pairs []Pair) int { + s := 0 + for _, p := range pairs { + s += 3*b2i(p.Recording) + 2*b2i(p.Length) + b2i(p.Title) + } + return s +} + +// better breaks a tie between equally good editions: an official one, then +// the earlier. +func better(a, b Edition) bool { + if (a.Status == "Official") != (b.Status == "Official") { + return a.Status == "Official" + } + return a.Date != "" && (b.Date == "" || a.Date < b.Date) +} + +func majority(files []File, get func(File) string) string { + counts := map[string]int{} + best, n := "", 0 + for _, f := range files { + v := strings.TrimSpace(get(f)) + if v == "" { + continue + } + counts[strings.ToLower(v)]++ + if c := counts[strings.ToLower(v)]; c > n { + best, n = v, c + } + } + return best +} + +func countIf(files []File, f func(File) bool) int { + n := 0 + for _, x := range files { + if f(x) { + n++ + } + } + return n +} + +func span(a, b int) string { + if a == b { + return fmt.Sprint(a) + } + return fmt.Sprintf("%d to %d", a, b) +} + +func b2i(b bool) int { + if b { + return 1 + } + return 0 +} diff --git a/internal/importer/verify_test.go b/internal/importer/verify_test.go new file mode 100644 index 00000000..6bf8425c --- /dev/null +++ b/internal/importer/verify_test.go @@ -0,0 +1,243 @@ +package importer + +import ( + "fmt" + "path/filepath" + "strings" + "testing" +) + +var ram = Want{GroupMBID: "rg-ram", ArtistMBID: "mb-dp", Artist: "Daft Punk", Title: "Random Access Memories", + Year: 2013, PrimaryType: "Album"} + +var ramTitles = []string{"Give Life Back to Music", "The Game of Love", "Giorgio by Moroder", "Within", + "Instant Crush", "Lose Yourself to Dance", "Touch", "Get Lucky", "Beyond", "Motherboard", + "Fragments of Time", "Doin' It Right", "Contact"} + +// edition is RAM's 13-track CD, recordings rec-01.., lengths 4:00 + n seconds. +func edition(mbid string, n int) Edition { + e := Edition{MBID: mbid, Title: "Random Access Memories", Status: "Official", Date: "2013-05-17"} + for i := range n { + e.Tracks = append(e.Tracks, Track{Medium: 1, Format: "CD", Position: i + 1, Title: ramTitles[i%len(ramTitles)], + RecordingMBID: fmt.Sprintf("rec-%02d", i+1), LengthMs: 240000 + i*1000}) + } + return e +} + +// download is n files lined up with the edition: tagged positions, lengths +// within a second, AcoustID naming each file's recording. +func download(n int) []File { + var fs []File + for i := range n { + fs = append(fs, File{Path: fmt.Sprintf("/dl/%02d.flac", i+1), Rel: fmt.Sprintf("%02d.flac", i+1), + Disc: 1, Track: i + 1, Title: ramTitles[i%len(ramTitles)], Album: "Random Access Memories", + LengthMs: 240000 + i*1000 + 400, Recordings: []string{fmt.Sprintf("rec-%02d", i+1)}}) + } + return fs +} + +func TestVerify_TheAlbumAndWhatIsNot(t *testing.T) { + eds := []Edition{edition("rel-cd", 13), edition("rel-deluxe", 15)} + cases := []struct { + name string + files func() []File + printed bool + out Outcome + code string + edition string + }{ + {"the album, fingerprinted", func() []File { return download(13) }, true, Accept, CodeMatched, "rel-cd"}, + {"the deluxe by count", func() []File { return download(15) }, true, Accept, CodeMatched, "rel-deluxe"}, + {"no audio", func() []File { return nil }, true, Refuse, CodeNoAudio, ""}, + {"music videos", func() []File { + fs := download(13) + for i := range fs { + fs[i].Video = i < 10 + } + return fs + }, true, Refuse, CodeVideo, ""}, + {"tags name another group", func() []File { + fs := download(13) + for i := range fs { + fs[i].GroupMBID = "rg-homework" + } + return fs + }, true, Refuse, CodeWrongAlbum, ""}, + {"fingerprints say other recordings", func() []File { + fs := download(13) + for i := range fs { + fs[i].Recordings = []string{"rec-other"} + } + return fs + }, true, Refuse, CodeWrongAlbum, "rel-cd"}, + {"the drumless edition: right lengths, album tag says otherwise", func() []File { + fs := download(13) + for i := range fs { + fs[i].Recordings, fs[i].Album = nil, "Random Access Memories (Drumless Edition)" + } + return fs + }, false, Refuse, CodeOtherVersion, "rel-cd"}, + {"fewer files than any edition", func() []File { return download(9) }, true, Refuse, CodeIncomplete, ""}, + {"no edition this long", func() []File { return download(14) }, true, Review, CodeNoEdition, ""}, + {"no fingerprints, lengths and titles agree", func() []File { + fs := download(13) + for i := range fs { + fs[i].Recordings = nil + } + return fs + }, false, Accept, CodeMatched, "rel-cd"}, + {"no fingerprints, no titles, nothing to go on but lengths", func() []File { + fs := download(13) + for i := range fs { + fs[i].Recordings, fs[i].Title, fs[i].Album = nil, "", "" + } + return fs + }, false, Review, CodeWeak, "rel-cd"}, + {"no fingerprints, lengths all wrong", func() []File { + fs := download(13) + for i := range fs { + fs[i].Recordings, fs[i].LengthMs = nil, 600000 + } + return fs + }, false, Refuse, CodeWrongAlbum, "rel-cd"}, + {"tags name the group: strong even unprinted", func() []File { + fs := download(13) + for i := range fs { + fs[i].Recordings, fs[i].Title, fs[i].GroupMBID = nil, "", "rg-ram" + } + return fs + }, false, Accept, CodeMatched, "rel-cd"}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + v := Verify(ram, eds, c.files(), c.printed, false) + if v.Outcome != c.out || v.Code != c.code { + t.Fatalf("got %s/%s (%s), want %s/%s", v.Outcome, v.Code, v.Reason, c.out, c.code) + } + got := "" + if v.Edition >= 0 { + got = eds[v.Edition].MBID + } + if got != c.edition { + t.Errorf("edition %q, want %q", got, c.edition) + } + if v.Reason == "" { + t.Error("no reason given") + } + }) + } +} + +func TestVerify_UnnumberedFilesPairByPath(t *testing.T) { + fs := download(13) + for i := range fs { + fs[i].Track, fs[i].Disc = 0, 0 + } + fs[0], fs[5] = fs[5], fs[0] // listed out of order; their names sort right + v := Verify(ram, []Edition{edition("rel-cd", 13)}, fs, true, false) + if v.Outcome != Accept || v.Evidence.Recordings != 13 { + t.Fatalf("got %+v", v) + } +} + +func TestVerify_ImportAnywayTakesTheBestEdition(t *testing.T) { + v := Verify(ram, []Edition{edition("rel-cd", 13)}, download(14), true, true) + if v.Outcome != Accept || v.Code != CodeForced || v.Edition != 0 || len(v.Pairs) != 13 { + t.Fatalf("got %+v", v) + } + if v := Verify(ram, nil, nil, true, true); v.Outcome != Refuse { + t.Errorf("forced with no audio: %+v", v) + } +} + +func TestVerify_WithoutAcoustIDTheReasonSaysSo(t *testing.T) { + fs := download(13) + for i := range fs { + fs[i].Recordings = nil + } + v := Verify(ram, []Edition{edition("rel-cd", 13)}, fs, false, false) + if !strings.Contains(v.Reason, "AcoustID was not asked") { + t.Errorf("reason %q", v.Reason) + } +} + +func TestLengthFits(t *testing.T) { + for _, c := range []struct { + file, track int + ok bool + }{ + {240000, 244000, true}, {240000, 246000, false}, {600000, 615000, true}, {600000, 620000, false}, + {0, 240000, false}, {240000, 0, true}, + } { + if got := lengthFits(c.file, c.track); got != c.ok { + t.Errorf("lengthFits(%d, %d) = %v", c.file, c.track, got) + } + } +} + +func TestPlan_LidarrsNaming(t *testing.T) { + e := edition("rel-cd", 2) + e.Tracks[1].Title = "Within: Part 2/3?" + pl := Plan("/music", ram, &e, download(2), Verify(ram, []Edition{e}, download(2), true, false).Pairs) + want := []string{ + "/music/Daft Punk/Random Access Memories (2013)/Daft Punk - Random Access Memories - 01 - Give Life Back to Music.flac", + "/music/Daft Punk/Random Access Memories (2013)/Daft Punk - Random Access Memories - 02 - Within - Part 2+3!.flac", + } + for i, p := range pl { + if p.Dest != want[i] { + t.Errorf("dest %d = %q, want %q", i, p.Dest, want[i]) + } + } + if pl[0].Recording != "rec-01" || pl[0].Tracks != 2 { + t.Errorf("placement %+v", pl[0]) + } +} + +func TestPlan_MultiDiscGetsAMediumFolder(t *testing.T) { + e := edition("rel-2cd", 4) + e.Tracks[2].Medium, e.Tracks[2].Position = 2, 1 + e.Tracks[3].Medium, e.Tracks[3].Position = 2, 2 + fs := download(4) + fs[2].Disc, fs[2].Track, fs[3].Disc, fs[3].Track = 2, 1, 2, 2 + pl := Plan("/music", ram, &e, fs, Verify(ram, []Edition{e}, fs, true, false).Pairs) + if got := filepath.Dir(pl[3].Dest); got != "/music/Daft Punk/Random Access Memories (2013)/CD 02" { + t.Errorf("disc folder %q", got) + } + if !strings.HasSuffix(pl[3].Dest, " - 02 - Get Lucky.flac") && !strings.Contains(pl[3].Dest, " - 02 - ") { + t.Errorf("dest %q", pl[3].Dest) + } +} + +func TestSegment(t *testing.T) { + for in, want := range map[string]string{ + "AC/DC": "AC+DC", `What?`: "What!", "Title: Sub": "Title - Sub", "a:b": "a-b", " ..x.. ": "x", + "": "_", "..": "_", `"Quoted" *y* |z|`: "Quoted x -y- z", + } { + if got := Segment(in); got != want { + t.Errorf("Segment(%q) = %q, want %q", in, got, want) + } + } + if long := Segment(strings.Repeat("é", 200)); len(long) > segmentMax { + t.Errorf("long segment is %d bytes", len(long)) + } +} + +func TestTagsArgs(t *testing.T) { + tg := Tags{Title: "Get Lucky", Artist: "Daft Punk", Track: 8, Tracks: 13, Disc: 1, Discs: 1, Year: 2013, + RecordingMBID: "rec-08", GroupMBID: "rg-ram"} + flac := strings.Join(tg.args(".flac"), " ") + for _, s := range []string{"title=Get Lucky", "track=8/13", "date=2013", "MUSICBRAINZ_TRACKID=rec-08", "MUSICBRAINZ_RELEASEGROUPID=rg-ram"} { + if !strings.Contains(flac, s) { + t.Errorf("flac args lack %q: %s", s, flac) + } + } + if strings.Contains(flac, "MUSICBRAINZ_ALBUMID") { + t.Errorf("an empty release id was written: %s", flac) + } + if mp3 := strings.Join(tg.args(".mp3"), " "); !strings.Contains(mp3, "MusicBrainz Track Id=rec-08") { + t.Errorf("mp3 args %s", mp3) + } + if m4a := strings.Join(tg.args(".m4a"), " "); strings.Contains(m4a, "MusicBrainz") { + t.Errorf("m4a args %s", m4a) + } +} diff --git a/internal/library/fingerprint.go b/internal/library/fingerprint.go index 12890a78..8b69449a 100644 --- a/internal/library/fingerprint.go +++ b/internal/library/fingerprint.go @@ -422,3 +422,16 @@ func notURLBase64(r rune) bool { } return true } + +// LookupFingerprint is the compressed fingerprint AcoustID's lookup takes and +// the file's whole length in seconds, for a file not in the library yet: the +// import checks a download with it (M502 #5522). IsInconclusive tells a +// missing or stalled fpcalc from a file it rejected. +func LookupFingerprint(ctx context.Context, path string) (fingerprint string, durationSec int, err error) { + fp, err := computeLookupFingerprint(ctx, path) + return fp.fingerprint, fp.durationSec, err +} + +// IsInconclusive reports that a fingerprint tool could not give an answer +// (missing, timed out), as opposed to rejecting the file. +func IsInconclusive(err error) bool { return isInconclusive(err) } diff --git a/internal/library/scanner.go b/internal/library/scanner.go index 4fcd73c9..2c986caa 100644 --- a/internal/library/scanner.go +++ b/internal/library/scanner.go @@ -674,3 +674,16 @@ func probeDurationMs(ctx context.Context, path string) (int32, error) { } return int32(ms), nil } + +// ProbeDurationMs is a file's length in milliseconds, read by ffprobe. +func ProbeDurationMs(ctx context.Context, path string) (int32, error) { + return probeDurationMs(ctx, path) +} + +// IsAudioFile reports whether the scanner reads a file with this name. +func IsAudioFile(path string) bool { + return audioExtensions[strings.ToLower(filepath.Ext(path))] +} + +// Roots are the folders the scanner reads: the library. +func (s *Scanner) Roots() []string { return append([]string(nil), s.paths...) } diff --git a/internal/wanted/search.go b/internal/wanted/search.go index e13b6e8a..c8aa7885 100644 --- a/internal/wanted/search.go +++ b/internal/wanted/search.go @@ -180,6 +180,7 @@ func (s *Searcher) pass(ctx context.Context) Report { rep.Nothing++ default: rep.Failed++ + s.logger.Warn("wanted: search failed", "want", want.Artist+" - "+want.Title, "err", out.Error) } } return rep diff --git a/web/src/lib/api/queries.ts b/web/src/lib/api/queries.ts index 8abd41d4..2877ca37 100644 --- a/web/src/lib/api/queries.ts +++ b/web/src/lib/api/queries.ts @@ -48,6 +48,9 @@ export const qk = { // Search and grab (M502 #5521). wantCandidates: (wantId: string) => ['wantCandidates', wantId] as const, searchSettings: () => ['searchSettings'] as const, + // The Wanted view's Activity and Blocked tabs (#5522). + downloads: () => ['adminWants', 'downloads'] as const, + releaseBlocks: () => ['adminWants', 'releaseBlocks'] as const, myQuarantine: () => ['myQuarantine'] as const, adminQuarantine: () => ['adminQuarantine'] as const, adminQuarantineActions: (limit?: number) => diff --git a/web/src/lib/api/types.ts b/web/src/lib/api/types.ts index 783a5985..c0cc5728 100644 --- a/web/src/lib/api/types.ts +++ b/web/src/lib/api/types.ts @@ -606,11 +606,32 @@ export type WantSearch = { error: string; }; +// What an import made of a finished download (M502 #5522). +export type ImportState = + | '' + | 'downloading' + | 'waiting' + | 'importing' + | 'imported' + | 'refused' + | 'review' + | 'held' + | 'failed'; + +// A want's latest download, running or ended. export type WantGrab = { + id: string; name: string; grabbed_at: string; origin: 'schedule' | 'operator'; quality: string; + live: boolean; + end_reason: string; + progress: number; + import_state: ImportState; + import_reason: string; + imported_at: string | null; + imported_files: number; }; // Search and grab (M502 #5521) ---------------------------------------------- @@ -685,9 +706,77 @@ export type SearchSettings = { tiers: QualityTier[]; last_pass_at: string | null; last_pass: SearchPass; + // The library folder new music is filed into ("" = the first of + // scan_paths), and the library's folders (#5522). + import_root: string; + scan_paths: string[]; }; -export type WantCounts = { wanted: number; missing: number; held: number; following: number }; +export type WantCounts = { + wanted: number; + missing: number; + held: number; + following: number; + // Downloads running, those of them waiting on a person, releases set aside. + activity: number; + attention: number; + blocked: number; +}; + +// What an import's check counted (#5522). +export type ImportEvidence = { + files: number; + tracks: number; + identified: number; + recordings: number; + lengths: number; + titles: number; + group_tag?: string; + album_tag?: string; + fingerprinted: boolean; +}; + +// One download on the Activity tab. +export type Download = { + id: string; + want_id: string | null; + want_title: string; + want_artist: string; + name: string; + indexer: string; + client: string; + quality: string; + origin: 'schedule' | 'operator'; + grabbed_at: string; + ended_at: string | null; + end_reason: string; + progress: number; + import_state: ImportState; + import_reason: string; + imported_at: string | null; + imported_files: number; + destination: string; + edition?: string; + evidence?: ImportEvidence; + files: string[]; + skipped: number; + rehomed_from?: string; +}; + +export type DownloadsPage = { items: Download[]; limit: number; offset: number }; + +// A release set aside: by a person, by its files, or by an import's check. +export type ReleaseBlock = { + info_hash: string; + name: string; + reason: 'operator' | 'files' | 'import'; + detail: string; + blocked_at: string; + want_title: string; + want_artist: string; +}; + +export type ReleaseBlocksPage = { items: ReleaseBlock[]; limit: number; offset: number }; export type WantsPage = Page & { counts: WantCounts }; diff --git a/web/src/lib/api/wanted.ts b/web/src/lib/api/wanted.ts index 4d201d45..f09aa9aa 100644 --- a/web/src/lib/api/wanted.ts +++ b/web/src/lib/api/wanted.ts @@ -3,7 +3,9 @@ import { api } from './client'; import { qk } from './queries'; import type { Candidate, + DownloadsPage, Follow, + ReleaseBlocksPage, FollowState, ReleaseProfile, ReleaseProfileInput, @@ -157,6 +159,56 @@ export async function putSearchSettings(body: { auto_grab: boolean; search_interval_hours: number; allowed_qualities: string[]; + import_root: string; }): Promise { return api.put('/api/admin/acquisition/settings', body); } + +// Downloads and their imports (M502 #5522) ---------------------------------- + +// A page holding fewer than it asked for is the last. +function nextOffset(last: { items: unknown[]; limit: number; offset: number }): number | undefined { + return last.items.length < last.limit ? undefined : last.offset + last.items.length; +} + +export async function listDownloads(offset: number = 0): Promise { + return api.get(`/api/admin/downloads?offset=${offset}`); +} + +// Running downloads first; refreshed every few seconds while one runs. +export function createDownloadsQuery() { + return createInfiniteQuery({ + queryKey: qk.downloads(), + queryFn: ({ pageParam }) => listDownloads(pageParam as number), + initialPageParam: 0, + getNextPageParam: nextOffset, + refetchInterval: (q) => + q.state.data?.pages[0]?.items.some((d) => !d.ended_at) ? 5_000 : false + }); +} + +export async function importAnyway(id: string): Promise { + await api.post(`/api/admin/downloads/${id}/import`, {}); +} + +export async function retryImport(id: string): Promise { + await api.post(`/api/admin/downloads/${id}/retry`, {}); +} + +export async function refuseDownload(id: string): Promise { + await api.post(`/api/admin/downloads/${id}/refuse`, {}); +} + +export async function listReleaseBlocks(offset: number = 0): Promise { + return api.get(`/api/admin/release-blocks?offset=${offset}`); +} + +export function createReleaseBlocksQuery() { + return createInfiniteQuery({ + queryKey: qk.releaseBlocks(), + queryFn: ({ pageParam }) => listReleaseBlocks(pageParam as number), + initialPageParam: 0, + getNextPageParam: nextOffset, + staleTime: 30_000 + }); +} diff --git a/web/src/lib/components/AdminTabs.svelte b/web/src/lib/components/AdminTabs.svelte index 8c51bc8e..e75b3e54 100644 --- a/web/src/lib/components/AdminTabs.svelte +++ b/web/src/lib/components/AdminTabs.svelte @@ -7,7 +7,6 @@ { href: '/admin', label: 'Overview' }, { href: '/admin/integrations', label: 'Integrations' }, { href: '/admin/requests', label: 'Requests' }, - { href: '/admin/wanted', label: 'Wanted' }, { href: '/admin/quarantine', label: 'Quarantine' }, { href: '/admin/missing-files', label: 'Missing files' }, { href: '/admin/duplicates', label: 'Duplicates' }, diff --git a/web/src/lib/components/AdminTabs.test.ts b/web/src/lib/components/AdminTabs.test.ts index 8f180ea5..205a257a 100644 --- a/web/src/lib/components/AdminTabs.test.ts +++ b/web/src/lib/components/AdminTabs.test.ts @@ -52,7 +52,7 @@ describe('AdminTabs', () => { ); }); - test('renders all twelve tabs in order', () => { + test('renders all eleven tabs in order', () => { state.pageUrl = new URL('http://localhost/admin'); render(AdminTabs); const links = screen.getAllByRole('link'); @@ -60,8 +60,6 @@ describe('AdminTabs', () => { 'Overview', 'Integrations', 'Requests', - // Wanted follows Requests: an approved request goes on the wanted list. - 'Wanted', 'Quarantine', // Missing files sits with Quarantine and Playback errors: the three // surfaces that show tracks needing an operator's attention. diff --git a/web/src/lib/components/ReacquisitionSettingsCard.svelte b/web/src/lib/components/ReacquisitionSettingsCard.svelte index 3c210dbc..490b7664 100644 --- a/web/src/lib/components/ReacquisitionSettingsCard.svelte +++ b/web/src/lib/components/ReacquisitionSettingsCard.svelte @@ -74,7 +74,7 @@

Automatic re-acquisition

When a file has been missing for a while, Minstrel can put its album back on the - wanted list + wanted list by itself. It's wanted per album, not per track: a whole folder going missing is one want, not forty.

diff --git a/web/src/lib/components/ReleaseProfilesCard.svelte b/web/src/lib/components/ReleaseProfilesCard.svelte index 47af14e7..58a3e292 100644 --- a/web/src/lib/components/ReleaseProfilesCard.svelte +++ b/web/src/lib/components/ReleaseProfilesCard.svelte @@ -167,8 +167,7 @@

Release-type profiles

Each followed artist is on one profile, which says which of their new releases are - wanted. Requests for a - specific album aren't filtered. + wanted. Requests for a specific album aren't filtered.

diff --git a/web/src/lib/components/SearchSettingsCard.svelte b/web/src/lib/components/SearchSettingsCard.svelte index fb97fdcf..f7831722 100644 --- a/web/src/lib/components/SearchSettingsCard.svelte +++ b/web/src/lib/components/SearchSettingsCard.svelte @@ -10,13 +10,14 @@ // Searching and grabbing (M502 #5521): how often each want is searched // for, whether the best release is taken without asking, and which // qualities are taken at all. The schedule wakes every ten minutes and - // keeps to each indexer's searches-per-hour. + // keeps to each indexer's searches-per-hour. A finished download is + // checked and filed into one of the library's folders (#5522). const client = useQueryClient(); const store = createSearchSettingsQuery(); const query = $derived($store); - type Form = { auto_grab: boolean; search_interval_hours: number; allowed: string[] }; + type Form = { auto_grab: boolean; search_interval_hours: number; allowed: string[]; import_root: string }; // The form is the server's settings until edited; saved holds what a save // returned until the query catches up. @@ -27,7 +28,12 @@ const form = $derived
(edited ?? (base ? fromSettings(base) : null)); function fromSettings(s: SearchSettings): Form { - return { auto_grab: s.auto_grab, search_interval_hours: s.search_interval_hours, allowed: [...s.allowed_qualities] }; + return { + auto_grab: s.auto_grab, + search_interval_hours: s.search_interval_hours, + allowed: [...s.allowed_qualities], + import_root: s.import_root + }; } const dirty = $derived(!!edited && !!base && JSON.stringify(edited) !== JSON.stringify(fromSettings(base))); @@ -48,7 +54,8 @@ saved = await putSearchSettings({ auto_grab: form.auto_grab, search_interval_hours: Number(form.search_interval_hours), - allowed_qualities: form.allowed + allowed_qualities: form.allowed, + import_root: form.import_root }); edited = null; await client.invalidateQueries({ queryKey: qk.searchSettings() }); @@ -81,7 +88,7 @@

Searching and grabbing

Minstrel searches your indexers for each album on the - wanted list, judges every + wanted list, judges every release it finds, and hands the best one to your download client. Each album's releases and the reason for every verdict are on the Wanted page.

@@ -146,6 +153,32 @@ + {#if query.data.scan_paths.length > 1} + + {:else if query.data.scan_paths.length === 1} +

+ A download that checks out is copied into {query.data.scan_paths[0]} + as Artist/Album (Year)/Artist - Album - NN - Title, tagged with its MusicBrainz ids. The + download itself keeps seeding. +

+ {/if} +

{lastPass(query.data)}

diff --git a/web/src/lib/components/SearchSettingsCard.test.ts b/web/src/lib/components/SearchSettingsCard.test.ts index c5b919be..b93a0c86 100644 --- a/web/src/lib/components/SearchSettingsCard.test.ts +++ b/web/src/lib/components/SearchSettingsCard.test.ts @@ -22,7 +22,9 @@ const settings: SearchSettings = { { tier: 'unknown', label: 'Unknown', what: 'Nothing says what format it is' } ], last_pass_at: new Date(Date.now() - 5 * 60_000).toISOString(), - last_pass: { looked: 12, grabbed: 3, waiting: 2, nothing: 7, failed: 0, paced: true } + last_pass: { looked: 12, grabbed: 3, waiting: 2, nothing: 7, failed: 0, paced: true }, + import_root: '', + scan_paths: ['/music', '/music2'] }; afterEach(() => vi.clearAllMocks()); @@ -57,11 +59,29 @@ describe('SearchSettingsCard', () => { expect(putSearchSettings).toHaveBeenCalledWith({ auto_grab: false, search_interval_hours: 24, - allowed_qualities: ['lossless', 'lossy_high', 'unknown'] + allowed_qualities: ['lossless', 'lossy_high', 'unknown'], + import_root: '' }) ); }); + test('new music is filed into the chosen library folder, the first by default', async () => { + setup(); + const folder = screen.getByLabelText(/file new music into/i) as HTMLSelectElement; + expect(folder.value).toBe('/music'); + await fireEvent.change(folder, { target: { value: '/music2' } }); + await fireEvent.click(screen.getByRole('button', { name: /^save$/i })); + await waitFor(() => + expect(putSearchSettings).toHaveBeenCalledWith(expect.objectContaining({ import_root: '/music2' })) + ); + }); + + test('with one library folder there is nothing to choose', () => { + setup({ ...settings, scan_paths: ['/music'] }); + expect(screen.queryByLabelText(/file new music into/i)).toBeNull(); + expect(screen.getByText('/music')).toBeTruthy(); + }); + test('before the first pass it says so', () => { setup({ ...settings, last_pass_at: null, last_pass: {} }); expect(screen.getByTestId('last-pass').textContent).toMatch(/hasn’t run yet/); diff --git a/web/src/lib/components/Shell.svelte b/web/src/lib/components/Shell.svelte index f534c562..5caf9946 100644 --- a/web/src/lib/components/Shell.svelte +++ b/web/src/lib/components/Shell.svelte @@ -1,7 +1,7 @@ e.key === 'Escape' && (menuOpen = false)} /> diff --git a/web/src/lib/components/Shell.test.ts b/web/src/lib/components/Shell.test.ts index f7b658df..92a63aab 100644 --- a/web/src/lib/components/Shell.test.ts +++ b/web/src/lib/components/Shell.test.ts @@ -67,6 +67,21 @@ describe('Shell', () => { expect(screen.getByRole('link', { name: 'Discover' })).toHaveAttribute('href', '/discover'); }); + test('admins get Wanted after Discover; others do not', () => { + const primary = () => + screen + .getAllByRole('link') + .filter((el) => el.closest('nav[aria-label="Primary"]') !== null) + .map((el) => el.textContent?.trim()); + const { unmount } = render(Shell); + expect(primary()).not.toContain('Wanted'); + unmount(); + userState.current = { id: '1', username: 'alice', is_admin: true }; + render(Shell); + expect(primary()).toEqual(['Home', 'Library', 'Discover', 'Wanted']); + expect(screen.getByRole('link', { name: 'Wanted' })).toHaveAttribute('href', '/wanted'); + }); + test('main nav omits Artists/Albums/Liked/History/Playlists/Search/Settings/Admin', () => { userState.current = { id: '1', username: 'alice', is_admin: true }; render(Shell); diff --git a/web/src/lib/components/WantedTabs.svelte b/web/src/lib/components/WantedTabs.svelte new file mode 100644 index 00000000..3a9bf1a4 --- /dev/null +++ b/web/src/lib/components/WantedTabs.svelte @@ -0,0 +1,54 @@ + + + diff --git a/web/src/lib/components/wanted/BlockList.svelte b/web/src/lib/components/wanted/BlockList.svelte new file mode 100644 index 00000000..cd41332a --- /dev/null +++ b/web/src/lib/components/wanted/BlockList.svelte @@ -0,0 +1,88 @@ + + +{#if blocks.isPending} +

Reading the blocked releases…

+{:else if blocks.isError && !blocks.isFetchNextPageError} +

Couldn't load the blocked releases.

+{:else if items.length === 0} +
+ +

No releases are blocked.

+

+ A release you set aside, or one an import found wasn't the album, lands here so it's never + taken again. +

+
+{:else} +
    + {#each items as b (b.info_hash)} +
  • +
    +
    {b.name || b.info_hash}
    + {#if b.want_title} +
    For {b.want_artist} – {b.want_title}
    + {/if} +
    {by(b)} {relativeTime(b.blocked_at)}
    + {#if b.detail} +

    {b.detail}

    + {/if} +
    + +
    • + {/each} +
+ + blocks.fetchNextPage()} + /> +{/if} diff --git a/web/src/lib/components/wanted/BlockList.test.ts b/web/src/lib/components/wanted/BlockList.test.ts new file mode 100644 index 00000000..bd3574c3 --- /dev/null +++ b/web/src/lib/components/wanted/BlockList.test.ts @@ -0,0 +1,50 @@ +import { afterEach, describe, expect, test, vi } from 'vitest'; +import { fireEvent, render, screen, waitFor } from '@testing-library/svelte'; +import { mockInfiniteQuery } from '../../../test-utils/query'; +import type { ReleaseBlock } from '#lib/api/types.js'; + +vi.mock('#lib/api/wanted.js', () => ({ + createReleaseBlocksQuery: vi.fn(), + unblockRelease: vi.fn().mockResolvedValue(undefined) +})); +vi.mock('#lib/stores/toast.svelte.js', () => ({ pushToast: vi.fn() })); + +import BlockList from './BlockList.svelte'; +import { createReleaseBlocksQuery, unblockRelease } from '#lib/api/wanted.js'; + +const block: ReleaseBlock = { + info_hash: 'abc123', + name: 'Daft Punk - Random Access Memories (Drumless Edition)', + reason: 'import', + detail: 'The album tag says another version.', + blocked_at: new Date(Date.now() - 3_600_000).toISOString(), + want_title: 'Random Access Memories', + want_artist: 'Daft Punk' +}; + +afterEach(() => vi.clearAllMocks()); + +function setup(items: ReleaseBlock[]) { + vi.mocked(createReleaseBlocksQuery).mockReturnValue( + mockInfiniteQuery({ pages: [{ items, total: items.length, limit: 50, offset: 0 }] }) as unknown as ReturnType< + typeof createReleaseBlocksQuery + > + ); + return render(BlockList); +} + +describe('BlockList', () => { + test('a blocked release says why and can be allowed again', async () => { + setup([block]); + expect(screen.getByText(/refused on import/i)).toBeTruthy(); + expect(screen.getByText(/another version/)).toBeTruthy(); + expect(screen.getByText(/for daft punk – random access memories/i)).toBeTruthy(); + await fireEvent.click(screen.getByRole('button', { name: /allow .* again/i })); + await waitFor(() => expect(unblockRelease).toHaveBeenCalledWith('abc123')); + }); + + test('with nothing blocked it says so', () => { + setup([]); + expect(screen.getByText(/no releases are blocked/i)).toBeTruthy(); + }); +}); diff --git a/web/src/lib/components/wanted/DownloadList.svelte b/web/src/lib/components/wanted/DownloadList.svelte new file mode 100644 index 00000000..183babbb --- /dev/null +++ b/web/src/lib/components/wanted/DownloadList.svelte @@ -0,0 +1,192 @@ + + +{#if downloads.isPending} +

Reading the downloads…

+{:else if downloads.isError && !downloads.isFetchNextPageError} +

Couldn't load the downloads.

+{:else if items.length === 0} +
+ +

Nothing has been downloaded yet.

+

+ A release the search takes, or one you take from a want's Releases, shows here while it + downloads and once it's imported. +

+
+{:else} +
    + {#each items as d (d.id)} +
  • +
    +
    +
    + {d.want_artist ? `${d.want_artist} – ${d.want_title}` : d.name} +
    +
    + {d.name}{#if d.quality} · {d.quality}{/if} +
    +
    + + {label(d)} + +
    + + {#if !d.ended_at && (d.import_state === 'downloading' || d.import_state === '')} +
    +
    +
    + {/if} + + {#if d.import_reason} +

    + {d.import_reason} +

    + {/if} + {#if d.evidence} +

    {counted(d.evidence)}

    + {/if} + {#if filed(d)} +

    {filed(d)}

    + {/if} + {#if d.rehomed_from} +

    Taken for {d.rehomed_from}; it turned out to be this album.

    + {/if} +

    {when(d)}

    + + {#if needsYou(d)} +
    + {#if d.import_state === 'review'} + + {:else} + + {/if} + +
    + {/if} +
    • + {/each} +
+ + downloads.fetchNextPage()} + /> +{/if} diff --git a/web/src/lib/components/wanted/DownloadList.test.ts b/web/src/lib/components/wanted/DownloadList.test.ts new file mode 100644 index 00000000..812353c2 --- /dev/null +++ b/web/src/lib/components/wanted/DownloadList.test.ts @@ -0,0 +1,109 @@ +import { afterEach, describe, expect, test, vi } from 'vitest'; +import { fireEvent, render, screen, waitFor } from '@testing-library/svelte'; +import { mockInfiniteQuery } from '../../../test-utils/query'; +import type { Download } from '#lib/api/types.js'; + +vi.mock('#lib/api/wanted.js', () => ({ + createDownloadsQuery: vi.fn(), + importAnyway: vi.fn().mockResolvedValue(undefined), + retryImport: vi.fn().mockResolvedValue(undefined), + refuseDownload: vi.fn().mockResolvedValue(undefined) +})); +vi.mock('#lib/stores/toast.svelte.js', () => ({ pushToast: vi.fn() })); + +import DownloadList from './DownloadList.svelte'; +import { createDownloadsQuery, importAnyway, refuseDownload, retryImport } from '#lib/api/wanted.js'; + +const ago = (ms: number) => new Date(Date.now() - ms).toISOString(); + +function download(id: string, extra: Partial = {}): Download { + return { + id, + want_id: 'w1', + want_title: 'Random Access Memories', + want_artist: 'Daft Punk', + name: 'Daft Punk - Random Access Memories (2013) [FLAC]', + indexer: 'Prowlarr', + client: 'qbittorrent', + quality: 'lossless', + origin: 'schedule', + grabbed_at: ago(3_600_000), + ended_at: null, + end_reason: '', + progress: 0, + import_state: 'downloading', + import_reason: '', + imported_at: null, + imported_files: 0, + destination: '', + files: [], + skipped: 0, + ...extra + }; +} + +afterEach(() => vi.clearAllMocks()); + +function setup(items: Download[]) { + vi.mocked(createDownloadsQuery).mockReturnValue( + mockInfiniteQuery({ pages: [{ items, total: items.length, limit: 50, offset: 0 }] }) as unknown as ReturnType< + typeof createDownloadsQuery + > + ); + return render(DownloadList); +} + +describe('DownloadList', () => { + test('a running download shows how far it has got', () => { + setup([download('d1', { progress: 0.375 })]); + expect(screen.getByTestId('download-state').textContent).toMatch(/downloading/i); + expect(screen.getByRole('progressbar', { name: /downloaded/i }).getAttribute('aria-valuenow')).toBe('37'); + expect(screen.queryByRole('button')).toBeNull(); + }); + + test('a doubtful download says why and can be imported anyway or refused', async () => { + setup([ + download('d1', { + import_state: 'review', + import_reason: 'Only the lengths agree. A person decides.', + evidence: { files: 13, tracks: 13, identified: 0, recordings: 0, lengths: 13, titles: 0, fingerprinted: false } + }) + ]); + expect(screen.getByTestId('download-state').textContent).toMatch(/needs you/i); + expect(screen.getByTestId('download-reason').textContent).toMatch(/a person decides/i); + expect(screen.getByText(/13 files against 13 tracks · no AcoustID · 13 lengths agree/)).toBeTruthy(); + await fireEvent.click(screen.getByRole('button', { name: /import anyway/i })); + await waitFor(() => expect(importAnyway).toHaveBeenCalledWith('d1')); + await fireEvent.click(screen.getByRole('button', { name: /refuse/i })); + await waitFor(() => expect(refuseDownload).toHaveBeenCalledWith('d1')); + }); + + test('a failed import can be tried again', async () => { + setup([download('d1', { import_state: 'failed', import_reason: 'The library folder is not writable.' })]); + expect(screen.queryByRole('button', { name: /import anyway/i })).toBeNull(); + await fireEvent.click(screen.getByRole('button', { name: /try again/i })); + await waitFor(() => expect(retryImport).toHaveBeenCalledWith('d1')); + }); + + test('an imported download says what was filed where', () => { + setup([ + download('d1', { + ended_at: ago(60_000), + end_reason: 'imported', + import_state: 'imported', + imported_at: ago(60_000), + imported_files: 12, + skipped: 1, + destination: '/music/Daft Punk/Random Access Memories (2013)' + }) + ]); + expect(screen.getByTestId('download-state').textContent).toMatch(/imported/i); + expect(screen.getByText(/12 tracks filed, 1 already held into \/music\/Daft Punk/)).toBeTruthy(); + expect(screen.queryByRole('button')).toBeNull(); + }); + + test('with nothing downloaded it says what lands here', () => { + setup([]); + expect(screen.getByText(/nothing has been downloaded yet/i)).toBeTruthy(); + }); +}); diff --git a/web/src/lib/components/wanted/FollowList.svelte b/web/src/lib/components/wanted/FollowList.svelte new file mode 100644 index 00000000..bd1e3fbd --- /dev/null +++ b/web/src/lib/components/wanted/FollowList.svelte @@ -0,0 +1,117 @@ + + +{#if follows.isPending} +

Reading who's followed…

+{:else if follows.isError} +

Couldn't load the followed artists.

+{:else if (follows.data ?? []).length === 0} +

No artists are followed. Follow one from its artist page.

+{:else} +
    + {#each follows.data ?? [] as f (f.artist_mbid)} +
  • + +
    +
    + {#if f.artist_id} + {f.artist_name} + {:else} + {f.artist_name} + {/if} +
    +
    + {f.open_wants === 1 ? '1 album wanted' : `${f.open_wants} albums wanted`} · + {f.checked_at ? `checked ${relativeTime(f.checked_at)}` : 'not checked yet'} +
    +
    + + +
    • + {/each} +
+{/if} diff --git a/web/src/lib/components/wanted/FollowList.test.ts b/web/src/lib/components/wanted/FollowList.test.ts new file mode 100644 index 00000000..7ae23c05 --- /dev/null +++ b/web/src/lib/components/wanted/FollowList.test.ts @@ -0,0 +1,73 @@ +import { afterEach, describe, expect, test, vi } from 'vitest'; +import { fireEvent, render, screen, waitFor } from '@testing-library/svelte'; +import { mockQuery } from '../../../test-utils/query'; +import type { Follow, ReleaseProfilesResponse } from '#lib/api/types.js'; + +vi.mock('#lib/api/wanted.js', () => ({ + createFollowsQuery: vi.fn(), + createReleaseProfilesQuery: vi.fn(), + putFollow: vi.fn().mockResolvedValue({ following: true }), + deleteFollow: vi.fn().mockResolvedValue(undefined) +})); +vi.mock('#lib/stores/toast.svelte.js', () => ({ pushToast: vi.fn() })); + +import FollowList from './FollowList.svelte'; +import { createFollowsQuery, createReleaseProfilesQuery, deleteFollow, putFollow } from '#lib/api/wanted.js'; + +const ago = (ms: number) => new Date(Date.now() - ms).toISOString(); + +const follow: Follow = { + artist_mbid: 'blur', + artist_name: 'Blur', + artist_id: 'art-1', + profile_id: 'p-std', + profile_name: 'Standard', + followed_at: ago(10 * 86_400_000), + checked_at: ago(3 * 3_600_000), + open_wants: 2, + image_url: '' +}; + +const profiles: ReleaseProfilesResponse = { + profiles: [ + { id: 'p-std', name: 'Standard', primary_types: ['Album'], secondary_types: [], is_default: true, follow_count: 1 }, + { id: 'p-live', name: 'With live', primary_types: ['Album'], secondary_types: ['Live'], is_default: false, follow_count: 0 } + ], + primary_types: ['Album'], + secondary_types: ['Live'] +}; + +afterEach(() => vi.clearAllMocks()); + +function setup(follows: Follow[] = [follow]) { + vi.mocked(createFollowsQuery).mockReturnValue( + mockQuery({ data: follows }) as unknown as ReturnType + ); + vi.mocked(createReleaseProfilesQuery).mockReturnValue( + mockQuery({ data: profiles }) as unknown as ReturnType + ); + return render(FollowList); +} + +describe('FollowList', () => { + test('lists artists; the profile can change and the artist be unfollowed', async () => { + setup(); + expect(screen.getAllByTestId('follow-row')).toHaveLength(1); + expect(screen.getByText(/2 albums wanted/)).toBeTruthy(); + + const select = screen.getByLabelText(/profile for blur/i) as HTMLSelectElement; + expect(select.value).toBe('p-std'); + await fireEvent.change(select, { target: { value: 'p-live' } }); + await waitFor(() => + expect(putFollow).toHaveBeenCalledWith('blur', { artist_name: 'Blur', profile_id: 'p-live' }) + ); + + await fireEvent.click(screen.getByRole('button', { name: /unfollow blur/i })); + await waitFor(() => expect(deleteFollow).toHaveBeenCalledWith('blur')); + }); + + test('with no follows it says where to follow from', () => { + setup([]); + expect(screen.getByText(/follow one from its artist page/i)).toBeTruthy(); + }); +}); diff --git a/web/src/lib/components/wanted/WantList.svelte b/web/src/lib/components/wanted/WantList.svelte new file mode 100644 index 00000000..d13e390f --- /dev/null +++ b/web/src/lib/components/wanted/WantList.svelte @@ -0,0 +1,263 @@ + + +{#if wants.isPending} +

Reading the wanted list…

+{:else if wants.isError && !wants.isFetchNextPageError} +

Couldn't load the wanted list.

+{:else if items.length === 0} +
+ +

+ {#if state === 'held'}Nothing on the list has been held yet. + {:else if state === 'missing'}No missing albums are wanted back. + {:else}Nothing is wanted.{/if} +

+

+ {#if state === 'missing'} + An album whose files stay missing past the grace window on the Missing files page lands here. + {:else} + A request you approve, a new release from a followed artist, or an album whose files went + missing lands here. + {/if} +

+
+{:else} +
    + {#each items as w (w.id)} +
  • +
    + +
    +
    + {#if w.album_id && w.held_at} + {w.title} + {:else} + {w.title} + {/if} +
    +
    + {#if w.artist_id} + {w.artist_name} + {:else} + {w.artist_name} + {/if} + {#if kind(w)} · {kind(w)}{/if} +
    +
    + {why(w)} · {w.held_at ? `held ${relativeTime(w.held_at)}` : `wanted ${relativeTime(w.wanted_at)}`} +
    + {#if !w.held_at} +
    + {status(w)} + {#if attention(w)} + Decide on Activity + {/if} +
    + {:else if arrived(w)} +
    {arrived(w)}
    + {/if} +
    + {#if !w.held_at} +
    + + + +
    + {/if} +
    + {#if open === w.id && !w.held_at} +
    + +
    + {/if} +
    • + {/each} +
+ + wants.fetchNextPage()} + /> +{/if} diff --git a/web/src/routes/admin/wanted/wanted.test.ts b/web/src/lib/components/wanted/WantList.test.ts similarity index 58% rename from web/src/routes/admin/wanted/wanted.test.ts rename to web/src/lib/components/wanted/WantList.test.ts index 410505ae..47e6ae09 100644 --- a/web/src/routes/admin/wanted/wanted.test.ts +++ b/web/src/lib/components/wanted/WantList.test.ts @@ -1,15 +1,11 @@ import { afterEach, describe, expect, test, vi } from 'vitest'; import { fireEvent, render, screen, waitFor } from '@testing-library/svelte'; import { mockInfiniteQuery, mockQuery } from '../../../test-utils/query'; -import type { Follow, ReleaseProfilesResponse, Want, WantsPage } from '#lib/api/types.js'; +import type { Want, WantGrab, WantsPage } from '#lib/api/types.js'; vi.mock('#lib/api/wanted.js', () => ({ createWantsQuery: vi.fn(), - createFollowsQuery: vi.fn(), - createReleaseProfilesQuery: vi.fn(), removeWant: vi.fn().mockResolvedValue(undefined), - putFollow: vi.fn().mockResolvedValue({ following: true }), - deleteFollow: vi.fn().mockResolvedValue(undefined), searchWant: vi.fn().mockResolvedValue({ outcome: 'nothing', considered: 0, best_release: '', best_verdict: '', narrowed: [] }), @@ -20,18 +16,10 @@ vi.mock('#lib/api/wanted.js', () => ({ })); vi.mock('#lib/stores/toast.svelte.js', () => ({ pushToast: vi.fn() })); -import AdminWantedPage from './+page.svelte'; +import WantList from './WantList.svelte'; import { pushToast } from '#lib/stores/toast.svelte.js'; -import { - createCandidatesQuery, - createFollowsQuery, - createReleaseProfilesQuery, - createWantsQuery, - deleteFollow, - putFollow, - removeWant, - searchWant -} from '#lib/api/wanted.js'; +import { createCandidatesQuery, createWantsQuery, removeWant, searchWant } from '#lib/api/wanted.js'; +import type { WantState } from '#lib/api/types.js'; const DAY = 24 * 3_600_000; const ago = (ms: number) => new Date(Date.now() - ms).toISOString(); @@ -61,54 +49,48 @@ function page(items: Want[]): WantsPage { total: items.length, limit: 50, offset: 0, - counts: { wanted: 2, missing: 1, held: 4, following: 1 } + counts: { wanted: 2, missing: 1, held: 4, following: 1, activity: 1, attention: 0, blocked: 0 } }; } -const follow: Follow = { - artist_mbid: 'blur', - artist_name: 'Blur', - artist_id: 'art-1', - profile_id: 'p-std', - profile_name: 'Standard', - followed_at: ago(10 * DAY), - checked_at: ago(3 * 3_600_000), - open_wants: 2, - image_url: '' -}; - -const profiles: ReleaseProfilesResponse = { - profiles: [ - { id: 'p-std', name: 'Standard', primary_types: ['Album'], secondary_types: [], is_default: true, follow_count: 1 }, - { id: 'p-live', name: 'With live', primary_types: ['Album'], secondary_types: ['Live'], is_default: false, follow_count: 0 } - ], - primary_types: ['Album'], - secondary_types: ['Live'] -}; +function grab(extra: Partial = {}): WantGrab { + return { + id: 'g1', + name: 'Blur - Blur (1997) [FLAC]', + grabbed_at: ago(60_000), + origin: 'schedule', + quality: 'lossless', + live: true, + end_reason: '', + progress: 0, + import_state: '', + import_reason: '', + imported_at: null, + imported_files: 0, + ...extra + }; +} afterEach(() => vi.clearAllMocks()); -function setup(items: Want[] = [want('w1'), want('w2', { reason: 'request', requested_by: 'sam', title: 'Parklife' })]) { +function setup( + items: Want[] = [want('w1'), want('w2', { reason: 'request', requested_by: 'sam', title: 'Parklife' })], + state: WantState = 'wanted' +) { vi.mocked(createWantsQuery).mockReturnValue( mockInfiniteQuery({ pages: [page(items)] }) as unknown as ReturnType ); - vi.mocked(createFollowsQuery).mockReturnValue( - mockQuery({ data: [follow] }) as unknown as ReturnType - ); - vi.mocked(createReleaseProfilesQuery).mockReturnValue( - mockQuery({ data: profiles }) as unknown as ReturnType - ); vi.mocked(createCandidatesQuery).mockReturnValue( mockQuery({ data: [] }) as unknown as ReturnType ); - return render(AdminWantedPage); + return render(WantList, { state }); } -describe('/admin/wanted', () => { - test('each tab shows its count', () => { - setup(); - expect(screen.getByRole('tab', { name: /wanted/i }).textContent).toMatch(/2/); - expect(screen.getByRole('tab', { name: /held/i }).textContent).toMatch(/4/); +describe('WantList', () => { + test('asks for its own state', () => { + setup([], 'missing'); + expect(createWantsQuery).toHaveBeenCalledWith('missing'); + expect(screen.getByText(/no missing albums are wanted back/i)).toBeTruthy(); }); test('a want says why it is wanted', () => { @@ -118,12 +100,6 @@ describe('/admin/wanted', () => { expect(why[1]).toMatch(/requested by sam/i); }); - test('switching tab asks for that state', async () => { - setup(); - await fireEvent.click(screen.getByRole('tab', { name: /missing files/i })); - expect(createWantsQuery).toHaveBeenCalledWith('missing'); - }); - test('remove takes a want off the list', async () => { setup(); await fireEvent.click(screen.getByRole('button', { name: /stop wanting parklife/i })); @@ -136,22 +112,6 @@ describe('/admin/wanted', () => { expect(screen.getByRole('link', { name: /the ballad of darren/i }).getAttribute('href')).toBe('/albums/al-1'); }); - test('following lists artists; the profile can change and the artist be unfollowed', async () => { - setup(); - await fireEvent.click(screen.getByRole('tab', { name: /following/i })); - expect(screen.getAllByTestId('follow-row')).toHaveLength(1); - - const select = screen.getByLabelText(/profile for blur/i) as HTMLSelectElement; - expect(select.value).toBe('p-std'); - await fireEvent.change(select, { target: { value: 'p-live' } }); - await waitFor(() => - expect(putFollow).toHaveBeenCalledWith('blur', { artist_name: 'Blur', profile_id: 'p-live' }) - ); - - await fireEvent.click(screen.getByRole('button', { name: /unfollow blur/i })); - await waitFor(() => expect(deleteFollow).toHaveBeenCalledWith('blur')); - }); - test('an empty list explains what lands there', () => { setup([]); expect(screen.getByText(/nothing is wanted/i)).toBeTruthy(); @@ -164,15 +124,49 @@ describe('/admin/wanted', () => { title: 'Parklife', search: { searched_at: ago(3_600_000), outcome: 'nothing', considered: 4, best_release: 'x', best_verdict: 'rejected', error: '' } }), - want('w3', { - title: 'Blur', - grab: { name: 'Blur - Blur (1997) [FLAC]', grabbed_at: ago(60_000), origin: 'schedule', quality: 'lossless' } - }) + want('w3', { title: 'Blur', grab: grab() }), + want('w4', { title: '13', grab: grab({ import_state: 'downloading', progress: 0.42 }) }) ]); const lines = screen.getAllByTestId('want-search').map((e) => e.textContent ?? ''); expect(lines[0]).toMatch(/not searched yet/i); expect(lines[1]).toMatch(/4 found, none of them this album/i); expect(lines[2]).toMatch(/downloading blur - blur \(1997\) \[flac\], taken by the schedule/i); + expect(lines[3]).toMatch(/downloading 42%/i); + }); + + test('a download waiting on a person says why and points to Activity', () => { + setup([ + want('w1', { grab: grab({ import_state: 'review', import_reason: 'Only the lengths agree. A person decides.' }) }) + ]); + const line = screen.getByTestId('want-search'); + expect(line.textContent).toMatch(/needs you: only the lengths agree/i); + expect(screen.getByRole('link', { name: /decide on activity/i }).getAttribute('href')).toBe('/wanted/activity'); + // A live download can't be searched for again. + expect((screen.getByRole('button', { name: /search for the ballad of darren now/i }) as HTMLButtonElement).disabled).toBe(true); + }); + + test('a refused download is named beside the search that follows it', () => { + setup([ + want('w1', { + search: { searched_at: ago(3_600_000), outcome: 'nothing', considered: 0, best_release: '', best_verdict: '', error: '' }, + grab: grab({ live: false, end_reason: 'refused', import_state: 'refused', import_reason: 'Another album.' }) + }) + ]); + expect(screen.getByTestId('want-search').textContent).toMatch(/last download refused: another album/i); + }); + + test('a held album says how it arrived', () => { + setup( + [ + want('w1', { + held_at: ago(DAY), + album_id: 'al-1', + grab: grab({ live: false, end_reason: 'imported', import_state: 'imported', imported_at: ago(DAY), imported_files: 11 }) + }) + ], + 'held' + ); + expect(screen.getByTestId('want-arrived').textContent).toMatch(/imported 11 tracks .* from blur - blur/i); }); test('Search runs one now and says what came of it', async () => { diff --git a/web/src/lib/styles/error-copy.json b/web/src/lib/styles/error-copy.json index 90abec17..e9a072c6 100644 --- a/web/src/lib/styles/error-copy.json +++ b/web/src/lib/styles/error-copy.json @@ -26,6 +26,9 @@ "bad_body": "Couldn't read the request.", "bad_kind": "Invalid request type.", "bad_paging": "Invalid page size or offset.", + "invalid_paging": "Invalid page size or offset.", + "not_decidable": "That download isn't waiting on a decision any more. Refresh the list.", + "bad_import_root": "New music has to be filed into one of the library's folders.", "bad_reason": "Invalid quarantine reason.", "mbid_required": "An MBID is required for this lookup.", "system_playlist_readonly": "System playlists can't be edited directly.", diff --git a/web/src/routes/admin/integrations/+page.svelte b/web/src/routes/admin/integrations/+page.svelte index 94542a0d..3095e4cf 100644 --- a/web/src/routes/admin/integrations/+page.svelte +++ b/web/src/routes/admin/integrations/+page.svelte @@ -29,7 +29,6 @@ import Modal from '#lib/components/Modal.svelte'; import AcoustIDCard from '#lib/components/AcoustIDCard.svelte'; import AcquisitionCard from '#lib/components/AcquisitionCard.svelte'; - import ReleaseProfilesCard from '#lib/components/ReleaseProfilesCard.svelte'; import SearchSettingsCard from '#lib/components/SearchSettingsCard.svelte'; import NetworkSettingsCard from '#lib/components/NetworkSettingsCard.svelte'; import PublicAddressCard from '#lib/components/PublicAddressCard.svelte'; @@ -427,8 +426,6 @@ - -
diff --git a/web/src/routes/admin/integrations/integrations.test.ts b/web/src/routes/admin/integrations/integrations.test.ts index 26b475d7..5bd63952 100644 --- a/web/src/routes/admin/integrations/integrations.test.ts +++ b/web/src/routes/admin/integrations/integrations.test.ts @@ -43,25 +43,10 @@ vi.mock('#lib/api/admin.js', () => ({ addAcquisitionService: vi.fn() })); -// ReleaseProfilesCard reads its own query; settled, so it adds no "Loading…". +// SearchSettingsCard reads its own query; settled, so it adds no "Loading…". vi.mock('#lib/api/wanted.js', async () => { const { mockQuery } = await import('../../../test-utils/query'); return { - createReleaseProfilesQuery: vi.fn(() => - mockQuery({ - data: { - profiles: [ - { id: 'p-std', name: 'Standard', primary_types: ['Album', 'EP', 'Single'], secondary_types: [], is_default: true, follow_count: 3 } - ], - primary_types: ['Album', 'EP', 'Single'], - secondary_types: ['Live'] - } - }) - ), - createReleaseProfile: vi.fn(), - updateReleaseProfile: vi.fn(), - deleteReleaseProfile: vi.fn(), - // SearchSettingsCard, settled the same way. createSearchSettingsQuery: vi.fn(() => mockQuery({ data: { @@ -71,7 +56,9 @@ vi.mock('#lib/api/wanted.js', async () => { allowed_qualities: ['lossy_high'], tiers: [{ tier: 'lossy_high', label: 'High quality lossy', what: 'MP3 320' }], last_pass_at: null, - last_pass: {} + last_pass: {}, + import_root: '', + scan_paths: ['/music'] } }) ), diff --git a/web/src/routes/admin/missing-files/+page.svelte b/web/src/routes/admin/missing-files/+page.svelte index f64636f5..fb89e25d 100644 --- a/web/src/routes/admin/missing-files/+page.svelte +++ b/web/src/routes/admin/missing-files/+page.svelte @@ -107,7 +107,7 @@ {#if r.held_at} Held again {relativeTime(r.held_at)}. The rows here clear on the next scan. {:else if r.wanted_at} - On the wanted list since {relativeTime(r.wanted_at)}. {/if} diff --git a/web/src/routes/admin/missing-files/missing-files.test.ts b/web/src/routes/admin/missing-files/missing-files.test.ts index e8adf9b7..cb804653 100644 --- a/web/src/routes/admin/missing-files/missing-files.test.ts +++ b/web/src/routes/admin/missing-files/missing-files.test.ts @@ -140,7 +140,7 @@ describe('admin missing files', () => { expect( within(screen.getByTestId('reacquisition-state')).getByRole('link', { name: /wanted list/i }).getAttribute('href') ).toBe( - '/admin/wanted?tab=missing' + '/wanted/missing' ); }); diff --git a/web/src/routes/admin/requests/+page.svelte b/web/src/routes/admin/requests/+page.svelte index 06395cd9..b2952c24 100644 --- a/web/src/routes/admin/requests/+page.svelte +++ b/web/src/routes/admin/requests/+page.svelte @@ -146,7 +146,7 @@

Requests

Approve or set aside what users have asked for. An approved album or track goes on the - wanted list; an approved + wanted list; an approved artist is followed, wanting what they've released and what they release next.

diff --git a/web/src/routes/admin/wanted/+page.svelte b/web/src/routes/admin/wanted/+page.svelte deleted file mode 100644 index d791c2d5..00000000 --- a/web/src/routes/admin/wanted/+page.svelte +++ /dev/null @@ -1,363 +0,0 @@ - - -{pageTitle('Admin · Wanted')} - -
-
-

Wanted

-

- Albums Minstrel is to get: ones users requested, new releases from the artists it follows, - and albums whose files went missing. Minstrel searches your indexers for each one on a - schedule and hands the best release to your download client; searching and quality are set - under Integrations. - An album leaves the list once the library holds it. -

-
- - - - {#if tab === 'following'} -

- Each artist is followed on a release-type profile, which says which kinds of release are - wanted. Profiles are edited under - Integrations. -

- {#if follows.isPending} -

Reading who's followed…

- {:else if follows.isError} -

Couldn't load the followed artists.

- {:else if (follows.data ?? []).length === 0} -

- No artists are followed. Follow one from its artist page. -

- {:else} -
    - {#each follows.data ?? [] as f (f.artist_mbid)} -
  • - -
    -
    - {#if f.artist_id} - {f.artist_name} - {:else} - {f.artist_name} - {/if} -
    -
    - {f.open_wants === 1 ? '1 album wanted' : `${f.open_wants} albums wanted`} · - {f.checked_at ? `checked ${relativeTime(f.checked_at)}` : 'not checked yet'} -
    -
    - - -
    • - {/each} -
- {/if} - {:else if wants.isPending} -

Reading the wanted list…

- {:else if wants.isError && !wants.isFetchNextPageError} -

Couldn't load the wanted list.

- {:else if items.length === 0} -
- -

- {#if tab === 'held'}Nothing on the list has been held yet. - {:else if tab === 'missing'}No missing albums are wanted back. - {:else}Nothing is wanted.{/if} -

-

- {#if tab === 'missing'} - An album whose files stay missing past the grace window on the Missing files page lands here. - {:else} - A request you approve, a new release from a followed artist, or an album whose files went - missing lands here. - {/if} -

-
- {:else} -
    - {#each items as w (w.id)} -
  • -
    - -
    -
    - {#if w.album_id && w.held_at} - {w.title} - {:else} - {w.title} - {/if} -
    -
    - {#if w.artist_id} - {w.artist_name} - {:else} - {w.artist_name} - {/if} - {#if kind(w)} · {kind(w)}{/if} -
    -
    - {why(w)} · {w.held_at ? `held ${relativeTime(w.held_at)}` : `wanted ${relativeTime(w.wanted_at)}`} -
    - {#if !w.held_at} -
    {status(w)}
    - {/if} -
    - {#if !w.held_at} -
    - - - -
    - {/if} -
    - {#if open === w.id && !w.held_at} -
    - -
    - {/if} -
    • - {/each} -
- - wants.fetchNextPage()} - /> - {/if} -
diff --git a/web/src/routes/wanted/+layout.svelte b/web/src/routes/wanted/+layout.svelte new file mode 100644 index 00000000..0e2116bf --- /dev/null +++ b/web/src/routes/wanted/+layout.svelte @@ -0,0 +1,48 @@ + + +
+
+

Wanted

+

+ Albums Minstrel is getting: ones users requested, new releases from the artists it follows, + and albums whose files went missing. It searches your indexers on a schedule, checks each + download against MusicBrainz, and files what's right into the library. Searching and quality + are set under Integrations. +

+ {#if verdict} +

+ {verdict} + {#if counts?.attention} + See Activity + {/if} +

+ {/if} +
+ + + + {@render children?.()} +
diff --git a/web/src/routes/wanted/+layout.ts b/web/src/routes/wanted/+layout.ts new file mode 100644 index 00000000..f22d5109 --- /dev/null +++ b/web/src/routes/wanted/+layout.ts @@ -0,0 +1,13 @@ +import { redirect } from '@sveltejs/kit'; +import { user } from '#lib/auth/store.svelte.js'; +import type { LayoutLoad } from './$types'; + +// Hard route gate: the Wanted view is an admin's (M502). Runs before the +// layout renders; auth is bootstrapped by the root +layout.ts, so user.value is +// either a settled User object or null. +export const load: LayoutLoad = () => { + if (!user.value || !user.value.is_admin) { + throw redirect(302, '/'); + } + return {}; +}; diff --git a/web/src/routes/wanted/+page.svelte b/web/src/routes/wanted/+page.svelte new file mode 100644 index 00000000..96b2637e --- /dev/null +++ b/web/src/routes/wanted/+page.svelte @@ -0,0 +1,8 @@ + + +{pageTitle('Wanted')} + + diff --git a/web/src/routes/wanted/activity/+page.svelte b/web/src/routes/wanted/activity/+page.svelte new file mode 100644 index 00000000..c98588ee --- /dev/null +++ b/web/src/routes/wanted/activity/+page.svelte @@ -0,0 +1,8 @@ + + +{pageTitle('Wanted · Activity')} + + diff --git a/web/src/routes/wanted/blocked/+page.svelte b/web/src/routes/wanted/blocked/+page.svelte new file mode 100644 index 00000000..3c4ea0d0 --- /dev/null +++ b/web/src/routes/wanted/blocked/+page.svelte @@ -0,0 +1,8 @@ + + +{pageTitle('Wanted · Blocked')} + + diff --git a/web/src/routes/wanted/following/+page.svelte b/web/src/routes/wanted/following/+page.svelte new file mode 100644 index 00000000..7303fe95 --- /dev/null +++ b/web/src/routes/wanted/following/+page.svelte @@ -0,0 +1,12 @@ + + +{pageTitle('Wanted · Following')} + +
+ + +
diff --git a/web/src/routes/wanted/held/+page.svelte b/web/src/routes/wanted/held/+page.svelte new file mode 100644 index 00000000..dd44902b --- /dev/null +++ b/web/src/routes/wanted/held/+page.svelte @@ -0,0 +1,8 @@ + + +{pageTitle('Wanted · Held')} + + diff --git a/web/src/routes/wanted/missing/+page.svelte b/web/src/routes/wanted/missing/+page.svelte new file mode 100644 index 00000000..f2b7e6df --- /dev/null +++ b/web/src/routes/wanted/missing/+page.svelte @@ -0,0 +1,8 @@ + + +{pageTitle('Wanted · Missing files')} + + diff --git a/web/src/routes/wanted/wanted.test.ts b/web/src/routes/wanted/wanted.test.ts new file mode 100644 index 00000000..9787f91b --- /dev/null +++ b/web/src/routes/wanted/wanted.test.ts @@ -0,0 +1,61 @@ +import { afterEach, describe, expect, test, vi } from 'vitest'; +import { render, screen } from '@testing-library/svelte'; +import { createRawSnippet } from 'svelte'; +import { mockInfiniteQuery } from '../../test-utils/query'; +import { pageUrlModule } from '#test-utils/mocks/appState.js'; +import type { WantCounts, WantsPage } from '#lib/api/types.js'; + +const state = vi.hoisted(() => ({ pageUrl: new URL('http://localhost/wanted') })); +vi.mock('$app/state', () => pageUrlModule(state)); +vi.mock('#lib/api/wanted.js', () => ({ createWantsQuery: vi.fn() })); + +import WantedLayout from './+layout.svelte'; +import { createWantsQuery } from '#lib/api/wanted.js'; + +const counts: WantCounts = { wanted: 5, missing: 1, held: 9, following: 2, activity: 3, attention: 0, blocked: 4 }; +const children = createRawSnippet(() => ({ render: () => `

body

` })); + +afterEach(() => vi.clearAllMocks()); + +function setup(c: WantCounts = counts, path = '/wanted') { + state.pageUrl = new URL(`http://localhost${path}`); + const page: WantsPage = { items: [], total: 0, limit: 50, offset: 0, counts: c }; + vi.mocked(createWantsQuery).mockReturnValue( + mockInfiniteQuery({ pages: [page] }) as unknown as ReturnType + ); + // eslint-disable-next-line @typescript-eslint/no-explicit-any + return render(WantedLayout, { children } as any); +} + +describe('/wanted layout', () => { + test('tabs are links carrying their counts, the current one marked', () => { + setup(counts, '/wanted/activity'); + const nav = screen.getByRole('navigation', { name: /wanted sections/i }); + const links = Array.from(nav.querySelectorAll('a')).map((a) => [a.getAttribute('href'), a.textContent?.replace(/\s+/g, ' ').trim()]); + expect(links).toEqual([ + ['/wanted', 'Wanted 5'], + ['/wanted/activity', 'Activity 3'], + ['/wanted/missing', 'Missing files 1'], + ['/wanted/held', 'Held 9'], + ['/wanted/following', 'Following 2'], + ['/wanted/blocked', 'Blocked 4'] + ]); + expect(screen.getByRole('link', { name: /activity/i })).toHaveAttribute('aria-current', 'page'); + expect(screen.getByRole('link', { name: /^wanted/i })).not.toHaveAttribute('aria-current'); + expect(screen.getByTestId('tab-body')).toBeTruthy(); + }); + + test('when nothing needs a person the verdict says so', () => { + setup(); + expect(screen.getByTestId('wanted-verdict').textContent).toMatch(/5 albums wanted, 3 downloading\. Nothing needs you\./); + expect(screen.queryByTestId('tab-attention')).toBeNull(); + }); + + test('downloads waiting on a person lead the page and mark Activity', () => { + setup({ ...counts, attention: 2 }); + const verdict = screen.getByTestId('wanted-verdict'); + expect(verdict.textContent).toMatch(/2 downloads need you/); + expect(screen.getByRole('link', { name: /see activity/i }).getAttribute('href')).toBe('/wanted/activity'); + expect(screen.getByTestId('tab-attention').textContent).toBe('2'); + }); +});