diff --git a/README.md b/README.md index 3437ba73..5e00f972 100644 --- a/README.md +++ b/README.md @@ -40,11 +40,13 @@ services: ports: ['4533:4533'] volumes: # Your music library. Point ./music at wherever your audio files - # live. Writable, because Minstrel deletes a file when an admin asks - # it to (for example, quarantine's "Delete file"). It never moves, - # renames or retags anything. The container runs as uid 1000, so that - # user needs write access to the folders. Mount it :ro to forbid even - # deletes: those actions then refuse, say why, and delete nothing. + # live. Writable, because Minstrel files the albums it downloads into + # it (as Artist/Album (Year)/..., tagged with MusicBrainz ids) and + # deletes a file when an admin asks it to (for example, quarantine's + # "Delete file"). Files already there are never moved, renamed or + # retagged. The container runs as uid 1000, so that user needs write + # access to the folders. Mount it :ro to forbid writes: those actions + # then refuse, say why, and change nothing. - ./music:/music # Generated data: playlist cover collages, artist art, caches. # The path must match MINSTREL_STORAGE_DATA_DIR, which the image diff --git a/cmd/minstrel/main.go b/cmd/minstrel/main.go index ef6a0e9d..028f6a82 100644 --- a/cmd/minstrel/main.go +++ b/cmd/minstrel/main.go @@ -22,6 +22,7 @@ import ( "git.fabledsword.com/bvandeusen/minstrel/internal/db" "git.fabledsword.com/bvandeusen/minstrel/internal/eventbus" "git.fabledsword.com/bvandeusen/minstrel/internal/gc" + "git.fabledsword.com/bvandeusen/minstrel/internal/importer" "git.fabledsword.com/bvandeusen/minstrel/internal/library" "git.fabledsword.com/bvandeusen/minstrel/internal/logging" "git.fabledsword.com/bvandeusen/minstrel/internal/mailer" @@ -148,8 +149,11 @@ func run() error { if acoustIDErr != nil { logger.Warn("acoustid settings: using defaults (lookup off)", "err", acoustIDErr) } + // One AcoustID client, so the lookup worker and the import's check share + // its pace (AcoustID allows three requests a second per key). + acoustIDClient := acoustid.New(acoustid.DefaultBaseURL) acoustIDLookup := library.NewAcoustIDLookupWorker(pool, logger.With("component", "acoustid_lookup"), - acoustIDSettings, acoustid.New(acoustid.DefaultBaseURL)) + acoustIDSettings, acoustIDClient) contact := cfg.Library.ContactEmail if contact == "" { @@ -328,6 +332,15 @@ func run() error { // when auto-grab is on. go wanted.NewSearcher(pool, cat, logger.With("component", "search")).Run(ctx) + // Import with verification (M502 #5522): every minute, finished + // downloads are checked against their album (AcoustID, lengths, tags) + // and filed into the library with MusicBrainz tags, or refused. + var printer importer.Printer + if acoustIDSettings != nil { + printer = &importer.AcoustIDPrinter{Settings: acoustIDSettings, Client: acoustIDClient} + } + go importer.New(pool, scanner, cat, printer, cfg.Library.ScanPaths, logger.With("component", "import")).Run(ctx) + // Duplicate resolver (M498, M502): classifies the sweep's groups and, with // the operator's auto-resolve on, merges the copies that are safe to merge. // Started after SetNotifier so its summaries reach admins. See diff --git a/internal/acquire/memory.go b/internal/acquire/memory.go index a9b0354a..bf8ab8f7 100644 --- a/internal/acquire/memory.go +++ b/internal/acquire/memory.go @@ -19,6 +19,8 @@ import ( const ( BlockOperator = "operator" BlockFiles = "files" + // BlockImport: the import found its files were not the album (#5522). + BlockImport = "import" ) // Grab origins, matching grabs_origin_check. diff --git a/internal/api/admin_import.go b/internal/api/admin_import.go new file mode 100644 index 00000000..71675013 --- /dev/null +++ b/internal/api/admin_import.go @@ -0,0 +1,205 @@ +package api + +import ( + "encoding/json" + "errors" + "net/http" + + "github.com/jackc/pgx/v5" + + "git.fabledsword.com/bvandeusen/minstrel/internal/apierror" + "git.fabledsword.com/bvandeusen/minstrel/internal/db/dbq" + "git.fabledsword.com/bvandeusen/minstrel/internal/importer" +) + +// Downloads and their imports (M502 #5522): the Wanted view's Activity tab, +// a person's "import anyway", "refuse" and "retry", and the Blocked tab. + +// downloadView is one grab and what its import made of it. +type downloadView struct { + ID string `json:"id"` + WantID *string `json:"want_id"` + WantTitle string `json:"want_title"` + WantArtist string `json:"want_artist"` + Name string `json:"name"` + Indexer string `json:"indexer"` + Client string `json:"client"` + Quality string `json:"quality"` + Origin string `json:"origin"` + GrabbedAt string `json:"grabbed_at"` + EndedAt *string `json:"ended_at"` + EndReason string `json:"end_reason"` + Progress float32 `json:"progress"` + ImportState string `json:"import_state"` + ImportReason string `json:"import_reason"` + ImportedAt *string `json:"imported_at"` + ImportedFiles int32 `json:"imported_files"` + Destination string `json:"destination"` + Edition string `json:"edition,omitempty"` + Evidence *importer.Evidence `json:"evidence,omitempty"` + Files []string `json:"files"` + Skipped int `json:"skipped"` + RehomedFrom string `json:"rehomed_from,omitempty"` +} + +func downloadFrom(g dbq.ListGrabsRow) downloadView { + v := downloadView{ + ID: uuidToString(g.ID), Name: g.Name, Indexer: g.Indexer, Client: g.Client, Quality: g.Quality, + Origin: g.Origin, GrabbedAt: formatTimestamp(g.GrabbedAt), EndReason: g.EndReason, + Progress: g.Progress, ImportState: g.ImportState, ImportReason: g.ImportReason, + ImportedFiles: g.ImportedFiles, Destination: g.Destination, Files: []string{}, + } + if g.WantID.Valid { + id := uuidToString(g.WantID) + v.WantID = &id + } + if g.WantTitle != nil { + v.WantTitle = *g.WantTitle + } + if g.WantArtist != nil { + v.WantArtist = *g.WantArtist + } + if g.EndedAt.Valid { + at := formatTimestamp(g.EndedAt) + v.EndedAt = &at + } + if g.ImportedAt.Valid { + at := formatTimestamp(g.ImportedAt) + v.ImportedAt = &at + } + var det importer.Detail + if len(g.ImportDetail) > 0 && json.Unmarshal(g.ImportDetail, &det) == nil && det.Verdict.Outcome != "" { + ev := det.Verdict.Evidence + v.Evidence, v.Edition, v.Skipped, v.RehomedFrom = &ev, det.Edition, det.Skipped, det.RehomedFrom + if det.Files != nil { + v.Files = det.Files + } + } + return v +} + +type downloadsPage struct { + Items []downloadView `json:"items"` + Limit int `json:"limit"` + Offset int `json:"offset"` +} + +// handleListDownloads implements GET /api/admin/downloads?offset=: running +// downloads first, then the rest newest first. +func (h *handlers) handleListDownloads(w http.ResponseWriter, r *http.Request) { + limit, offset, err := parsePaging(r.URL.Query()) + if err != nil { + writeErr(w, apierror.BadRequest("invalid_paging", "limit and offset must be numbers")) + return + } + rows, err := dbq.New(h.pool).ListGrabs(r.Context(), dbq.ListGrabsParams{MaxRows: int32(limit), Skip: int32(offset)}) + if err != nil { + writeErrWithLog(w, h.logger, "admin downloads: list", apierror.Internal(err)) + return + } + out := downloadsPage{Items: make([]downloadView, 0, len(rows)), Limit: limit, Offset: offset} + for _, g := range rows { + out.Items = append(out.Items, downloadFrom(g)) + } + writeJSON(w, http.StatusOK, out) +} + +// handleImportAnyway implements POST /api/admin/downloads/{id}/import: a +// download the check held for a person (or one that failed) is imported on +// the next pass whatever the check says. +func (h *handlers) handleImportAnyway(w http.ResponseWriter, r *http.Request) { + id, ok := requireURLUUID(w, r, "id") + if !ok { + return + } + if _, err := dbq.New(h.pool).ForceGrabImport(r.Context(), id); err != nil { + h.writeImportErr(w, "import anyway", err) + return + } + w.WriteHeader(http.StatusNoContent) +} + +// handleRetryImport implements POST /api/admin/downloads/{id}/retry. +func (h *handlers) handleRetryImport(w http.ResponseWriter, r *http.Request) { + id, ok := requireURLUUID(w, r, "id") + if !ok { + return + } + if _, err := dbq.New(h.pool).RetryGrabImport(r.Context(), id); err != nil { + h.writeImportErr(w, "retry", err) + return + } + w.WriteHeader(http.StatusNoContent) +} + +// handleRefuseDownload implements POST /api/admin/downloads/{id}/refuse: the +// release is blocked, the download removed and the album searched again. +func (h *handlers) handleRefuseDownload(w http.ResponseWriter, r *http.Request) { + admin, ok := requireUser(w, r) + if !ok { + return + } + id, ok := requireURLUUID(w, r, "id") + if !ok { + return + } + if _, err := h.importer.Refuse(r.Context(), id, admin.ID); err != nil { + h.writeImportErr(w, "refuse", err) + return + } + w.WriteHeader(http.StatusNoContent) +} + +func (h *handlers) writeImportErr(w http.ResponseWriter, what string, err error) { + switch { + case errors.Is(err, pgx.ErrNoRows), errors.Is(err, importer.ErrNotDecidable): + writeErr(w, apierror.Conflict("not_decidable", "that download is not waiting on a decision")) + default: + writeErrWithLog(w, h.logger, "admin downloads: "+what, apierror.Internal(err)) + } +} + +type releaseBlockView struct { + InfoHash string `json:"info_hash"` + Name string `json:"name"` + Reason string `json:"reason"` + Detail string `json:"detail"` + BlockedAt string `json:"blocked_at"` + WantTitle string `json:"want_title"` + WantArtist string `json:"want_artist"` +} + +type releaseBlocksPage struct { + Items []releaseBlockView `json:"items"` + Limit int `json:"limit"` + Offset int `json:"offset"` +} + +// handleListReleaseBlocks implements GET /api/admin/release-blocks?offset=: +// every release set aside, by a person or by a check, newest first. +func (h *handlers) handleListReleaseBlocks(w http.ResponseWriter, r *http.Request) { + limit, offset, err := parsePaging(r.URL.Query()) + if err != nil { + writeErr(w, apierror.BadRequest("invalid_paging", "limit and offset must be numbers")) + return + } + rows, err := dbq.New(h.pool).ListReleaseBlocksPage(r.Context(), dbq.ListReleaseBlocksPageParams{ + MaxRows: int32(limit), Skip: int32(offset)}) + if err != nil { + writeErrWithLog(w, h.logger, "admin release blocks: list", apierror.Internal(err)) + return + } + out := releaseBlocksPage{Items: make([]releaseBlockView, 0, len(rows)), Limit: limit, Offset: offset} + for _, b := range rows { + v := releaseBlockView{InfoHash: b.InfoHash, Name: b.Name, Reason: b.Reason, Detail: b.Detail, + BlockedAt: formatTimestamp(b.BlockedAt)} + if b.WantTitle != nil { + v.WantTitle = *b.WantTitle + } + if b.WantArtist != nil { + v.WantArtist = *b.WantArtist + } + out.Items = append(out.Items, v) + } + writeJSON(w, http.StatusOK, out) +} diff --git a/internal/api/admin_search.go b/internal/api/admin_search.go index 53b7dbe0..84a17101 100644 --- a/internal/api/admin_search.go +++ b/internal/api/admin_search.go @@ -5,6 +5,7 @@ import ( "encoding/json" "errors" "net/http" + "slices" "time" "github.com/go-chi/chi/v5" @@ -32,12 +33,20 @@ type searchSettingsView struct { Tiers []release.TierInfo `json:"tiers"` LastPassAt *string `json:"last_pass_at"` LastPass json.RawMessage `json:"last_pass"` + // ImportRoot is the library folder new music is filed into (#5522); "" + // is the first of ScanPaths, the library's folders. + ImportRoot string `json:"import_root"` + ScanPaths []string `json:"scan_paths"` } -func searchSettings(s dbq.AcquisitionSetting) searchSettingsView { +func searchSettings(s dbq.AcquisitionSetting, roots []string) searchSettingsView { v := searchSettingsView{ AutoGrab: s.AutoGrab, SearchIntervalHours: s.SearchIntervalHours, AllowedQualities: s.AllowedQualities, Tiers: release.Tiers, LastPass: s.LastPass, + ImportRoot: s.ImportRoot, ScanPaths: roots, + } + if v.ScanPaths == nil { + v.ScanPaths = []string{} } if v.AllowedQualities == nil { v.AllowedQualities = []string{} @@ -59,13 +68,14 @@ func (h *handlers) handleGetSearchSettings(w http.ResponseWriter, r *http.Reques writeErrWithLog(w, h.logger, "admin search settings: get", apierror.Internal(err)) return } - writeJSON(w, http.StatusOK, searchSettings(s)) + writeJSON(w, http.StatusOK, searchSettings(s, h.importer.Roots)) } type putSearchSettingsBody struct { AutoGrab bool `json:"auto_grab"` SearchIntervalHours int32 `json:"search_interval_hours"` AllowedQualities []string `json:"allowed_qualities"` + ImportRoot string `json:"import_root"` } // handlePutSearchSettings implements PUT /api/admin/acquisition/settings. @@ -88,14 +98,19 @@ func (h *handlers) handlePutSearchSettings(w http.ResponseWriter, r *http.Reques return } } + if body.ImportRoot != "" && !slices.Contains(h.importer.Roots, body.ImportRoot) { + writeErr(w, apierror.BadRequest("bad_import_root", "file new music into one of the library's folders")) + return + } s, err := dbq.New(h.pool).UpdateAcquisitionSettings(r.Context(), dbq.UpdateAcquisitionSettingsParams{ AutoGrab: body.AutoGrab, SearchIntervalHours: body.SearchIntervalHours, AllowedQualities: body.AllowedQualities, + ImportRoot: body.ImportRoot, }) if err != nil { writeErrWithLog(w, h.logger, "admin search settings: put", apierror.Internal(err)) return } - writeJSON(w, http.StatusOK, searchSettings(s)) + writeJSON(w, http.StatusOK, searchSettings(s, h.importer.Roots)) } // candidateView is one release a want's last search found. diff --git a/internal/api/admin_wanted.go b/internal/api/admin_wanted.go index 6c740ab6..d5cc061f 100644 --- a/internal/api/admin_wanted.go +++ b/internal/api/admin_wanted.go @@ -49,11 +49,21 @@ type wantSearchView struct { Error string `json:"error"` } +// wantGrabView is the want's latest download, running or ended, and what +// its import made of it (#5522). type wantGrabView struct { - Name string `json:"name"` - GrabbedAt string `json:"grabbed_at"` - Origin string `json:"origin"` - Quality string `json:"quality"` + ID string `json:"id"` + Name string `json:"name"` + GrabbedAt string `json:"grabbed_at"` + Origin string `json:"origin"` + Quality string `json:"quality"` + Live bool `json:"live"` + EndReason string `json:"end_reason"` + Progress float32 `json:"progress"` + ImportState string `json:"import_state"` + ImportReason string `json:"import_reason"` + ImportedAt *string `json:"imported_at"` + ImportedFiles int32 `json:"imported_files"` } // wantCounts sizes each tab of the Wanted page. @@ -62,6 +72,11 @@ type wantCounts struct { Missing int64 `json:"missing"` Held int64 `json:"held"` Following int64 `json:"following"` + // Activity: downloads running; Attention: of those, the ones waiting on + // a person; Blocked: releases set aside (#5522). + Activity int64 `json:"activity"` + Attention int64 `json:"attention"` + Blocked int64 `json:"blocked"` } type wantsPage struct { @@ -102,7 +117,8 @@ func (h *handlers) handleListWants(w http.ResponseWriter, r *http.Request) { } out := wantsPage{ Items: make([]wantView, 0, len(rows)), Limit: wantsPageSize, Offset: offset, - Counts: wantCounts{Wanted: counts.Wanted, Missing: counts.Missing, Held: counts.Held, Following: counts.Following}, + Counts: wantCounts{Wanted: counts.Wanted, Missing: counts.Missing, Held: counts.Held, Following: counts.Following, + Activity: counts.Activity, Attention: counts.Attention, Blocked: counts.Blocked}, } switch state { case "missing": @@ -145,12 +161,19 @@ func (h *handlers) handleListWants(w http.ResponseWriter, r *http.Request) { } } if row.GrabName != nil { - v.Grab = &wantGrabView{Name: *row.GrabName, GrabbedAt: formatTimestamp(row.GrabbedAt)} - if row.GrabOrigin != nil { - v.Grab.Origin = *row.GrabOrigin + v.Grab = &wantGrabView{ID: uuidToString(row.GrabID), Name: *row.GrabName, + GrabbedAt: formatTimestamp(row.GrabbedAt), Live: !row.GrabEndedAt.Valid, + Origin: deref(row.GrabOrigin), Quality: deref(row.GrabQuality), EndReason: deref(row.GrabEndReason), + ImportState: deref(row.GrabImportState), ImportReason: deref(row.GrabImportReason)} + if row.GrabProgress != nil { + v.Grab.Progress = *row.GrabProgress } - if row.GrabQuality != nil { - v.Grab.Quality = *row.GrabQuality + if row.GrabImportedFiles != nil { + v.Grab.ImportedFiles = *row.GrabImportedFiles + } + if row.GrabImportedAt.Valid { + at := formatTimestamp(row.GrabImportedAt) + v.Grab.ImportedAt = &at } } out.Items = append(out.Items, v) @@ -425,3 +448,11 @@ func (h *handlers) writeProfileErr(w http.ResponseWriter, err error) { writeErrWithLog(w, h.logger, "admin release profiles", apierror.Internal(err)) } } + +// deref is a nullable column's text, or "". +func deref(s *string) string { + if s == nil { + return "" + } + return *s +} diff --git a/internal/api/api.go b/internal/api/api.go index 08d2d231..fc4d6133 100644 --- a/internal/api/api.go +++ b/internal/api/api.go @@ -19,6 +19,7 @@ import ( "git.fabledsword.com/bvandeusen/minstrel/internal/config" "git.fabledsword.com/bvandeusen/minstrel/internal/coverart" "git.fabledsword.com/bvandeusen/minstrel/internal/eventbus" + "git.fabledsword.com/bvandeusen/minstrel/internal/importer" "git.fabledsword.com/bvandeusen/minstrel/internal/library" "git.fabledsword.com/bvandeusen/minstrel/internal/lidarrconfig" "git.fabledsword.com/bvandeusen/minstrel/internal/lidarrquarantine" @@ -78,6 +79,7 @@ func Mount(r chi.Router, pool *pgxpool.Pool, logger *slog.Logger, events *playev catalogue: cat, wanted: wanted.New(pool, cat, logger), searcher: wanted.NewSearcher(pool, cat, logger), + importer: importer.New(pool, scanner, cat, nil, scannerRoots(scanner), logger), catalogueSearches: newSearchCache(), catalogueArt: catalogue.NewArt(dataDir, artSources(coverSettings)), leveled: newLeveledRenderer(dataDir, loudSettings, logger), @@ -257,6 +259,11 @@ func Mount(r chi.Router, pool *pgxpool.Pool, logger *slog.Logger, events *playev admin.Post("/wanted/{id}/grab", h.handleGrabCandidate) admin.Post("/wanted/{id}/block", h.handleBlockCandidate) admin.Delete("/release-blocks/{hash}", h.handleUnblockRelease) + admin.Get("/release-blocks", h.handleListReleaseBlocks) + admin.Get("/downloads", h.handleListDownloads) + admin.Post("/downloads/{id}/import", h.handleImportAnyway) + admin.Post("/downloads/{id}/retry", h.handleRetryImport) + admin.Post("/downloads/{id}/refuse", h.handleRefuseDownload) admin.Get("/follows", h.handleListFollows) admin.Get("/follows/{mbid}", h.handleGetFollow) admin.Put("/follows/{mbid}", h.handlePutFollow) @@ -427,6 +434,9 @@ type handlers struct { wanted *wanted.Service // searcher searches for wants and grabs releases for them (#5521). searcher *wanted.Searcher + // importer refuses a download at a person's word (#5522). The import + // loop itself runs in main; this one only acts on a request. + importer *importer.Importer // catalogueSearches reuses recent MusicBrainz answers; nil asks every // time, which is what handler tests get. catalogueSearches *searchCache @@ -482,3 +492,12 @@ func artSources(s *coverart.SettingsService) catalogue.ArtSources { } return s } + +// scannerRoots are the library folders, or none when the API runs without a +// scanner (tests). +func scannerRoots(s *library.Scanner) []string { + if s == nil { + return nil + } + return s.Roots() +} diff --git a/internal/db/dbq/imports.sql.go b/internal/db/dbq/imports.sql.go new file mode 100644 index 00000000..df96ed4d --- /dev/null +++ b/internal/db/dbq/imports.sql.go @@ -0,0 +1,719 @@ +// Code generated by sqlc. DO NOT EDIT. +// versions: +// sqlc v1.31.1 +// source: imports.sql + +package dbq + +import ( + "context" + + "github.com/jackc/pgx/v5/pgtype" +) + +const claimGrabImport = `-- name: ClaimGrabImport :one +UPDATE grabs + SET import_state = 'importing', import_attempts = import_attempts + 1, + import_updated_at = now(), progress = 1 + WHERE id = $1 AND ended_at IS NULL AND import_state IN ('', 'downloading', 'waiting') +RETURNING id, want_id, info_hash, name, key, indexer, source, client, save_path, quality, origin, forced, grabbed_by, grabbed_at, ended_at, end_reason, end_detail, progress, import_state, import_reason, import_detail, import_forced, import_attempts, import_updated_at, imported_at, imported_files, destination +` + +// Takes a finished download for import. The update is the lock: a second +// pass finds nothing to claim. +func (q *Queries) ClaimGrabImport(ctx context.Context, id pgtype.UUID) (Grab, error) { + row := q.db.QueryRow(ctx, claimGrabImport, id) + var i Grab + err := row.Scan( + &i.ID, + &i.WantID, + &i.InfoHash, + &i.Name, + &i.Key, + &i.Indexer, + &i.Source, + &i.Client, + &i.SavePath, + &i.Quality, + &i.Origin, + &i.Forced, + &i.GrabbedBy, + &i.GrabbedAt, + &i.EndedAt, + &i.EndReason, + &i.EndDetail, + &i.Progress, + &i.ImportState, + &i.ImportReason, + &i.ImportDetail, + &i.ImportForced, + &i.ImportAttempts, + &i.ImportUpdatedAt, + &i.ImportedAt, + &i.ImportedFiles, + &i.Destination, + ) + return i, err +} + +const forceGrabImport = `-- name: ForceGrabImport :one +UPDATE grabs + SET import_state = '', import_forced = true, import_reason = '', import_updated_at = now() + WHERE id = $1 AND ended_at IS NULL AND import_state IN ('review', 'failed') +RETURNING id, want_id, info_hash, name, key, indexer, source, client, save_path, quality, origin, forced, grabbed_by, grabbed_at, ended_at, end_reason, end_detail, progress, import_state, import_reason, import_detail, import_forced, import_attempts, import_updated_at, imported_at, imported_files, destination +` + +// "Import anyway": the next pass imports the download whatever the check said. +func (q *Queries) ForceGrabImport(ctx context.Context, id pgtype.UUID) (Grab, error) { + row := q.db.QueryRow(ctx, forceGrabImport, id) + var i Grab + err := row.Scan( + &i.ID, + &i.WantID, + &i.InfoHash, + &i.Name, + &i.Key, + &i.Indexer, + &i.Source, + &i.Client, + &i.SavePath, + &i.Quality, + &i.Origin, + &i.Forced, + &i.GrabbedBy, + &i.GrabbedAt, + &i.EndedAt, + &i.EndReason, + &i.EndDetail, + &i.Progress, + &i.ImportState, + &i.ImportReason, + &i.ImportDetail, + &i.ImportForced, + &i.ImportAttempts, + &i.ImportUpdatedAt, + &i.ImportedAt, + &i.ImportedFiles, + &i.Destination, + ) + return i, err +} + +const getGrab = `-- name: GetGrab :one +SELECT id, want_id, info_hash, name, key, indexer, source, client, save_path, quality, origin, forced, grabbed_by, grabbed_at, ended_at, end_reason, end_detail, progress, import_state, import_reason, import_detail, import_forced, import_attempts, import_updated_at, imported_at, imported_files, destination FROM grabs WHERE id = $1 +` + +func (q *Queries) GetGrab(ctx context.Context, id pgtype.UUID) (Grab, error) { + row := q.db.QueryRow(ctx, getGrab, id) + var i Grab + err := row.Scan( + &i.ID, + &i.WantID, + &i.InfoHash, + &i.Name, + &i.Key, + &i.Indexer, + &i.Source, + &i.Client, + &i.SavePath, + &i.Quality, + &i.Origin, + &i.Forced, + &i.GrabbedBy, + &i.GrabbedAt, + &i.EndedAt, + &i.EndReason, + &i.EndDetail, + &i.Progress, + &i.ImportState, + &i.ImportReason, + &i.ImportDetail, + &i.ImportForced, + &i.ImportAttempts, + &i.ImportUpdatedAt, + &i.ImportedAt, + &i.ImportedFiles, + &i.Destination, + ) + return i, err +} + +const heldPositions = `-- name: HeldPositions :many +SELECT DISTINCT COALESCE(t.disc_number, 1)::int AS disc, t.track_number::int AS track + FROM tracks t + JOIN albums a ON a.id = t.album_id + WHERE a.release_group_mbid = $1 + AND t.missing_since IS NULL AND t.track_number IS NOT NULL +` + +type HeldPositionsRow struct { + Disc int32 + Track int32 +} + +// The disc and track positions the library holds for an album of this group. +func (q *Queries) HeldPositions(ctx context.Context, releaseGroupMbid *string) ([]HeldPositionsRow, error) { + rows, err := q.db.Query(ctx, heldPositions, releaseGroupMbid) + if err != nil { + return nil, err + } + defer rows.Close() + var items []HeldPositionsRow + for rows.Next() { + var i HeldPositionsRow + if err := rows.Scan(&i.Disc, &i.Track); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const heldRecordings = `-- name: HeldRecordings :many +SELECT DISTINCT mbid::text AS mbid FROM tracks + WHERE mbid = ANY($1::text[]) AND missing_since IS NULL +` + +// Which of these recordings the library holds a file for. +func (q *Queries) HeldRecordings(ctx context.Context, recordings []string) ([]string, error) { + rows, err := q.db.Query(ctx, heldRecordings, recordings) + if err != nil { + return nil, err + } + defer rows.Close() + var items []string + for rows.Next() { + var mbid string + if err := rows.Scan(&mbid); err != nil { + return nil, err + } + items = append(items, mbid) + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const listGrabs = `-- name: ListGrabs :many +SELECT g.id, g.want_id, g.name, g.indexer, g.client, g.quality, g.origin, g.forced, + g.grabbed_at, g.ended_at, g.end_reason, g.end_detail, g.progress, g.import_state, + g.import_reason, g.import_detail, g.import_updated_at, g.imported_at, g.imported_files, + g.destination, w.title AS want_title, w.artist_name AS want_artist + FROM grabs g + LEFT JOIN wants w ON w.id = g.want_id + ORDER BY (g.ended_at IS NULL AND g.import_state IN ('review', 'failed')) DESC, + g.ended_at IS NULL DESC, g.grabbed_at DESC, g.id + LIMIT $2 OFFSET $1 +` + +type ListGrabsParams struct { + Skip int32 + MaxRows int32 +} + +type ListGrabsRow struct { + ID pgtype.UUID + WantID pgtype.UUID + Name string + Indexer string + Client string + Quality string + Origin string + Forced bool + GrabbedAt pgtype.Timestamptz + EndedAt pgtype.Timestamptz + EndReason string + EndDetail string + Progress float32 + ImportState string + ImportReason string + ImportDetail []byte + ImportUpdatedAt pgtype.Timestamptz + ImportedAt pgtype.Timestamptz + ImportedFiles int32 + Destination string + WantTitle *string + WantArtist *string +} + +// The Wanted view's Activity tab: downloads waiting on a person first, then +// the rest still running, then the ended ones newest first. +func (q *Queries) ListGrabs(ctx context.Context, arg ListGrabsParams) ([]ListGrabsRow, error) { + rows, err := q.db.Query(ctx, listGrabs, arg.Skip, arg.MaxRows) + if err != nil { + return nil, err + } + defer rows.Close() + var items []ListGrabsRow + for rows.Next() { + var i ListGrabsRow + if err := rows.Scan( + &i.ID, + &i.WantID, + &i.Name, + &i.Indexer, + &i.Client, + &i.Quality, + &i.Origin, + &i.Forced, + &i.GrabbedAt, + &i.EndedAt, + &i.EndReason, + &i.EndDetail, + &i.Progress, + &i.ImportState, + &i.ImportReason, + &i.ImportDetail, + &i.ImportUpdatedAt, + &i.ImportedAt, + &i.ImportedFiles, + &i.Destination, + &i.WantTitle, + &i.WantArtist, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const listLiveGrabs = `-- name: ListLiveGrabs :many + +SELECT g.id, g.want_id, g.info_hash, g.name, g.key, g.indexer, g.source, g.client, g.save_path, g.quality, g.origin, g.forced, g.grabbed_by, g.grabbed_at, g.ended_at, g.end_reason, g.end_detail, g.progress, g.import_state, g.import_reason, g.import_detail, g.import_forced, g.import_attempts, g.import_updated_at, g.imported_at, g.imported_files, g.destination, w.release_group_mbid, w.artist_mbid, w.artist_name, w.title AS want_title, + w.reason AS want_reason, w.album_id AS want_album_id + FROM grabs g + LEFT JOIN wants w ON w.id = g.want_id + WHERE g.ended_at IS NULL + ORDER BY g.grabbed_at, g.id +` + +type ListLiveGrabsRow struct { + ID pgtype.UUID + WantID pgtype.UUID + InfoHash string + Name string + Key string + Indexer string + Source string + Client string + SavePath string + Quality string + Origin string + Forced bool + GrabbedBy pgtype.UUID + GrabbedAt pgtype.Timestamptz + EndedAt pgtype.Timestamptz + EndReason string + EndDetail string + Progress float32 + ImportState string + ImportReason string + ImportDetail []byte + ImportForced bool + ImportAttempts int32 + ImportUpdatedAt pgtype.Timestamptz + ImportedAt pgtype.Timestamptz + ImportedFiles int32 + Destination string + ReleaseGroupMbid *string + ArtistMbid *string + ArtistName *string + WantTitle *string + WantReason *string + WantAlbumID pgtype.UUID +} + +// Import with verification (M502 #5522). +// Every download still running or waiting on its import, oldest first. +func (q *Queries) ListLiveGrabs(ctx context.Context) ([]ListLiveGrabsRow, error) { + rows, err := q.db.Query(ctx, listLiveGrabs) + if err != nil { + return nil, err + } + defer rows.Close() + var items []ListLiveGrabsRow + for rows.Next() { + var i ListLiveGrabsRow + if err := rows.Scan( + &i.ID, + &i.WantID, + &i.InfoHash, + &i.Name, + &i.Key, + &i.Indexer, + &i.Source, + &i.Client, + &i.SavePath, + &i.Quality, + &i.Origin, + &i.Forced, + &i.GrabbedBy, + &i.GrabbedAt, + &i.EndedAt, + &i.EndReason, + &i.EndDetail, + &i.Progress, + &i.ImportState, + &i.ImportReason, + &i.ImportDetail, + &i.ImportForced, + &i.ImportAttempts, + &i.ImportUpdatedAt, + &i.ImportedAt, + &i.ImportedFiles, + &i.Destination, + &i.ReleaseGroupMbid, + &i.ArtistMbid, + &i.ArtistName, + &i.WantTitle, + &i.WantReason, + &i.WantAlbumID, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const listReleaseBlocksPage = `-- name: ListReleaseBlocksPage :many +SELECT b.info_hash, b.name, b.reason, b.detail, b.blocked_at, w.title AS want_title, + w.artist_name AS want_artist + FROM release_blocks b + LEFT JOIN wants w ON w.id = b.want_id + ORDER BY b.blocked_at DESC, b.info_hash + LIMIT $2 OFFSET $1 +` + +type ListReleaseBlocksPageParams struct { + Skip int32 + MaxRows int32 +} + +type ListReleaseBlocksPageRow struct { + InfoHash string + Name string + Reason string + Detail string + BlockedAt pgtype.Timestamptz + WantTitle *string + WantArtist *string +} + +func (q *Queries) ListReleaseBlocksPage(ctx context.Context, arg ListReleaseBlocksPageParams) ([]ListReleaseBlocksPageRow, error) { + rows, err := q.db.Query(ctx, listReleaseBlocksPage, arg.Skip, arg.MaxRows) + if err != nil { + return nil, err + } + defer rows.Close() + var items []ListReleaseBlocksPageRow + for rows.Next() { + var i ListReleaseBlocksPageRow + if err := rows.Scan( + &i.InfoHash, + &i.Name, + &i.Reason, + &i.Detail, + &i.BlockedAt, + &i.WantTitle, + &i.WantArtist, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const markWantSearched = `-- name: MarkWantSearched :exec +UPDATE wants SET searched_at = now() WHERE id = $1 +` + +// Imported: nothing to search for until the wanted pass settles the want. +func (q *Queries) MarkWantSearched(ctx context.Context, id pgtype.UUID) error { + _, err := q.db.Exec(ctx, markWantSearched, id) + return err +} + +const openWantsForGroups = `-- name: OpenWantsForGroups :many +SELECT w.id, w.release_group_mbid, w.artist_mbid, w.artist_name, w.title, w.primary_type, w.first_release_date, w.reason, w.album_id, w.created_by, w.wanted_at, w.fulfilled_at, w.created_at, w.updated_at, w.searched_at, w.search_outcome, w.search_considered, w.best_release, w.best_verdict, w.search_error FROM wants w + WHERE w.fulfilled_at IS NULL AND w.release_group_mbid = ANY($1::text[]) + AND NOT EXISTS (SELECT 1 FROM grabs g WHERE g.want_id = w.id AND g.ended_at IS NULL) +` + +// Open wants for these release groups with no download running. +func (q *Queries) OpenWantsForGroups(ctx context.Context, groups []string) ([]Want, error) { + rows, err := q.db.Query(ctx, openWantsForGroups, groups) + if err != nil { + return nil, err + } + defer rows.Close() + var items []Want + for rows.Next() { + var i Want + if err := rows.Scan( + &i.ID, + &i.ReleaseGroupMbid, + &i.ArtistMbid, + &i.ArtistName, + &i.Title, + &i.PrimaryType, + &i.FirstReleaseDate, + &i.Reason, + &i.AlbumID, + &i.CreatedBy, + &i.WantedAt, + &i.FulfilledAt, + &i.CreatedAt, + &i.UpdatedAt, + &i.SearchedAt, + &i.SearchOutcome, + &i.SearchConsidered, + &i.BestRelease, + &i.BestVerdict, + &i.SearchError, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const reclaimStaleImports = `-- name: ReclaimStaleImports :execrows +UPDATE grabs + SET import_state = 'waiting', + import_reason = 'The last import was cut short, so it is being tried again.' + WHERE ended_at IS NULL AND import_state = 'importing' + AND import_updated_at < now() - make_interval(mins => $1::int) +` + +// An import cut short (a restart mid-copy) is tried again. +func (q *Queries) ReclaimStaleImports(ctx context.Context, minutes int32) (int64, error) { + result, err := q.db.Exec(ctx, reclaimStaleImports, minutes) + if err != nil { + return 0, err + } + return result.RowsAffected(), nil +} + +const recordImport = `-- name: RecordImport :one +UPDATE grabs + SET import_state = $1, + import_reason = $2, + import_detail = $3, + import_updated_at = now(), + import_forced = false, + imported_at = CASE WHEN $1::text = 'imported' THEN now() ELSE imported_at END, + imported_files = $4, + destination = $5, + ended_at = CASE WHEN $6::text <> '' THEN now() ELSE ended_at END, + end_reason = $6, + end_detail = CASE WHEN $6::text <> '' THEN $2::text ELSE end_detail END + WHERE id = $7 +RETURNING id, want_id, info_hash, name, key, indexer, source, client, save_path, quality, origin, forced, grabbed_by, grabbed_at, ended_at, end_reason, end_detail, progress, import_state, import_reason, import_detail, import_forced, import_attempts, import_updated_at, imported_at, imported_files, destination +` + +type RecordImportParams struct { + ImportState string + ImportReason string + ImportDetail []byte + ImportedFiles int32 + Destination string + EndReason string + ID pgtype.UUID +} + +// The import's outcome. An end reason ends the grab: imported, refused or +// held. Without one the grab stays live: review and failed wait for a person. +func (q *Queries) RecordImport(ctx context.Context, arg RecordImportParams) (Grab, error) { + row := q.db.QueryRow(ctx, recordImport, + arg.ImportState, + arg.ImportReason, + arg.ImportDetail, + arg.ImportedFiles, + arg.Destination, + arg.EndReason, + arg.ID, + ) + var i Grab + err := row.Scan( + &i.ID, + &i.WantID, + &i.InfoHash, + &i.Name, + &i.Key, + &i.Indexer, + &i.Source, + &i.Client, + &i.SavePath, + &i.Quality, + &i.Origin, + &i.Forced, + &i.GrabbedBy, + &i.GrabbedAt, + &i.EndedAt, + &i.EndReason, + &i.EndDetail, + &i.Progress, + &i.ImportState, + &i.ImportReason, + &i.ImportDetail, + &i.ImportForced, + &i.ImportAttempts, + &i.ImportUpdatedAt, + &i.ImportedAt, + &i.ImportedFiles, + &i.Destination, + ) + return i, err +} + +const rehomeGrab = `-- name: RehomeGrab :one +UPDATE grabs g + SET want_id = $1 + WHERE g.id = $2 AND g.ended_at IS NULL + AND NOT EXISTS (SELECT 1 FROM grabs o WHERE o.want_id = $1 AND o.ended_at IS NULL) +RETURNING id, want_id, info_hash, name, key, indexer, source, client, save_path, quality, origin, forced, grabbed_by, grabbed_at, ended_at, end_reason, end_detail, progress, import_state, import_reason, import_detail, import_forced, import_attempts, import_updated_at, imported_at, imported_files, destination +` + +type RehomeGrabParams struct { + WantID pgtype.UUID + ID pgtype.UUID +} + +// A download that turned out to be another wanted album is that want's grab, +// when the want has none running. +func (q *Queries) RehomeGrab(ctx context.Context, arg RehomeGrabParams) (Grab, error) { + row := q.db.QueryRow(ctx, rehomeGrab, arg.WantID, arg.ID) + var i Grab + err := row.Scan( + &i.ID, + &i.WantID, + &i.InfoHash, + &i.Name, + &i.Key, + &i.Indexer, + &i.Source, + &i.Client, + &i.SavePath, + &i.Quality, + &i.Origin, + &i.Forced, + &i.GrabbedBy, + &i.GrabbedAt, + &i.EndedAt, + &i.EndReason, + &i.EndDetail, + &i.Progress, + &i.ImportState, + &i.ImportReason, + &i.ImportDetail, + &i.ImportForced, + &i.ImportAttempts, + &i.ImportUpdatedAt, + &i.ImportedAt, + &i.ImportedFiles, + &i.Destination, + ) + return i, err +} + +const retryGrabImport = `-- name: RetryGrabImport :one +UPDATE grabs + SET import_state = '', import_reason = '', import_updated_at = now() + WHERE id = $1 AND ended_at IS NULL AND import_state = 'failed' +RETURNING id, want_id, info_hash, name, key, indexer, source, client, save_path, quality, origin, forced, grabbed_by, grabbed_at, ended_at, end_reason, end_detail, progress, import_state, import_reason, import_detail, import_forced, import_attempts, import_updated_at, imported_at, imported_files, destination +` + +func (q *Queries) RetryGrabImport(ctx context.Context, id pgtype.UUID) (Grab, error) { + row := q.db.QueryRow(ctx, retryGrabImport, id) + var i Grab + err := row.Scan( + &i.ID, + &i.WantID, + &i.InfoHash, + &i.Name, + &i.Key, + &i.Indexer, + &i.Source, + &i.Client, + &i.SavePath, + &i.Quality, + &i.Origin, + &i.Forced, + &i.GrabbedBy, + &i.GrabbedAt, + &i.EndedAt, + &i.EndReason, + &i.EndDetail, + &i.Progress, + &i.ImportState, + &i.ImportReason, + &i.ImportDetail, + &i.ImportForced, + &i.ImportAttempts, + &i.ImportUpdatedAt, + &i.ImportedAt, + &i.ImportedFiles, + &i.Destination, + ) + return i, err +} + +const searchWantAgain = `-- name: SearchWantAgain :exec +UPDATE wants SET searched_at = NULL, search_outcome = '' WHERE id = $1 +` + +// Refused: search again on the next pass. +func (q *Queries) SearchWantAgain(ctx context.Context, id pgtype.UUID) error { + _, err := q.db.Exec(ctx, searchWantAgain, id) + return err +} + +const setGrabProgress = `-- name: SetGrabProgress :exec +UPDATE grabs + SET progress = $1, + import_state = $2, + import_reason = $3, + import_updated_at = now() + WHERE id = $4 AND ended_at IS NULL + AND import_state IN ('', 'downloading', 'waiting') +` + +type SetGrabProgressParams struct { + Progress float32 + ImportState string + ImportReason string + ID pgtype.UUID +} + +// What the download client says of a download not yet finished, or finished +// and waiting for what the import needs. Leaves a claimed, decided or failed +// import alone. +func (q *Queries) SetGrabProgress(ctx context.Context, arg SetGrabProgressParams) error { + _, err := q.db.Exec(ctx, setGrabProgress, + arg.Progress, + arg.ImportState, + arg.ImportReason, + arg.ID, + ) + return err +} diff --git a/internal/db/dbq/models.go b/internal/db/dbq/models.go index 0d4214dd..c42a4152 100644 --- a/internal/db/dbq/models.go +++ b/internal/db/dbq/models.go @@ -225,6 +225,7 @@ type AcquisitionSetting struct { LastPassAt pgtype.Timestamptz LastPass []byte UpdatedAt pgtype.Timestamptz + ImportRoot string } type Album struct { @@ -431,23 +432,33 @@ type GeneralLikesArtist struct { } type Grab struct { - ID pgtype.UUID - WantID pgtype.UUID - InfoHash string - Name string - Key string - Indexer string - Source string - Client string - SavePath string - Quality string - Origin string - Forced bool - GrabbedBy pgtype.UUID - GrabbedAt pgtype.Timestamptz - EndedAt pgtype.Timestamptz - EndReason string - EndDetail string + ID pgtype.UUID + WantID pgtype.UUID + InfoHash string + Name string + Key string + Indexer string + Source string + Client string + SavePath string + Quality string + Origin string + Forced bool + GrabbedBy pgtype.UUID + GrabbedAt pgtype.Timestamptz + EndedAt pgtype.Timestamptz + EndReason string + EndDetail string + Progress float32 + ImportState string + ImportReason string + ImportDetail []byte + ImportForced bool + ImportAttempts int32 + ImportUpdatedAt pgtype.Timestamptz + ImportedAt pgtype.Timestamptz + ImportedFiles int32 + Destination string } type LibraryChange struct { diff --git a/internal/db/dbq/search.sql.go b/internal/db/dbq/search.sql.go index b1043de2..9ecde58f 100644 --- a/internal/db/dbq/search.sql.go +++ b/internal/db/dbq/search.sql.go @@ -89,7 +89,7 @@ func (q *Queries) EndGrab(ctx context.Context, arg EndGrabParams) error { const getAcquisitionSettings = `-- name: GetAcquisitionSettings :one -SELECT id, auto_grab, search_interval_hours, allowed_qualities, last_pass_at, last_pass, updated_at FROM acquisition_settings WHERE id = true +SELECT id, auto_grab, search_interval_hours, allowed_qualities, last_pass_at, last_pass, updated_at, import_root FROM acquisition_settings WHERE id = true ` // Searching for wants and grabbing releases (M502 #5521). @@ -104,12 +104,13 @@ func (q *Queries) GetAcquisitionSettings(ctx context.Context) (AcquisitionSettin &i.LastPassAt, &i.LastPass, &i.UpdatedAt, + &i.ImportRoot, ) return i, err } const getLiveGrab = `-- name: GetLiveGrab :one -SELECT id, want_id, info_hash, name, key, indexer, source, client, save_path, quality, origin, forced, grabbed_by, grabbed_at, ended_at, end_reason, end_detail FROM grabs WHERE want_id = $1 AND ended_at IS NULL +SELECT id, want_id, info_hash, name, key, indexer, source, client, save_path, quality, origin, forced, grabbed_by, grabbed_at, ended_at, end_reason, end_detail, progress, import_state, import_reason, import_detail, import_forced, import_attempts, import_updated_at, imported_at, imported_files, destination FROM grabs WHERE want_id = $1 AND ended_at IS NULL ` func (q *Queries) GetLiveGrab(ctx context.Context, wantID pgtype.UUID) (Grab, error) { @@ -133,6 +134,16 @@ func (q *Queries) GetLiveGrab(ctx context.Context, wantID pgtype.UUID) (Grab, er &i.EndedAt, &i.EndReason, &i.EndDetail, + &i.Progress, + &i.ImportState, + &i.ImportReason, + &i.ImportDetail, + &i.ImportForced, + &i.ImportAttempts, + &i.ImportUpdatedAt, + &i.ImportedAt, + &i.ImportedFiles, + &i.Destination, ) return i, err } @@ -173,7 +184,7 @@ func (q *Queries) GetWantCandidate(ctx context.Context, arg GetWantCandidatePara } const getWantForSearch = `-- name: GetWantForSearch :one -SELECT w.id, w.release_group_mbid, w.artist_name, w.title, w.reason, w.fulfilled_at, +SELECT w.id, w.release_group_mbid, w.artist_mbid, w.artist_name, w.title, w.reason, w.fulfilled_at, COALESCE(NULLIF(w.primary_type, ''), g.primary_type, '')::text AS primary_type, COALESCE(g.secondary_types, '{}')::text[] AS secondary_types, COALESCE(NULLIF(w.first_release_date, ''), g.first_release_date, '')::text AS first_release_date @@ -185,6 +196,7 @@ SELECT w.id, w.release_group_mbid, w.artist_name, w.title, w.reason, w.fulfilled type GetWantForSearchRow struct { ID pgtype.UUID ReleaseGroupMbid string + ArtistMbid string ArtistName string Title string Reason string @@ -200,6 +212,7 @@ func (q *Queries) GetWantForSearch(ctx context.Context, id pgtype.UUID) (GetWant err := row.Scan( &i.ID, &i.ReleaseGroupMbid, + &i.ArtistMbid, &i.ArtistName, &i.Title, &i.Reason, @@ -217,7 +230,7 @@ INSERT INTO grabs (want_id, info_hash, name, key, indexer, source, client, save_ VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12) -RETURNING id, want_id, info_hash, name, key, indexer, source, client, save_path, quality, origin, forced, grabbed_by, grabbed_at, ended_at, end_reason, end_detail +RETURNING id, want_id, info_hash, name, key, indexer, source, client, save_path, quality, origin, forced, grabbed_by, grabbed_at, ended_at, end_reason, end_detail, progress, import_state, import_reason, import_detail, import_forced, import_attempts, import_updated_at, imported_at, imported_files, destination ` type InsertGrabParams struct { @@ -269,6 +282,16 @@ func (q *Queries) InsertGrab(ctx context.Context, arg InsertGrabParams) (Grab, e &i.EndedAt, &i.EndReason, &i.EndDetail, + &i.Progress, + &i.ImportState, + &i.ImportReason, + &i.ImportDetail, + &i.ImportForced, + &i.ImportAttempts, + &i.ImportUpdatedAt, + &i.ImportedAt, + &i.ImportedFiles, + &i.Destination, ) return i, err } @@ -558,19 +581,26 @@ UPDATE acquisition_settings SET auto_grab = $1, search_interval_hours = $2, allowed_qualities = $3, + import_root = $4, updated_at = now() WHERE id = true -RETURNING id, auto_grab, search_interval_hours, allowed_qualities, last_pass_at, last_pass, updated_at +RETURNING id, auto_grab, search_interval_hours, allowed_qualities, last_pass_at, last_pass, updated_at, import_root ` type UpdateAcquisitionSettingsParams struct { AutoGrab bool SearchIntervalHours int32 AllowedQualities []string + ImportRoot string } func (q *Queries) UpdateAcquisitionSettings(ctx context.Context, arg UpdateAcquisitionSettingsParams) (AcquisitionSetting, error) { - row := q.db.QueryRow(ctx, updateAcquisitionSettings, arg.AutoGrab, arg.SearchIntervalHours, arg.AllowedQualities) + row := q.db.QueryRow(ctx, updateAcquisitionSettings, + arg.AutoGrab, + arg.SearchIntervalHours, + arg.AllowedQualities, + arg.ImportRoot, + ) var i AcquisitionSetting err := row.Scan( &i.ID, @@ -580,6 +610,7 @@ func (q *Queries) UpdateAcquisitionSettings(ctx context.Context, arg UpdateAcqui &i.LastPassAt, &i.LastPass, &i.UpdatedAt, + &i.ImportRoot, ) return i, err } diff --git a/internal/db/dbq/wanted.sql.go b/internal/db/dbq/wanted.sql.go index 1374189c..3f467f48 100644 --- a/internal/db/dbq/wanted.sql.go +++ b/internal/db/dbq/wanted.sql.go @@ -24,7 +24,13 @@ const countWants = `-- name: CountWants :one SELECT COUNT(*) FILTER (WHERE fulfilled_at IS NULL AND reason <> 'missing')::bigint AS wanted, COUNT(*) FILTER (WHERE fulfilled_at IS NULL AND reason = 'missing')::bigint AS missing, COUNT(*) FILTER (WHERE fulfilled_at IS NOT NULL)::bigint AS held, - (SELECT COUNT(*) FROM follows)::bigint AS following + (SELECT COUNT(*) FROM follows)::bigint AS following, + -- The Wanted view's other tabs (#5522): downloads running, those + -- waiting on a person, and releases set aside. + (SELECT COUNT(*) FROM grabs WHERE ended_at IS NULL)::bigint AS activity, + (SELECT COUNT(*) FROM grabs + WHERE ended_at IS NULL AND import_state IN ('review', 'failed'))::bigint AS attention, + (SELECT COUNT(*) FROM release_blocks)::bigint AS blocked FROM wants ` @@ -33,6 +39,9 @@ type CountWantsRow struct { Missing int64 Held int64 Following int64 + Activity int64 + Attention int64 + Blocked int64 } func (q *Queries) CountWants(ctx context.Context) (CountWantsRow, error) { @@ -43,6 +52,9 @@ func (q *Queries) CountWants(ctx context.Context) (CountWantsRow, error) { &i.Missing, &i.Held, &i.Following, + &i.Activity, + &i.Attention, + &i.Blocked, ) return i, err } @@ -520,11 +532,18 @@ SELECT w.id, w.release_group_mbid, w.artist_mbid, w.artist_name, w.title, w.reas (SELECT ar.id FROM artists ar WHERE ar.mbid = w.artist_mbid) AS library_artist_id, -- The search schedule's last look (#5521), and the download running. w.searched_at, w.search_outcome, w.search_considered, w.best_release, w.best_verdict, w.search_error, - gr.name AS grab_name, gr.grabbed_at, gr.origin AS grab_origin, gr.quality AS grab_quality + -- Its latest download, running or ended, and what the import made of + -- it (#5522). + gr.id AS grab_id, gr.name AS grab_name, gr.grabbed_at, gr.origin AS grab_origin, + gr.quality AS grab_quality, gr.ended_at AS grab_ended_at, gr.end_reason AS grab_end_reason, + gr.progress AS grab_progress, gr.import_state AS grab_import_state, + gr.import_reason AS grab_import_reason, gr.imported_at AS grab_imported_at, + gr.imported_files AS grab_imported_files FROM wants w LEFT JOIN users u ON u.id = w.created_by LEFT JOIN mb_release_groups g ON g.mbid = w.release_group_mbid - LEFT JOIN grabs gr ON gr.want_id = w.id AND gr.ended_at IS NULL + LEFT JOIN grabs gr ON gr.id = (SELECT x.id FROM grabs x WHERE x.want_id = w.id + ORDER BY x.ended_at IS NULL DESC, x.grabbed_at DESC LIMIT 1) WHERE CASE $1::text WHEN 'held' THEN w.fulfilled_at IS NOT NULL WHEN 'missing' THEN w.fulfilled_at IS NULL AND w.reason = 'missing' @@ -541,30 +560,38 @@ type ListWantsParams struct { } type ListWantsRow struct { - ID pgtype.UUID - ReleaseGroupMbid string - ArtistMbid string - ArtistName string - Title string - Reason string - AlbumID pgtype.UUID - WantedAt pgtype.Timestamptz - FulfilledAt pgtype.Timestamptz - PrimaryType string - SecondaryTypes []string - FirstReleaseDate string - RequestedBy *string - LibraryArtistID pgtype.UUID - SearchedAt pgtype.Timestamptz - SearchOutcome string - SearchConsidered int32 - BestRelease string - BestVerdict string - SearchError string - GrabName *string - GrabbedAt pgtype.Timestamptz - GrabOrigin *string - GrabQuality *string + ID pgtype.UUID + ReleaseGroupMbid string + ArtistMbid string + ArtistName string + Title string + Reason string + AlbumID pgtype.UUID + WantedAt pgtype.Timestamptz + FulfilledAt pgtype.Timestamptz + PrimaryType string + SecondaryTypes []string + FirstReleaseDate string + RequestedBy *string + LibraryArtistID pgtype.UUID + SearchedAt pgtype.Timestamptz + SearchOutcome string + SearchConsidered int32 + BestRelease string + BestVerdict string + SearchError string + GrabID pgtype.UUID + GrabName *string + GrabbedAt pgtype.Timestamptz + GrabOrigin *string + GrabQuality *string + GrabEndedAt pgtype.Timestamptz + GrabEndReason *string + GrabProgress *float32 + GrabImportState *string + GrabImportReason *string + GrabImportedAt pgtype.Timestamptz + GrabImportedFiles *int32 } // The Wanted page: open wants (all but missing ones, or only missing ones), @@ -599,10 +626,18 @@ func (q *Queries) ListWants(ctx context.Context, arg ListWantsParams) ([]ListWan &i.BestRelease, &i.BestVerdict, &i.SearchError, + &i.GrabID, &i.GrabName, &i.GrabbedAt, &i.GrabOrigin, &i.GrabQuality, + &i.GrabEndedAt, + &i.GrabEndReason, + &i.GrabProgress, + &i.GrabImportState, + &i.GrabImportReason, + &i.GrabImportedAt, + &i.GrabImportedFiles, ); err != nil { return nil, err } diff --git a/internal/db/migrations/0083_import.down.sql b/internal/db/migrations/0083_import.down.sql new file mode 100644 index 00000000..a7677dcb --- /dev/null +++ b/internal/db/migrations/0083_import.down.sql @@ -0,0 +1,24 @@ +ALTER TABLE acquisition_settings DROP COLUMN import_root; + +DELETE FROM release_blocks WHERE reason = 'import'; +ALTER TABLE release_blocks DROP CONSTRAINT release_blocks_reason_check; +ALTER TABLE release_blocks ADD CONSTRAINT release_blocks_reason_check + CHECK (reason IN ('operator', 'files')); + +DROP INDEX grabs_live_idx; + +UPDATE grabs SET end_reason = 'superseded' WHERE end_reason IN ('imported', 'refused', 'held'); +ALTER TABLE grabs DROP CONSTRAINT grabs_end_reason_check; +ALTER TABLE grabs ADD CONSTRAINT grabs_end_reason_check CHECK (end_reason IN ('', 'superseded')); + +ALTER TABLE grabs + DROP COLUMN progress, + DROP COLUMN import_state, + DROP COLUMN import_reason, + DROP COLUMN import_detail, + DROP COLUMN import_forced, + DROP COLUMN import_attempts, + DROP COLUMN import_updated_at, + DROP COLUMN imported_at, + DROP COLUMN imported_files, + DROP COLUMN destination; diff --git a/internal/db/migrations/0083_import.up.sql b/internal/db/migrations/0083_import.up.sql new file mode 100644 index 00000000..eb30d143 --- /dev/null +++ b/internal/db/migrations/0083_import.up.sql @@ -0,0 +1,37 @@ +-- Import with verification (M502 #5522): a finished download is checked +-- against the album it was grabbed for before anything reaches the library. +-- +-- grabs (columns) how far the download has got, and what the import +-- made of it: the verdict, its reason and evidence, +-- where the files went. +-- grabs.end_reason a grab also ends by being imported, refused, or +-- found to be an album already held. +-- release_blocks.reason import: the files were not the album. +-- acquisition_settings which library folder new music is filed into +-- ('' = the first scan path). +ALTER TABLE grabs + ADD COLUMN progress real NOT NULL DEFAULT 0, + ADD COLUMN import_state text NOT NULL DEFAULT '' + CONSTRAINT grabs_import_state_check CHECK (import_state IN + ('', 'downloading', 'waiting', 'importing', 'imported', 'refused', 'review', 'held', 'failed')), + ADD COLUMN import_reason text NOT NULL DEFAULT '', + ADD COLUMN import_detail jsonb NOT NULL DEFAULT '{}', + -- A person said "import anyway" to a download the check held for them. + ADD COLUMN import_forced boolean NOT NULL DEFAULT false, + ADD COLUMN import_attempts integer NOT NULL DEFAULT 0, + ADD COLUMN import_updated_at timestamptz, + ADD COLUMN imported_at timestamptz, + ADD COLUMN imported_files integer NOT NULL DEFAULT 0, + ADD COLUMN destination text NOT NULL DEFAULT ''; + +ALTER TABLE grabs DROP CONSTRAINT grabs_end_reason_check; +ALTER TABLE grabs ADD CONSTRAINT grabs_end_reason_check + CHECK (end_reason IN ('', 'superseded', 'imported', 'refused', 'held')); + +CREATE INDEX grabs_live_idx ON grabs (grabbed_at) WHERE ended_at IS NULL; + +ALTER TABLE release_blocks DROP CONSTRAINT release_blocks_reason_check; +ALTER TABLE release_blocks ADD CONSTRAINT release_blocks_reason_check + CHECK (reason IN ('operator', 'files', 'import')); + +ALTER TABLE acquisition_settings ADD COLUMN import_root text NOT NULL DEFAULT ''; diff --git a/internal/db/queries/imports.sql b/internal/db/queries/imports.sql new file mode 100644 index 00000000..28bf0ef0 --- /dev/null +++ b/internal/db/queries/imports.sql @@ -0,0 +1,130 @@ +-- Import with verification (M502 #5522). + +-- name: ListLiveGrabs :many +-- Every download still running or waiting on its import, oldest first. +SELECT g.*, w.release_group_mbid, w.artist_mbid, w.artist_name, w.title AS want_title, + w.reason AS want_reason, w.album_id AS want_album_id + FROM grabs g + LEFT JOIN wants w ON w.id = g.want_id + WHERE g.ended_at IS NULL + ORDER BY g.grabbed_at, g.id; + +-- name: GetGrab :one +SELECT * FROM grabs WHERE id = $1; + +-- name: SetGrabProgress :exec +-- What the download client says of a download not yet finished, or finished +-- and waiting for what the import needs. Leaves a claimed, decided or failed +-- import alone. +UPDATE grabs + SET progress = sqlc.arg(progress), + import_state = sqlc.arg(import_state), + import_reason = sqlc.arg(import_reason), + import_updated_at = now() + WHERE id = sqlc.arg(id) AND ended_at IS NULL + AND import_state IN ('', 'downloading', 'waiting'); + +-- name: ClaimGrabImport :one +-- Takes a finished download for import. The update is the lock: a second +-- pass finds nothing to claim. +UPDATE grabs + SET import_state = 'importing', import_attempts = import_attempts + 1, + import_updated_at = now(), progress = 1 + WHERE id = $1 AND ended_at IS NULL AND import_state IN ('', 'downloading', 'waiting') +RETURNING *; + +-- name: ReclaimStaleImports :execrows +-- An import cut short (a restart mid-copy) is tried again. +UPDATE grabs + SET import_state = 'waiting', + import_reason = 'The last import was cut short, so it is being tried again.' + WHERE ended_at IS NULL AND import_state = 'importing' + AND import_updated_at < now() - make_interval(mins => sqlc.arg(minutes)::int); + +-- name: RecordImport :one +-- The import's outcome. An end reason ends the grab: imported, refused or +-- held. Without one the grab stays live: review and failed wait for a person. +UPDATE grabs + SET import_state = sqlc.arg(import_state), + import_reason = sqlc.arg(import_reason), + import_detail = sqlc.arg(import_detail), + import_updated_at = now(), + import_forced = false, + imported_at = CASE WHEN sqlc.arg(import_state)::text = 'imported' THEN now() ELSE imported_at END, + imported_files = sqlc.arg(imported_files), + destination = sqlc.arg(destination), + ended_at = CASE WHEN sqlc.arg(end_reason)::text <> '' THEN now() ELSE ended_at END, + end_reason = sqlc.arg(end_reason), + end_detail = CASE WHEN sqlc.arg(end_reason)::text <> '' THEN sqlc.arg(import_reason)::text ELSE end_detail END + WHERE id = sqlc.arg(id) +RETURNING *; + +-- name: ForceGrabImport :one +-- "Import anyway": the next pass imports the download whatever the check said. +UPDATE grabs + SET import_state = '', import_forced = true, import_reason = '', import_updated_at = now() + WHERE id = $1 AND ended_at IS NULL AND import_state IN ('review', 'failed') +RETURNING *; + +-- name: RetryGrabImport :one +UPDATE grabs + SET import_state = '', import_reason = '', import_updated_at = now() + WHERE id = $1 AND ended_at IS NULL AND import_state = 'failed' +RETURNING *; + +-- name: RehomeGrab :one +-- A download that turned out to be another wanted album is that want's grab, +-- when the want has none running. +UPDATE grabs g + SET want_id = sqlc.arg(want_id) + WHERE g.id = sqlc.arg(id) AND g.ended_at IS NULL + AND NOT EXISTS (SELECT 1 FROM grabs o WHERE o.want_id = sqlc.arg(want_id) AND o.ended_at IS NULL) +RETURNING *; + +-- name: OpenWantsForGroups :many +-- Open wants for these release groups with no download running. +SELECT w.* FROM wants w + WHERE w.fulfilled_at IS NULL AND w.release_group_mbid = ANY(sqlc.arg(groups)::text[]) + AND NOT EXISTS (SELECT 1 FROM grabs g WHERE g.want_id = w.id AND g.ended_at IS NULL); + +-- name: MarkWantSearched :exec +-- Imported: nothing to search for until the wanted pass settles the want. +UPDATE wants SET searched_at = now() WHERE id = $1; + +-- name: SearchWantAgain :exec +-- Refused: search again on the next pass. +UPDATE wants SET searched_at = NULL, search_outcome = '' WHERE id = $1; + +-- name: HeldRecordings :many +-- Which of these recordings the library holds a file for. +SELECT DISTINCT mbid::text AS mbid FROM tracks + WHERE mbid = ANY(sqlc.arg(recordings)::text[]) AND missing_since IS NULL; + +-- name: HeldPositions :many +-- The disc and track positions the library holds for an album of this group. +SELECT DISTINCT COALESCE(t.disc_number, 1)::int AS disc, t.track_number::int AS track + FROM tracks t + JOIN albums a ON a.id = t.album_id + WHERE a.release_group_mbid = sqlc.arg(release_group_mbid) + AND t.missing_since IS NULL AND t.track_number IS NOT NULL; + +-- name: ListGrabs :many +-- The Wanted view's Activity tab: downloads waiting on a person first, then +-- the rest still running, then the ended ones newest first. +SELECT g.id, g.want_id, g.name, g.indexer, g.client, g.quality, g.origin, g.forced, + g.grabbed_at, g.ended_at, g.end_reason, g.end_detail, g.progress, g.import_state, + g.import_reason, g.import_detail, g.import_updated_at, g.imported_at, g.imported_files, + g.destination, w.title AS want_title, w.artist_name AS want_artist + FROM grabs g + LEFT JOIN wants w ON w.id = g.want_id + ORDER BY (g.ended_at IS NULL AND g.import_state IN ('review', 'failed')) DESC, + g.ended_at IS NULL DESC, g.grabbed_at DESC, g.id + LIMIT sqlc.arg(max_rows) OFFSET sqlc.arg(skip); + +-- name: ListReleaseBlocksPage :many +SELECT b.info_hash, b.name, b.reason, b.detail, b.blocked_at, w.title AS want_title, + w.artist_name AS want_artist + FROM release_blocks b + LEFT JOIN wants w ON w.id = b.want_id + ORDER BY b.blocked_at DESC, b.info_hash + LIMIT sqlc.arg(max_rows) OFFSET sqlc.arg(skip); diff --git a/internal/db/queries/search.sql b/internal/db/queries/search.sql index 832e35f2..9dc88be8 100644 --- a/internal/db/queries/search.sql +++ b/internal/db/queries/search.sql @@ -8,6 +8,7 @@ UPDATE acquisition_settings SET auto_grab = sqlc.arg(auto_grab), search_interval_hours = sqlc.arg(search_interval_hours), allowed_qualities = sqlc.arg(allowed_qualities), + import_root = sqlc.arg(import_root), updated_at = now() WHERE id = true RETURNING *; @@ -34,7 +35,7 @@ SELECT w.id, w.release_group_mbid, w.artist_name, w.title, w.reason, LIMIT sqlc.arg(max_rows); -- name: GetWantForSearch :one -SELECT w.id, w.release_group_mbid, w.artist_name, w.title, w.reason, w.fulfilled_at, +SELECT w.id, w.release_group_mbid, w.artist_mbid, w.artist_name, w.title, w.reason, w.fulfilled_at, COALESCE(NULLIF(w.primary_type, ''), g.primary_type, '')::text AS primary_type, COALESCE(g.secondary_types, '{}')::text[] AS secondary_types, COALESCE(NULLIF(w.first_release_date, ''), g.first_release_date, '')::text AS first_release_date diff --git a/internal/db/queries/wanted.sql b/internal/db/queries/wanted.sql index bde3219d..d52d47a6 100644 --- a/internal/db/queries/wanted.sql +++ b/internal/db/queries/wanted.sql @@ -189,11 +189,18 @@ SELECT w.id, w.release_group_mbid, w.artist_mbid, w.artist_name, w.title, w.reas (SELECT ar.id FROM artists ar WHERE ar.mbid = w.artist_mbid) AS library_artist_id, -- The search schedule's last look (#5521), and the download running. w.searched_at, w.search_outcome, w.search_considered, w.best_release, w.best_verdict, w.search_error, - gr.name AS grab_name, gr.grabbed_at, gr.origin AS grab_origin, gr.quality AS grab_quality + -- Its latest download, running or ended, and what the import made of + -- it (#5522). + gr.id AS grab_id, gr.name AS grab_name, gr.grabbed_at, gr.origin AS grab_origin, + gr.quality AS grab_quality, gr.ended_at AS grab_ended_at, gr.end_reason AS grab_end_reason, + gr.progress AS grab_progress, gr.import_state AS grab_import_state, + gr.import_reason AS grab_import_reason, gr.imported_at AS grab_imported_at, + gr.imported_files AS grab_imported_files FROM wants w LEFT JOIN users u ON u.id = w.created_by LEFT JOIN mb_release_groups g ON g.mbid = w.release_group_mbid - LEFT JOIN grabs gr ON gr.want_id = w.id AND gr.ended_at IS NULL + LEFT JOIN grabs gr ON gr.id = (SELECT x.id FROM grabs x WHERE x.want_id = w.id + ORDER BY x.ended_at IS NULL DESC, x.grabbed_at DESC LIMIT 1) WHERE CASE sqlc.arg(state)::text WHEN 'held' THEN w.fulfilled_at IS NOT NULL WHEN 'missing' THEN w.fulfilled_at IS NULL AND w.reason = 'missing' @@ -206,5 +213,11 @@ SELECT w.id, w.release_group_mbid, w.artist_mbid, w.artist_name, w.title, w.reas SELECT COUNT(*) FILTER (WHERE fulfilled_at IS NULL AND reason <> 'missing')::bigint AS wanted, COUNT(*) FILTER (WHERE fulfilled_at IS NULL AND reason = 'missing')::bigint AS missing, COUNT(*) FILTER (WHERE fulfilled_at IS NOT NULL)::bigint AS held, - (SELECT COUNT(*) FROM follows)::bigint AS following + (SELECT COUNT(*) FROM follows)::bigint AS following, + -- The Wanted view's other tabs (#5522): downloads running, those + -- waiting on a person, and releases set aside. + (SELECT COUNT(*) FROM grabs WHERE ended_at IS NULL)::bigint AS activity, + (SELECT COUNT(*) FROM grabs + WHERE ended_at IS NULL AND import_state IN ('review', 'failed'))::bigint AS attention, + (SELECT COUNT(*) FROM release_blocks)::bigint AS blocked FROM wants; diff --git a/internal/dbtest/reset.go b/internal/dbtest/reset.go index fd1b7865..8b138e24 100644 --- a/internal/dbtest/reset.go +++ b/internal/dbtest/reset.go @@ -186,7 +186,7 @@ func ResetDB(t *testing.T, pool *pgxpool.Pool) { // Search and grab settings (M502 #5521), a singleton reset the same way. if _, err := pool.Exec(ctx, ` UPDATE acquisition_settings - SET auto_grab = DEFAULT, search_interval_hours = DEFAULT, allowed_qualities = DEFAULT, + SET auto_grab = DEFAULT, search_interval_hours = DEFAULT, allowed_qualities = DEFAULT, import_root = DEFAULT, last_pass_at = NULL, last_pass = DEFAULT, updated_at = DEFAULT`, ); err != nil { t.Fatalf("dbtest.ResetDB reset acquisition settings: %v", err) diff --git a/internal/importer/importer.go b/internal/importer/importer.go new file mode 100644 index 00000000..5811f03d --- /dev/null +++ b/internal/importer/importer.go @@ -0,0 +1,608 @@ +package importer + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "log/slog" + "os" + "path/filepath" + "slices" + "strconv" + "sync" + "time" + + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgtype" + "github.com/jackc/pgx/v5/pgxpool" + + "git.fabledsword.com/bvandeusen/minstrel/internal/acquire" + "git.fabledsword.com/bvandeusen/minstrel/internal/catalogue" + "git.fabledsword.com/bvandeusen/minstrel/internal/db/dbq" + "git.fabledsword.com/bvandeusen/minstrel/internal/downloader" + "git.fabledsword.com/bvandeusen/minstrel/internal/indexer" +) + +// Import states, as grabs.import_state holds them. +const ( + StateDownloading = "downloading" + StateWaiting = "waiting" + StateImporting = "importing" + StateImported = "imported" + StateRefused = "refused" + StateReview = "review" + StateHeld = "held" + StateFailed = "failed" +) + +const ( + // interval: a finished download is noticed within a minute. One + // Torrents call per download client, so the client barely notices. + interval = time.Minute + // passTimeout bounds a pass: fingerprinting a long album against + // AcoustID's pace takes a minute or two, so half an hour is a hang. + passTimeout = 30 * time.Minute + // staleMinutes: an import still "importing" after this was cut short + // (a restart mid-copy) and is tried again. + staleMinutes = 30 +) + +// Scanner reads new files into the library. +type Scanner interface { + ScanFiles(ctx context.Context, paths []string) ([]pgtype.UUID, error) +} + +// Catalogue lists a release group's editions. +type Catalogue interface { + Releases(ctx context.Context, groupMBID string) (catalogue.Editions, error) +} + +// Importer watches every live grab's download and imports it once finished. +type Importer struct { + pool *pgxpool.Pool + Scanner Scanner + Catalogue Catalogue + Printer Printer + // Roots are the library's folders (the scan paths). New music is filed + // under the one the settings name, or the first. + Roots []string + // ClientFor builds the download client a grab went to. Defaults to the + // acquisition_services row of that code. + ClientFor func(ctx context.Context, code string) (downloader.Driver, error) + logger *slog.Logger + wake chan struct{} + mu sync.Mutex // one pass at a time +} + +// New returns an Importer. printer may be nil: the check then rests on +// lengths and tags. +func New(pool *pgxpool.Pool, scanner Scanner, cat Catalogue, printer Printer, roots []string, logger *slog.Logger) *Importer { + if logger == nil { + logger = slog.Default() + } + im := &Importer{pool: pool, Scanner: scanner, Catalogue: cat, Printer: printer, Roots: roots, + logger: logger, wake: make(chan struct{}, 1)} + im.ClientFor = im.clientFor + return im +} + +func (im *Importer) clientFor(ctx context.Context, code string) (downloader.Driver, error) { + row, err := dbq.New(im.pool).GetAcquisitionService(ctx, code) + if err != nil { + if errors.Is(err, pgx.ErrNoRows) { + return nil, fmt.Errorf("the download client %q is no longer set up", code) + } + return nil, err + } + return acquire.DownloadClient(row) +} + +// Wake asks for a pass now: after a grab, or a person's decision. +func (im *Importer) Wake() { + select { + case im.wake <- struct{}{}: + default: + } +} + +// Run passes every minute, and when woken, until ctx ends. +func (im *Importer) Run(ctx context.Context) { + t := time.NewTicker(interval) + defer t.Stop() + for { + select { + case <-ctx.Done(): + return + case <-t.C: + case <-im.wake: + } + rep := im.Pass(ctx) + if rep.Imported+rep.Refused+rep.Review+rep.Held+rep.Failed > 0 || rep.Error != "" { + im.logger.Info("import: pass", "imported", rep.Imported, "refused", rep.Refused, + "review", rep.Review, "held", rep.Held, "failed", rep.Failed, + "downloading", rep.Downloading, "error", rep.Error) + } + } +} + +// Report is one pass. +type Report struct { + Downloading, Imported, Refused, Review, Held, Failed, Waiting int + Error string +} + +// Pass looks at every live grab once. +func (im *Importer) Pass(ctx context.Context) Report { + im.mu.Lock() + defer im.mu.Unlock() + ctx, cancel := context.WithTimeout(ctx, passTimeout) + defer cancel() + var rep Report + q := dbq.New(im.pool) + if _, err := q.ReclaimStaleImports(ctx, staleMinutes); err != nil { + rep.Error = err.Error() + return rep + } + grabs, err := q.ListLiveGrabs(ctx) + if err != nil { + rep.Error = err.Error() + return rep + } + byClient := map[string][]dbq.ListLiveGrabsRow{} + var order []string + for _, g := range grabs { + if _, ok := byClient[g.Client]; !ok { + order = append(order, g.Client) + } + byClient[g.Client] = append(byClient[g.Client], g) + } + for _, code := range order { + im.passClient(ctx, q, code, byClient[code], &rep) + } + return rep +} + +func (im *Importer) passClient(ctx context.Context, q *dbq.Queries, code string, gs []dbq.ListLiveGrabsRow, rep *Report) { + wait := func(g dbq.ListLiveGrabsRow, why string) { + rep.Waiting++ + _ = q.SetGrabProgress(ctx, dbq.SetGrabProgressParams{ + ID: g.ID, Progress: g.Progress, ImportState: StateWaiting, ImportReason: why}) + } + d, err := im.ClientFor(ctx, code) + if err != nil { + for _, g := range gs { + wait(g, fmt.Sprintf("Can't reach its download client: %v.", err)) + } + return + } + hashes := make([]string, 0, len(gs)) + for _, g := range gs { + hashes = append(hashes, g.InfoHash) + } + ts, err := d.Torrents(ctx, hashes...) + if err != nil { + for _, g := range gs { + wait(g, fmt.Sprintf("Can't reach its download client %s: %s", code, indexer.FirstLine(err.Error()))) + } + return + } + byHash := map[string]downloader.Torrent{} + for _, t := range ts { + byHash[indexer.NormalizeHash(t.Hash)] = t + } + for _, g := range gs { + if ctx.Err() != nil { + return + } + switch g.ImportState { + case StateReview, StateFailed, StateImporting: + continue + } + t, ok := byHash[indexer.NormalizeHash(g.InfoHash)] + if !ok { + continue // gone from the client: the download lifecycle's (#5523) + } + if !g.WantID.Valid { + wait(g, "Its album is no longer wanted.") + continue + } + if !finished(ctx, d, t) { + rep.Downloading++ + _ = q.SetGrabProgress(ctx, dbq.SetGrabProgressParams{ + ID: g.ID, Progress: float32(min(t.Progress, 0.999)), ImportState: StateDownloading}) + continue + } + claimed, err := q.ClaimGrabImport(ctx, g.ID) + if err != nil { + continue // claimed by a pass racing this one, or ended + } + switch out := im.importOne(ctx, q, claimed, d, t); out { + case StateImported: + rep.Imported++ + case StateRefused: + rep.Refused++ + case StateReview: + rep.Review++ + case StateHeld: + rep.Held++ + case StateFailed: + rep.Failed++ + case StateWaiting: + rep.Waiting++ + } + } +} + +// upStates are qBittorrent's states for a torrent that has every byte and +// is seeding, paused or queued to seed. Exchange's states arrive in the same +// vocabulary. +var upStates = map[string]bool{ + "uploading": true, "stalledUP": true, "pausedUP": true, "stoppedUP": true, + "queuedUP": true, "forcedUP": true, "checkingUP": true, +} + +// finished: whole, in a seeding state, and no wanted file short of bytes. +func finished(ctx context.Context, d downloader.Driver, t downloader.Torrent) bool { + if !t.Done() || !upStates[t.State] { + return false + } + if fp, ok := d.(downloader.FileProgress); ok { + if states, err := fp.FileStates(ctx, t.Hash); err == nil { + for _, f := range states { + if f.Unfinished() { + return false + } + } + } + } + return true +} + +// Detail is what grabs.import_detail keeps: the verdict and what it rested +// on, the edition chosen, and what happened to the files. +type Detail struct { + Verdict Verdict `json:"verdict"` + Edition string `json:"edition,omitempty"` + EditionMBID string `json:"edition_mbid,omitempty"` + Files []string `json:"files,omitempty"` + Placed int `json:"placed,omitempty"` + Skipped int `json:"skipped,omitempty"` + Untagged int `json:"untagged,omitempty"` + // RehomedFrom: the album it was grabbed for, when it turned out to be + // another wanted album. + RehomedFrom string `json:"rehomed_from,omitempty"` +} + +// importOne checks and files one finished download and records the outcome. +func (im *Importer) importOne(ctx context.Context, q *dbq.Queries, g dbq.Grab, d downloader.Driver, t downloader.Torrent) string { + record := func(state, reason, end string, det Detail, files int, dest string) string { + body, _ := json.Marshal(det) + // The outcome is written even when the pass's context is ending: + // a claimed grab left "importing" waits half an hour to be retried. + wctx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 15*time.Second) + defer cancel() + if _, err := q.RecordImport(wctx, dbq.RecordImportParams{ + ID: g.ID, ImportState: state, ImportReason: reason, ImportDetail: body, + ImportedFiles: int32(files), Destination: dest, EndReason: end, + }); err != nil { + im.logger.Warn("import: record outcome", "grab", g.Name, "state", state, "err", err) + } + im.logger.Info("import: "+state, "grab", g.Name, "reason", reason) + return state + } + + w, err := im.want(ctx, q, g.WantID) + if err != nil { + return record(StateWaiting, "Its album could not be read: "+err.Error(), "", Detail{}, 0, "") + } + editions, err := im.editions(ctx, w.GroupMBID) + if err != nil { + return record(StateWaiting, "MusicBrainz could not be reached to check it against the album; trying again shortly.", + "", Detail{}, 0, "") + } + src := t.ContentPath + if src == "" { + src = filepath.Join(g.SavePath, t.Name) + } + files, printed, err := inspect(ctx, src, im.Printer) + if err != nil { + return record(StateFailed, fmt.Sprintf("Couldn't read the download at %s: %v", src, err), "", Detail{}, 0, "") + } + v := Verify(w, editions, files, printed, g.ImportForced) + det := Detail{Verdict: v} + for _, f := range files { + det.Files = append(det.Files, f.Rel) + } + if v.Edition >= 0 { + det.Edition, det.EditionMBID = describe(editions[v.Edition]), editions[v.Edition].MBID + } + + switch v.Outcome { + case Review: + return record(StateReview, v.Reason, "", det, 0, "") + case Refuse: + if v.Code == CodeWrongAlbum { + if out, ok := im.rehome(ctx, q, g, d, w, files, printed, record); ok { + return out + } + } + im.refuse(ctx, q, g, d, w, v.Reason, pgtype.UUID{}) + return record(StateRefused, v.Reason, StateRefused, det, 0, "") + } + return im.place(ctx, q, g, d, w, editions, files, v, det, record) +} + +type recordFunc func(state, reason, end string, det Detail, files int, dest string) string + +// place files an accepted download and records it imported, or held when +// the library already had every track. +func (im *Importer) place(ctx context.Context, q *dbq.Queries, g dbq.Grab, d downloader.Driver, w Want, + editions []Edition, files []File, v Verdict, det Detail, record recordFunc) string { + root, err := im.root(ctx, q) + if err != nil { + return record(StateFailed, "Not imported: "+err.Error()+".", "", det, 0, "") + } + var e *Edition + if v.Edition >= 0 { + e = &editions[v.Edition] + } + plan := Plan(root, w, e, files, v.Pairs) + heldRec, heldPos, err := im.held(ctx, q, w, plan) + if err != nil { + return record(StateWaiting, "Couldn't read what the library holds: "+err.Error(), "", det, 0, "") + } + + var placed []string + for _, pl := range plan { + if (pl.Recording != "" && heldRec[pl.Recording]) || heldPos[[2]int{pl.Disc, pl.Track}] { + det.Skipped++ + continue + } + tags := Tags{Title: pl.Title, Artist: w.Artist, Album: w.Title, AlbumArtist: w.Artist, Year: w.Year, + Disc: pl.Disc, Discs: pl.Discs, Track: pl.Track, Tracks: pl.Tracks, + GroupMBID: w.GroupMBID, RecordingMBID: pl.Recording, ArtistMBID: w.ArtistMBID} + if e != nil { + tags.ReleaseMBID = e.MBID + } + tagged, err := Place(ctx, files[pl.File].Path, pl.Dest, tags) + if errors.Is(err, ErrOccupied) { + det.Skipped++ + continue + } + if err != nil { + for _, p := range placed { + _ = os.Remove(p) + } + return record(StateFailed, fmt.Sprintf("Copying into the library failed, and nothing was left behind: %v", err), + "", det, 0, "") + } + placed = append(placed, pl.Dest) + if !tagged { + det.Untagged++ + } + } + det.Placed = len(placed) + if len(placed) == 0 { + im.removeFromClient(ctx, d, g) + return record(StateHeld, fmt.Sprintf("The library already holds every track of %s, so nothing was imported.", w.Title), + StateHeld, det, 0, "") + } + + if _, err := im.Scanner.ScanFiles(ctx, placed); err != nil { + // The watcher and the next scan find the files anyway. + im.logger.Warn("import: scan the new files", "grab", g.Name, "err", err) + } + if err := q.MarkWantSearched(ctx, g.WantID); err != nil { + im.logger.Warn("import: mark the want searched", "grab", g.Name, "err", err) + } + dest := filepath.Dir(placed[0]) + if e != nil && e.Media() > 1 { + dest = filepath.Dir(dest) + } + reason := fmt.Sprintf("Imported %s into %s. %s", tracks(len(placed)), dest, v.Reason) + if det.Skipped > 0 { + reason += fmt.Sprintf(" %s already in the library left as they were.", tracks(det.Skipped)) + } + if det.Untagged > 0 { + reason += fmt.Sprintf(" %s copied with their own tags: ffmpeg could not rewrite them.", tracks(det.Untagged)) + } + return record(StateImported, reason, StateImported, det, len(placed), dest) +} + +// rehome imports a download that turned out to be another wanted album, +// named by its tags, for that album, when its own grab is not running. +func (im *Importer) rehome(ctx context.Context, q *dbq.Queries, g dbq.Grab, d downloader.Driver, + from Want, files []File, printed bool, record recordFunc) (string, bool) { + group := majority(files, func(f File) string { return f.GroupMBID }) + if group == "" { + return "", false + } + others, err := q.OpenWantsForGroups(ctx, []string{group}) + if err != nil || len(others) == 0 { + return "", false + } + other := others[0] + w, err := im.want(ctx, q, other.ID) + if err != nil { + return "", false + } + editions, err := im.editions(ctx, w.GroupMBID) + if err != nil { + return "", false + } + v := Verify(w, editions, files, printed, false) + if v.Outcome != Accept { + return "", false + } + moved, err := q.RehomeGrab(ctx, dbq.RehomeGrabParams{ID: g.ID, WantID: other.ID}) + if err != nil { + return "", false + } + if err := q.SearchWantAgain(ctx, g.WantID); err != nil { + im.logger.Warn("import: search the first album again", "want", from.Title, "err", err) + } + det := Detail{Verdict: v, RehomedFrom: from.Title} + for _, f := range files { + det.Files = append(det.Files, f.Rel) + } + if v.Edition >= 0 { + det.Edition, det.EditionMBID = describe(editions[v.Edition]), editions[v.Edition].MBID + } + v.Reason = fmt.Sprintf("Grabbed for %s, it is %s, which is wanted too. %s", from.Title, w.Title, v.Reason) + det.Verdict = v + return im.place(ctx, q, moved, d, w, editions, files, v, det, record), true +} + +// refuse blocks the release, asks for the album to be searched again and +// removes the download with its files: it is not the album. +func (im *Importer) refuse(ctx context.Context, q *dbq.Queries, g dbq.Grab, d downloader.Driver, w Want, why string, by pgtype.UUID) { + if err := acquire.NewMemory(im.pool).Block(ctx, dbq.BlockReleaseParams{ + InfoHash: g.InfoHash, Name: g.Name, Reason: acquire.BlockImport, Detail: why, WantID: g.WantID, BlockedBy: by, + }); err != nil { + im.logger.Warn("import: block the release", "grab", g.Name, "err", err) + } + if err := q.SearchWantAgain(ctx, g.WantID); err != nil { + im.logger.Warn("import: search again", "want", w.Title, "err", err) + } + im.removeFromClient(ctx, d, g) +} + +func (im *Importer) removeFromClient(ctx context.Context, d downloader.Driver, g dbq.Grab) { + if d == nil || g.InfoHash == "" { + return + } + if err := d.Delete(ctx, true, g.InfoHash); err != nil { + im.logger.Warn("import: remove the download", "grab", g.Name, "err", err) + } +} + +// Refuse is a person's "refuse" on a download the check held for them, or +// one that failed: it is refused as the check would have. +func (im *Importer) Refuse(ctx context.Context, grabID, by pgtype.UUID) (dbq.Grab, error) { + q := dbq.New(im.pool) + g, err := q.GetGrab(ctx, grabID) + if err != nil { + return dbq.Grab{}, err + } + if g.EndedAt.Valid || (g.ImportState != StateReview && g.ImportState != StateFailed) { + return dbq.Grab{}, ErrNotDecidable + } + var w Want + if g.WantID.Valid { + w, _ = im.want(ctx, q, g.WantID) + } + d, err := im.ClientFor(ctx, g.Client) + if err != nil { + d = nil + } + why := "A person refused it: it is not the album." + im.refuse(ctx, q, g, d, w, why, by) + out, err := q.RecordImport(ctx, dbq.RecordImportParams{ + ID: g.ID, ImportState: StateRefused, ImportReason: why, ImportDetail: g.ImportDetail, + EndReason: StateRefused, + }) + im.Wake() + return out, err +} + +// ErrNotDecidable: the grab is not waiting on a person. +var ErrNotDecidable = errors.New("importer: that download is not waiting on a decision") + +func (im *Importer) want(ctx context.Context, q *dbq.Queries, id pgtype.UUID) (Want, error) { + row, err := q.GetWantForSearch(ctx, id) + if err != nil { + return Want{}, err + } + return Want{ + GroupMBID: row.ReleaseGroupMbid, ArtistMBID: row.ArtistMbid, Artist: row.ArtistName, Title: row.Title, + Year: yearOf(row.FirstReleaseDate), PrimaryType: row.PrimaryType, SecondaryTypes: row.SecondaryTypes, + }, nil +} + +func (im *Importer) editions(ctx context.Context, group string) ([]Edition, error) { + if im.Catalogue == nil { + return nil, errors.New("no catalogue") + } + eds, err := im.Catalogue.Releases(ctx, group) + if err != nil { + return nil, err + } + out := make([]Edition, 0, len(eds.Editions)) + for _, e := range eds.Editions { + ed := Edition{MBID: e.Release.Mbid, Title: e.Release.Title, Status: e.Release.Status, + Date: e.Release.ReleaseDate, Country: e.Release.Country} + for _, t := range e.Tracks { + ed.Tracks = append(ed.Tracks, Track{Medium: int(t.MediumPosition), Format: t.MediumFormat, + Position: int(t.Position), Title: t.Title, RecordingMBID: t.RecordingMbid, LengthMs: int(t.LengthMs)}) + } + slices.SortStableFunc(ed.Tracks, func(a, b Track) int { + if a.Medium != b.Medium { + return a.Medium - b.Medium + } + return a.Position - b.Position + }) + out = append(out, ed) + } + return out, nil +} + +// root is the library folder new music is filed into. +func (im *Importer) root(ctx context.Context, q *dbq.Queries) (string, error) { + if len(im.Roots) == 0 { + return "", errors.New("there is no library folder to file new music into (MINSTREL_LIBRARY_SCAN_PATHS)") + } + s, err := q.GetAcquisitionSettings(ctx) + if err == nil && s.ImportRoot != "" && slices.Contains(im.Roots, s.ImportRoot) { + return s.ImportRoot, nil + } + return im.Roots[0], nil +} + +// held reads which of the planned tracks the library already has: by +// recording, and by disc and track of an album of the same group. +func (im *Importer) held(ctx context.Context, q *dbq.Queries, w Want, plan []Placement) (map[string]bool, map[[2]int]bool, error) { + var recs []string + for _, pl := range plan { + if pl.Recording != "" { + recs = append(recs, pl.Recording) + } + } + rec := map[string]bool{} + if len(recs) > 0 { + got, err := q.HeldRecordings(ctx, recs) + if err != nil { + return nil, nil, err + } + for _, r := range got { + rec[r] = true + } + } + pos := map[[2]int]bool{} + got, err := q.HeldPositions(ctx, &w.GroupMBID) + if err != nil { + return nil, nil, err + } + for _, p := range got { + pos[[2]int{int(p.Disc), int(p.Track)}] = true + } + return rec, pos, nil +} + +func yearOf(date string) int { + if len(date) < 4 { + return 0 + } + y, err := strconv.Atoi(date[:4]) + if err != nil { + return 0 + } + return y +} + +func tracks(n int) string { + if n == 1 { + return "1 track" + } + return strconv.Itoa(n) + " tracks" +} diff --git a/internal/importer/importer_test.go b/internal/importer/importer_test.go new file mode 100644 index 00000000..36766b81 --- /dev/null +++ b/internal/importer/importer_test.go @@ -0,0 +1,268 @@ +package importer + +import ( + "context" + "io" + "log/slog" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/jackc/pgx/v5/pgtype" + "github.com/jackc/pgx/v5/pgxpool" + + "git.fabledsword.com/bvandeusen/minstrel/internal/acquire" + "git.fabledsword.com/bvandeusen/minstrel/internal/catalogue" + "git.fabledsword.com/bvandeusen/minstrel/internal/db" + "git.fabledsword.com/bvandeusen/minstrel/internal/db/dbq" + "git.fabledsword.com/bvandeusen/minstrel/internal/dbtest" + "git.fabledsword.com/bvandeusen/minstrel/internal/downloader" + "git.fabledsword.com/bvandeusen/minstrel/internal/wanted" +) + +func newPool(t *testing.T) *pgxpool.Pool { + t.Helper() + if testing.Short() { + t.Skip("skipping integration test in -short mode") + } + dsn := os.Getenv("MINSTREL_TEST_DATABASE_URL") + if dsn == "" { + t.Skip("MINSTREL_TEST_DATABASE_URL not set") + } + if err := db.Migrate(dsn, slog.New(slog.NewTextHandler(io.Discard, nil))); err != nil { + t.Fatalf("migrate: %v", err) + } + pool, err := pgxpool.New(context.Background(), dsn) + if err != nil { + t.Fatalf("pool: %v", err) + } + t.Cleanup(pool.Close) + dbtest.ResetDB(t, pool) + return pool +} + +const hash = "0123456789abcdef0123456789abcdef01234567" + +// fakeClient is a download client holding one finished torrent. +type fakeClient struct { + downloader.Driver + content string + deleted []string +} + +func (c *fakeClient) Torrents(_ context.Context, _ ...string) ([]downloader.Torrent, error) { + return []downloader.Torrent{{Hash: hash, Name: "Daft Punk - RAM [FLAC]", Progress: 1, State: "uploading", + ContentPath: c.content}}, nil +} + +func (c *fakeClient) Delete(_ context.Context, _ bool, hashes ...string) error { + c.deleted = append(c.deleted, hashes...) + return nil +} + +// fakeCatalogue knows RAM as one three-track edition. +type fakeCatalogue struct{} + +func (fakeCatalogue) Releases(_ context.Context, _ string) (catalogue.Editions, error) { + return catalogue.Editions{Editions: []catalogue.Edition{{ + Release: dbq.MbRelease{Mbid: "rel-ram", Title: "Random Access Memories", Status: "Official", ReleaseDate: "2013-05-17"}, + Tracks: []dbq.MbReleaseTrack{ + {MediumPosition: 1, MediumFormat: "CD", Position: 1, Title: "Give Life Back to Music", RecordingMbid: "rec-01"}, + {MediumPosition: 1, MediumFormat: "CD", Position: 2, Title: "The Game of Love", RecordingMbid: "rec-02"}, + {MediumPosition: 1, MediumFormat: "CD", Position: 3, Title: "Giorgio by Moroder", RecordingMbid: "rec-03"}, + }, + }}}, nil +} + +// fakePrinter names each file's recording by its file name: 01.flac is +// rec-01, unless wrong is set. +type fakePrinter struct{ wrong bool } + +func (p fakePrinter) Recordings(_ context.Context, path string) ([]string, error) { + if p.wrong { + return []string{"rec-other"}, nil + } + return []string{"rec-" + strings.TrimSuffix(filepath.Base(path), ".flac")}, nil +} + +type fakeScanner struct{ scanned []string } + +func (s *fakeScanner) ScanFiles(_ context.Context, paths []string) ([]pgtype.UUID, error) { + s.scanned = append(s.scanned, paths...) + return nil, nil +} + +type stack struct { + pool *pgxpool.Pool + q *dbq.Queries + im *Importer + client *fakeClient + scanner *fakeScanner + library string + want dbq.Want + grab dbq.Grab +} + +// newStack wants RAM, has a finished download of it waiting in a fake +// client, and an importer filing into a temp library. +func newStack(t *testing.T, printer Printer) *stack { + t.Helper() + ctx := context.Background() + pool := newPool(t) + q := dbq.New(pool) + content := t.TempDir() + for _, n := range []string{"01", "02", "03"} { + if err := os.WriteFile(filepath.Join(content, n+".flac"), []byte("not really flac "+n), 0o644); err != nil { + t.Fatal(err) + } + } + w, err := wanted.WantWith(ctx, q, wanted.WantParams{GroupMBID: "rg-ram", ArtistMBID: "mb-dp", ArtistName: "Daft Punk", + Title: "Random Access Memories", PrimaryType: "Album", FirstReleaseDate: "2013-05-17", Reason: wanted.ReasonRequest}) + if err != nil { + t.Fatal(err) + } + g, err := acquire.NewMemory(pool).Grabbed(ctx, dbq.InsertGrabParams{WantID: w.ID, InfoHash: hash, + Name: "Daft Punk - RAM [FLAC]", Key: hash, Client: "qbittorrent", SavePath: content, Origin: acquire.OriginSchedule}, + pgtype.UUID{}, "") + if err != nil { + t.Fatal(err) + } + s := &stack{pool: pool, q: q, client: &fakeClient{content: content}, scanner: &fakeScanner{}, + library: t.TempDir(), want: w, grab: g} + s.im = New(pool, s.scanner, fakeCatalogue{}, printer, []string{s.library}, slog.New(slog.NewTextHandler(io.Discard, nil))) + s.im.ClientFor = func(context.Context, string) (downloader.Driver, error) { return s.client, nil } + return s +} + +func (s *stack) grabNow(t *testing.T) dbq.Grab { + t.Helper() + g, err := s.q.GetGrab(context.Background(), s.grab.ID) + if err != nil { + t.Fatal(err) + } + return g +} + +func TestImport_AFinishedDownloadOfTheAlbumIsFiled_Integration(t *testing.T) { + s := newStack(t, fakePrinter{}) + rep := s.im.Pass(context.Background()) + if rep.Imported != 1 { + t.Fatalf("pass = %+v", rep) + } + g := s.grabNow(t) + if g.ImportState != StateImported || g.EndReason != StateImported || !g.EndedAt.Valid || g.ImportedFiles != 3 { + t.Fatalf("grab = %s %s ended=%v files=%d: %s", g.ImportState, g.EndReason, g.EndedAt.Valid, g.ImportedFiles, g.ImportReason) + } + dir := filepath.Join(s.library, "Daft Punk", "Random Access Memories (2013)") + if g.Destination != dir { + t.Errorf("destination %q", g.Destination) + } + got, err := os.ReadFile(filepath.Join(dir, "Daft Punk - Random Access Memories - 02 - The Game of Love.flac")) + if err != nil || string(got) != "not really flac 02" { + t.Errorf("filed copy: %q %v", got, err) + } + if _, err := os.Stat(filepath.Join(s.client.content, "02.flac")); err != nil { + t.Errorf("the download's own file went: %v", err) + } + if len(s.scanner.scanned) != 3 || len(s.client.deleted) != 0 { + t.Errorf("scanned %v, deleted %v", s.scanner.scanned, s.client.deleted) + } + w, _ := s.q.GetWantByReleaseGroup(context.Background(), "rg-ram") + if !w.SearchedAt.Valid { + t.Error("an imported want is due for search again") + } + if s.im.Pass(context.Background()).Imported != 0 { + t.Error("imported twice") + } +} + +func TestImport_TheWrongAlbumIsRefusedBlockedAndSearchedAgain_Integration(t *testing.T) { + s := newStack(t, fakePrinter{wrong: true}) + ctx := context.Background() + if err := s.q.MarkWantSearched(ctx, s.want.ID); err != nil { + t.Fatal(err) + } + if rep := s.im.Pass(ctx); rep.Refused != 1 { + t.Fatalf("pass = %+v", rep) + } + g := s.grabNow(t) + if g.ImportState != StateRefused || g.EndReason != StateRefused || !strings.Contains(g.ImportReason, "recordings") { + t.Fatalf("grab = %s %s: %s", g.ImportState, g.EndReason, g.ImportReason) + } + blocks, err := s.q.ListReleaseBlocks(ctx) + if err != nil || len(blocks) != 1 || blocks[0].InfoHash != hash || blocks[0].Reason != acquire.BlockImport { + t.Errorf("blocks = %+v %v", blocks, err) + } + if len(s.client.deleted) != 1 { + t.Errorf("the refused download was not removed: %v", s.client.deleted) + } + w, _ := s.q.GetWantByReleaseGroup(ctx, "rg-ram") + if w.SearchedAt.Valid { + t.Error("a refused want is not due for search") + } + entries, _ := os.ReadDir(s.library) + if len(entries) != 0 { + t.Errorf("the library got %d entries", len(entries)) + } +} + +func TestImport_ADoubtfulDownloadWaitsForAPersonThenImportsAnyway_Integration(t *testing.T) { + s := newStack(t, nil) // no AcoustID; untagged files: nothing but lengths + ctx := context.Background() + if rep := s.im.Pass(ctx); rep.Review != 1 { + t.Fatalf("pass = %+v", rep) + } + g := s.grabNow(t) + if g.ImportState != StateReview || g.EndedAt.Valid || !strings.Contains(g.ImportReason, "A person decides") { + t.Fatalf("grab = %s ended=%v: %s", g.ImportState, g.EndedAt.Valid, g.ImportReason) + } + if rep := s.im.Pass(ctx); rep.Review+rep.Imported != 0 { + t.Errorf("a held download was looked at again: %+v", rep) + } + if _, err := s.q.ForceGrabImport(ctx, g.ID); err != nil { + t.Fatal(err) + } + if rep := s.im.Pass(ctx); rep.Imported != 1 { + t.Fatalf("after import anyway: %+v", rep) + } + if g := s.grabNow(t); g.ImportState != StateImported || !strings.Contains(g.ImportReason, "anyway") { + t.Errorf("grab = %s: %s", g.ImportState, g.ImportReason) + } +} + +func TestImport_APersonRefusesADoubtfulDownload_Integration(t *testing.T) { + s := newStack(t, nil) + ctx := context.Background() + s.im.Pass(ctx) + g, err := s.im.Refuse(ctx, s.grab.ID, pgtype.UUID{}) + if err != nil { + t.Fatal(err) + } + if g.ImportState != StateRefused || !g.EndedAt.Valid || len(s.client.deleted) != 1 { + t.Errorf("grab = %s ended=%v deleted=%v", g.ImportState, g.EndedAt.Valid, s.client.deleted) + } + if _, err := s.im.Refuse(ctx, s.grab.ID, pgtype.UUID{}); err == nil { + t.Error("refused twice") + } +} + +func TestImport_AnUnfinishedDownloadIsOnlyWatched_Integration(t *testing.T) { + s := newStack(t, fakePrinter{}) + s.im.ClientFor = func(context.Context, string) (downloader.Driver, error) { return &halfClient{s.client}, nil } + if rep := s.im.Pass(context.Background()); rep.Downloading != 1 { + t.Fatalf("pass = %+v", rep) + } + g := s.grabNow(t) + if g.ImportState != StateDownloading || g.Progress < 0.49 || g.Progress > 0.51 { + t.Errorf("grab = %s %.2f", g.ImportState, g.Progress) + } +} + +type halfClient struct{ *fakeClient } + +func (c *halfClient) Torrents(ctx context.Context, h ...string) ([]downloader.Torrent, error) { + ts, _ := c.fakeClient.Torrents(ctx, h...) + ts[0].Progress, ts[0].State = 0.5, "downloading" + return ts, nil +} diff --git a/internal/importer/inspect.go b/internal/importer/inspect.go new file mode 100644 index 00000000..a2a5f863 --- /dev/null +++ b/internal/importer/inspect.go @@ -0,0 +1,195 @@ +package importer + +import ( + "context" + "errors" + "io/fs" + "os" + "path/filepath" + "sort" + "strings" + + "github.com/dhowden/tag" + "github.com/dhowden/tag/mbz" + + "git.fabledsword.com/bvandeusen/minstrel/internal/acoustid" + "git.fabledsword.com/bvandeusen/minstrel/internal/library" +) + +// Printer says which MusicBrainz recordings a file's audio is. ErrNoPrints: +// it cannot say for any file (AcoustID is not set up, or not answering), so +// the check rests on lengths and tags. +type Printer interface { + Recordings(ctx context.Context, path string) ([]string, error) +} + +// ErrNoPrints: fingerprints are not available for this download. +var ErrNoPrints = errors.New("importer: fingerprints are not available") + +// AcoustIDPrinter fingerprints with fpcalc and asks AcoustID, under the +// operator's settings (key, minimum score). Each call reads the settings, so +// a change takes effect on the next file. +type AcoustIDPrinter struct { + Settings interface { + Get() library.AcoustIDSettings + } + Client *acoustid.Client +} + +// Recordings is every recording AcoustID gives at or above the minimum score: +// one audio can be several recordings (the album track and the single). +func (p *AcoustIDPrinter) Recordings(ctx context.Context, path string) ([]string, error) { + s := p.Settings.Get() + if !s.Ready() || p.Client == nil { + return nil, ErrNoPrints + } + fp, dur, err := library.LookupFingerprint(ctx, path) + if err != nil { + if library.IsInconclusive(err) { + return nil, ErrNoPrints + } + return nil, nil // fpcalc rejected the file: no evidence either way + } + cands, err := p.Client.Lookup(ctx, s.APIKey, fp, dur) + switch { + case errors.Is(err, acoustid.ErrInvalidKey), errors.Is(err, acoustid.ErrUnavailable): + return nil, ErrNoPrints + case err != nil: + return nil, nil + } + var out []string + for _, c := range cands { + if c.Score >= float64(s.MinScore) && c.ID != "" { + out = append(out, c.ID) + } + } + return out, nil +} + +// formatRank picks which format's files are the album when a download holds +// more than one (a FLAC folder and an MP3 folder): the most files, then the +// better format. +var formatRank = map[string]int{".flac": 6, ".wav": 5, ".m4a": 4, ".opus": 3, ".ogg": 2, ".mp3": 1, ".aac": 0} + +// audioFiles lists the audio files under root (a folder, or one file), in +// the one format that is the album. Dot-named files and folders are skipped: +// they are partial downloads, OS litter or Minstrel's own temp files. +func audioFiles(root string) ([]string, error) { + info, err := os.Stat(root) + if err != nil { + return nil, err + } + if !info.IsDir() { + if library.IsAudioFile(root) { + return []string{root}, nil + } + return nil, nil + } + byExt := map[string][]string{} + err = filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + if strings.HasPrefix(d.Name(), ".") && path != root { + if d.IsDir() { + return filepath.SkipDir + } + return nil + } + if !d.IsDir() && library.IsAudioFile(path) { + ext := strings.ToLower(filepath.Ext(path)) + byExt[ext] = append(byExt[ext], path) + } + return nil + }) + if err != nil { + return nil, err + } + best := "" + for ext, paths := range byExt { + if best == "" || len(paths) > len(byExt[best]) || + (len(paths) == len(byExt[best]) && formatRank[ext] > formatRank[best]) { + best = ext + } + } + out := byExt[best] + sort.Strings(out) + return out, nil +} + +// readFile reads what one file's tags and length say. A file whose tags +// cannot be read still counts, by its length and name. +func readFile(ctx context.Context, root, path string) File { + f := File{Path: path, Rel: rel(root, path), Video: len(library.SourceMarkersFor(path)) > 0} + if ms, err := library.ProbeDurationMs(ctx, path); err == nil { + f.LengthMs = int(ms) + } + h, err := os.Open(path) + if err != nil { + return f + } + defer func() { _ = h.Close() }() + m, err := tag.ReadFrom(h) + if err != nil { + return f + } + f.Title, f.Artist, f.AlbumArtist, f.Album = m.Title(), m.Artist(), m.AlbumArtist(), m.Album() + f.Track, _ = m.Track() + f.Disc, _ = m.Disc() + info := mbz.Extract(m) + f.GroupMBID = clean(info.Get(mbz.ReleaseGroup)) + f.ReleaseMBID = clean(info.Get(mbz.Album)) + f.RecordingMBID = clean(info.Get(mbz.Recording)) + if f.RecordingMBID == "" && m.Format() == tag.VORBIS { + // Picard's Vorbis key for the recording is MUSICBRAINZ_TRACKID, which + // mbz files under its release-track name. + f.RecordingMBID = clean(info.Get(mbz.Track)) + } + return f +} + +func rel(root, path string) string { + if r, err := filepath.Rel(root, path); err == nil && r != "." { + return r + } + return filepath.Base(path) +} + +// clean takes the first of NUL-separated values: ID3v2.4 multi-values. +func clean(s string) string { + for _, part := range strings.Split(s, "\x00") { + if t := strings.TrimSpace(part); t != "" { + return t + } + } + return "" +} + +// inspect reads every audio file under root and asks the printer about each. +// fingerprinted reports whether the printer answered at all. +func inspect(ctx context.Context, root string, p Printer) (files []File, fingerprinted bool, err error) { + paths, err := audioFiles(root) + if err != nil { + return nil, false, err + } + fingerprinted = p != nil + for _, path := range paths { + if ctx.Err() != nil { + return nil, false, ctx.Err() + } + f := readFile(ctx, root, path) + if fingerprinted && len(paths) <= maxFiles { + recs, err := p.Recordings(ctx, path) + if errors.Is(err, ErrNoPrints) { + fingerprinted = false + for i := range files { + files[i].Recordings = nil + } + } else { + f.Recordings = recs + } + } + files = append(files, f) + } + return files, fingerprinted, nil +} diff --git a/internal/importer/name.go b/internal/importer/name.go new file mode 100644 index 00000000..c9342fef --- /dev/null +++ b/internal/importer/name.go @@ -0,0 +1,110 @@ +package importer + +import ( + "fmt" + "path/filepath" + "strings" + "unicode/utf8" +) + +// Placement is where one file goes and the names written into its tags. +type Placement struct { + File int // index into the files + Dest string // absolute + Title string + Disc int + Discs int + Track int + Tracks int // on its disc + // Recording is the MusicBrainz recording the file was matched to. + Recording string +} + +// Plan names every file's destination under root the way Lidarr named the +// library Minstrel took over (note #5527), so new arrivals sit beside what +// is there: +// +// {Artist}/{Album} ({Year})/{Artist} - {Album} - {track:00} - {Title}.ext +// +// with a "{Medium Format} {medium:00}" folder between when the edition has +// more than one medium. A file the check paired to a track takes the track's +// name and place; one it did not (an "import anyway" with more files than +// tracks) keeps its own tags. +func Plan(root string, w Want, e *Edition, files []File, pairs []Pair) []Placement { + album := w.Title + if w.Year > 0 { + album = fmt.Sprintf("%s (%d)", w.Title, w.Year) + } + dir := filepath.Join(root, Segment(w.Artist), Segment(album)) + media := 1 + if e != nil { + media = max(e.Media(), 1) + } + perDisc := map[int]int{} + if e != nil { + for _, t := range e.Tracks { + perDisc[t.Medium]++ + } + } + paired := map[int]Pair{} + for _, p := range pairs { + paired[p.File] = p + } + + out := make([]Placement, 0, len(files)) + for i, f := range files { + pl := Placement{File: i, Title: f.Title, Disc: max(f.Disc, 1), Track: f.Track, Discs: media} + format := "" + if p, ok := paired[i]; ok && e != nil { + t := e.Tracks[p.Track] + pl.Title, pl.Disc, pl.Track, pl.Recording, format = t.Title, max(t.Medium, 1), t.Position, t.RecordingMBID, t.Format + pl.Tracks = perDisc[t.Medium] + } + if pl.Title == "" { + pl.Title = strings.TrimSuffix(filepath.Base(f.Path), filepath.Ext(f.Path)) + } + d := dir + if media > 1 { + if format == "" { + format = "Disc" + } + d = filepath.Join(dir, Segment(fmt.Sprintf("%s %02d", format, pl.Disc))) + } + name := fmt.Sprintf("%s - %s - %02d - %s", w.Artist, w.Title, pl.Track, pl.Title) + pl.Dest = filepath.Join(d, Segment(name)+strings.ToLower(filepath.Ext(f.Path))) + out = append(out, pl) + } + return out +} + +// segmentMax keeps a folder or file name inside the 255 bytes most +// filesystems allow, with room for an extension and a temp prefix. +const segmentMax = 200 + +var illegal = strings.NewReplacer( + `\`, "+", "/", "+", "<", "", ">", "", "?", "!", "*", "-", "|", "", `"`, "", + // Lidarr's default "smart" colon: "Title: Subtitle" reads "Title - Subtitle". + ": ", " - ", ":", "-", +) + +// Segment makes a name safe as one path segment, the way Lidarr and Sonarr +// clean file names: their illegal-character mapping, no leading or trailing +// spaces or dots, at most segmentMax bytes, never "" or "..". +func Segment(s string) string { + s = illegal.Replace(s) + s = strings.Map(func(r rune) rune { + if r < 0x20 || r == 0x7f { + return -1 + } + return r + }, s) + s = strings.Trim(s, " .") + for len(s) > segmentMax { + _, size := utf8.DecodeLastRuneInString(s) + s = strings.TrimRight(s[:len(s)-size], " .") + } + if s == "" { + return "_" + } + return s +} diff --git a/internal/importer/place.go b/internal/importer/place.go new file mode 100644 index 00000000..5687ca4b --- /dev/null +++ b/internal/importer/place.go @@ -0,0 +1,182 @@ +package importer + +import ( + "context" + "crypto/rand" + "encoding/hex" + "errors" + "fmt" + "io" + "os" + "os/exec" + "path/filepath" + "strconv" + "strings" + "time" +) + +// ErrOccupied: the destination already has a file. Nothing is overwritten. +var ErrOccupied = errors.New("importer: destination already holds a file") + +// tagTimeout bounds one ffmpeg run. A remux copies the audio untouched, so a +// whole album track takes well under a second on local disk; two minutes is +// for a slow network mount, not for ffmpeg. +const tagTimeout = 2 * time.Minute + +// Tags are the names written into an imported file. Empty values are left +// out; the file's own tags stay for everything not named here. +type Tags struct { + Title, Artist, Album, AlbumArtist string + Year int + Disc, Discs, Track, Tracks int + // MusicBrainz ids, written the way Picard writes them. + ReleaseMBID, GroupMBID, RecordingMBID, ArtistMBID string +} + +// args turns the tags into ffmpeg -metadata pairs for a file of this +// extension. Vorbis comments (FLAC, Ogg, Opus) take Picard's key names; +// ID3 takes Picard's TXXX descriptions, which ffmpeg writes for any key it +// has no frame for. MP4's freeform atoms ffmpeg cannot write, so an M4A +// gets the plain names only. +func (t Tags) args(ext string) []string { + kv := [][2]string{ + {"title", t.Title}, {"artist", t.Artist}, {"album", t.Album}, {"album_artist", t.AlbumArtist}, + {"track", ofN(t.Track, t.Tracks)}, {"disc", ofN(t.Disc, t.Discs)}, + } + if t.Year > 0 { + kv = append(kv, [2]string{"date", strconv.Itoa(t.Year)}) + } + switch ext { + case ".flac", ".ogg", ".opus": + kv = append(kv, + [2]string{"MUSICBRAINZ_ALBUMID", t.ReleaseMBID}, + [2]string{"MUSICBRAINZ_RELEASEGROUPID", t.GroupMBID}, + [2]string{"MUSICBRAINZ_TRACKID", t.RecordingMBID}, + [2]string{"MUSICBRAINZ_ARTISTID", t.ArtistMBID}, + [2]string{"MUSICBRAINZ_ALBUMARTISTID", t.ArtistMBID}) + case ".mp3": + kv = append(kv, + [2]string{"MusicBrainz Album Id", t.ReleaseMBID}, + [2]string{"MusicBrainz Release Group Id", t.GroupMBID}, + [2]string{"MusicBrainz Track Id", t.RecordingMBID}, + [2]string{"MusicBrainz Artist Id", t.ArtistMBID}, + [2]string{"MusicBrainz Album Artist Id", t.ArtistMBID}) + } + var out []string + for _, p := range kv { + if p[1] != "" { + out = append(out, "-metadata", p[0]+"="+p[1]) + } + } + return out +} + +func ofN(n, of int) string { + switch { + case n <= 0: + return "" + case of > 0: + return fmt.Sprintf("%d/%d", n, of) + } + return strconv.Itoa(n) +} + +// taggable: the formats ffmpeg can rewrite tags in by remuxing. +var taggable = map[string]bool{".flac": true, ".ogg": true, ".opus": true, ".mp3": true, ".m4a": true} + +// Place copies src to dst with tags written, leaving src as it was (the +// download keeps seeding). The copy is written to a dot-named temp file +// beside dst, which the scanner skips, and renamed into place, so the +// library never sees half a file. Tags are written by an ffmpeg remux that +// copies the audio untouched; when ffmpeg cannot (it is missing, or the +// container will not take the file's streams) the file is copied as it is +// and tagged reports false. An existing dst is never overwritten. +func Place(ctx context.Context, src, dst string, t Tags) (tagged bool, err error) { + if _, err := os.Lstat(dst); err == nil { + return false, fmt.Errorf("%w: %s", ErrOccupied, dst) + } + dir := filepath.Dir(dst) + if err := os.MkdirAll(dir, 0o755); err != nil { + return false, fmt.Errorf("make %s: %w", dir, err) + } + ext := strings.ToLower(filepath.Ext(dst)) + tmp := filepath.Join(dir, ".minstrel-import-"+randomHex()+ext) + defer func() { _ = os.Remove(tmp) }() // gone already once renamed + + if taggable[ext] { + tagged = remux(ctx, src, tmp, t.args(ext)) == nil + } + if !tagged { + if err := copyFile(src, tmp); err != nil { + return false, err + } + } + if err := os.Chmod(tmp, 0o644); err != nil { + return false, fmt.Errorf("chmod %s: %w", tmp, err) + } + if _, err := os.Lstat(dst); err == nil { + return false, fmt.Errorf("%w: %s", ErrOccupied, dst) + } + if err := os.Rename(tmp, dst); err != nil { + return false, fmt.Errorf("move into place %s: %w", dst, err) + } + return tagged, nil +} + +// remux writes src to dst through ffmpeg with every stream copied as it is +// (the cover art too) and the tags set. Its output is checked against the +// source's size: a remux shifts container bytes, never the audio, so a file +// much smaller than its source lost something on the way. +func remux(ctx context.Context, src, dst string, meta []string) error { + ctx, cancel := context.WithTimeout(ctx, tagTimeout) + defer cancel() + args := append([]string{"-nostdin", "-v", "error", "-y", "-i", src, + "-map", "0", "-c", "copy", "-map_metadata", "0"}, meta...) + args = append(args, dst) + cmd := exec.CommandContext(ctx, "ffmpeg", args...) + if out, err := cmd.CombinedOutput(); err != nil { + _ = os.Remove(dst) + return fmt.Errorf("ffmpeg: %w: %s", err, strings.TrimSpace(string(out))) + } + si, err := os.Stat(src) + if err != nil { + return err + } + di, err := os.Stat(dst) + if err != nil { + return err + } + if di.Size() < si.Size()/2 { + _ = os.Remove(dst) + return fmt.Errorf("ffmpeg wrote %d bytes from %d", di.Size(), si.Size()) + } + return nil +} + +// copyFile copies src to dst byte for byte and syncs it. +func copyFile(src, dst string) error { + in, err := os.Open(src) + if err != nil { + return err + } + defer func() { _ = in.Close() }() + out, err := os.OpenFile(dst, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0o644) + if err != nil { + return err + } + if _, err := io.Copy(out, in); err != nil { + _ = out.Close() + return fmt.Errorf("copy %s: %w", src, err) + } + if err := out.Sync(); err != nil { + _ = out.Close() + return err + } + return out.Close() +} + +func randomHex() string { + b := make([]byte, 6) + _, _ = rand.Read(b) + return hex.EncodeToString(b) +} diff --git a/internal/importer/verify.go b/internal/importer/verify.go new file mode 100644 index 00000000..e5721fc2 --- /dev/null +++ b/internal/importer/verify.go @@ -0,0 +1,466 @@ +// Package importer checks a finished download against the album it was +// grabbed for and files it into the library (M502 #5522). Nothing reaches +// the library before the check says the files are the album: a wrong album, +// another version of it, a video rip or an incomplete copy is refused, its +// release is blocked and the album is searched for again. +// +// Ported from Fabled Librarian's importer and relocate packages, with the +// check rewritten for music: the files are matched to one of the album's +// MusicBrainz editions by AcoustID fingerprints, track lengths, titles and +// the tags they carry. +package importer + +import ( + "fmt" + "slices" + "sort" + "strings" + + "git.fabledsword.com/bvandeusen/minstrel/internal/release" +) + +// Outcome is what the check decided. +type Outcome string + +const ( + Accept Outcome = "accept" + // Review: the files may be the album and the evidence cannot say. A + // person decides. + Review Outcome = "review" + Refuse Outcome = "refuse" +) + +// Codes say why, beside the sentence in Verdict.Reason. +const ( + CodeMatched = "matched" + CodeForced = "forced" + CodeNoAudio = "no_audio" + CodeVideo = "video" + CodeWrongAlbum = "wrong_album" + CodeOtherVersion = release.VetoVersion + CodeWrongTitle = release.VetoTitle + CodeIncomplete = "incomplete" + CodeNoEdition = "no_edition" + CodeWeak = "weak" + CodeTooMany = "too_many" +) + +// maxFiles is the most audio files one album is checked for. More is a box +// set or a discography, which is a person's call, not a fingerprint queue. +const maxFiles = 60 + +// File is what one audio file in a download says about itself. +type File struct { + Path string // absolute, as Minstrel sees it + Rel string // under the download, for the person reading the verdict + // Disc and Track are the tags' positions; 0 when untagged. + Disc, Track int + Title string + Artist string + AlbumArtist string + Album string + // MusicBrainz ids from the tags, when a tagger wrote them. + GroupMBID string + ReleaseMBID string + RecordingMBID string + LengthMs int + // Recordings are the MusicBrainz recordings AcoustID says the audio is, + // at or above the operator's minimum score. + Recordings []string + // Video: the file name says it is a music video, a lyric video or the + // like (library.SourceMarkersFor). + Video bool +} + +// Track is one track of an edition. +type Track struct { + Medium int + Format string // the medium's: CD, Digital Media, Vinyl + Position int + Title string + RecordingMBID string + LengthMs int +} + +// Edition is one MusicBrainz release of the wanted release group, its +// tracks in medium and position order. +type Edition struct { + MBID string + Title string + Status string + Date string + Country string + Tracks []Track +} + +// Media is how many media the edition has. +func (e Edition) Media() int { + n := 0 + for _, t := range e.Tracks { + n = max(n, t.Medium) + } + return n +} + +// Want is the album the download was grabbed for. +type Want struct { + GroupMBID string + ArtistMBID string + Artist string + Title string + Year int + PrimaryType string + SecondaryTypes []string +} + +// Pair is a file matched to an edition's track, and what agreed. +type Pair struct { + File int `json:"file"` + Track int `json:"track"` + Recording bool `json:"recording"` + Length bool `json:"length"` + Title bool `json:"title"` +} + +// Evidence is what the check counted, kept with the verdict so a person can +// see what it rested on. +type Evidence struct { + Files int `json:"files"` + Tracks int `json:"tracks"` + // Identified: files with a recording from AcoustID or their tags. + Identified int `json:"identified"` + Recordings int `json:"recordings"` + Lengths int `json:"lengths"` + Titles int `json:"titles"` + // GroupTag: "same" when the tags name this release group, "other" when + // another, "" when they name none. + GroupTag string `json:"group_tag,omitempty"` + AlbumTag string `json:"album_tag,omitempty"` + // Fingerprinted: AcoustID was asked. False when it is not set up or + // could not be reached; the check then rests on lengths and tags. + Fingerprinted bool `json:"fingerprinted"` +} + +// Verdict is the check's answer. +type Verdict struct { + Outcome Outcome `json:"outcome"` + Code string `json:"code"` + Reason string `json:"reason"` + // Edition indexes the editions passed in; -1 when none was chosen. + Edition int `json:"-"` + Pairs []Pair `json:"pairs,omitempty"` + Evidence Evidence `json:"evidence"` +} + +// Verify decides whether files are the wanted album and, if so, which +// edition and which track each file is. force is a person's "import anyway": +// the best edition is taken whatever the evidence, but a download with no +// audio is still refused. +func Verify(w Want, editions []Edition, files []File, fingerprinted, force bool) Verdict { + v := Verdict{Edition: -1, Evidence: Evidence{Files: len(files), Fingerprinted: fingerprinted}} + if len(files) == 0 { + return v.set(Refuse, CodeNoAudio, "The download holds no audio files.") + } + if !force && len(files) > maxFiles { + return v.set(Review, CodeTooMany, fmt.Sprintf( + "The download holds %d audio files, more than one album; a person decides.", len(files))) + } + if video := countIf(files, func(f File) bool { return f.Video }); !force && video*2 > len(files) { + return v.set(Refuse, CodeVideo, fmt.Sprintf( + "%d of its %d audio files are named as music or lyric videos: the sound of a video, not the album.", video, len(files))) + } + + group := majority(files, func(f File) string { return f.GroupMBID }) + switch { + case group == "": + case strings.EqualFold(group, w.GroupMBID): + v.Evidence.GroupTag = "same" + default: + v.Evidence.GroupTag = "other" + } + album := majority(files, func(f File) string { return f.Album }) + v.Evidence.AlbumTag = album + var tagVeto release.Match + albumAgrees := false + if album != "" { + // The want's own artist goes in front: the tags' artist spelling is + // not what this asks. Only the album title and its version words. + m := release.Judge(release.Want{Artist: w.Artist, Title: w.Title, PrimaryType: w.PrimaryType, + SecondaryTypes: w.SecondaryTypes}, release.Parse(w.Artist+" - "+album)) + switch m.Veto { + case release.VetoVersion, release.VetoTitle: + tagVeto = m + case "": + albumAgrees = true + } + } + if !force && v.Evidence.GroupTag == "other" { + return v.set(Refuse, CodeWrongAlbum, fmt.Sprintf( + "Its tags name a different MusicBrainz release group (%s), not %s.", group, w.Title)) + } + + if len(editions) == 0 { + if force { + v.Code, v.Outcome, v.Reason = CodeForced, Accept, "Imported anyway at a person's word; MusicBrainz lists no edition to match it to." + return v + } + return v.set(Review, CodeNoEdition, fmt.Sprintf( + "MusicBrainz lists no edition of %s to check the files against; a person decides.", w.Title)) + } + + best, bestScore := -1, -1 + shortest, longest := len(editions[0].Tracks), 0 + for i, e := range editions { + shortest, longest = min(shortest, len(e.Tracks)), max(longest, len(e.Tracks)) + if len(e.Tracks) != len(files) && !force { + continue + } + pairs := pairUp(files, e.Tracks) + if s := score(pairs); s > bestScore || (s == bestScore && better(e, editions[best])) { + best, bestScore = i, s + } + } + if best < 0 { + if len(files) < shortest { + return v.set(Refuse, CodeIncomplete, fmt.Sprintf( + "It holds %d audio files; the shortest edition of %s has %d tracks.", len(files), w.Title, shortest)) + } + return v.set(Review, CodeNoEdition, fmt.Sprintf( + "It holds %d audio files; no edition of %s has %d tracks (they have %s). A person decides.", + len(files), w.Title, len(files), span(shortest, longest))) + } + + e := editions[best] + v.Edition = best + v.Pairs = pairUp(files, e.Tracks) + n := len(e.Tracks) + ev := &v.Evidence + ev.Tracks = n + for _, p := range v.Pairs { + f := files[p.File] + if f.RecordingMBID != "" || len(f.Recordings) > 0 { + ev.Identified++ + } + ev.Recordings += b2i(p.Recording) + ev.Lengths += b2i(p.Length) + ev.Titles += b2i(p.Title) + } + if force { + v.Outcome, v.Code = Accept, CodeForced + v.Reason = fmt.Sprintf("Imported anyway at a person's word, as %s. %s", describe(e), ev.sentence()) + return v + } + + half := (n + 1) / 2 + strong := ev.GroupTag == "same" || (ev.Identified >= half && ev.Recordings*10 >= ev.Identified*7) + switch { + case ev.Identified >= half && ev.Recordings*10 < ev.Identified*3 && ev.GroupTag != "same": + return v.set(Refuse, CodeWrongAlbum, fmt.Sprintf( + "Only %d of the %d tracks AcoustID knows are recordings on %s.", ev.Recordings, ev.Identified, w.Title)) + case tagVeto.Veto != "" && !strong: + return v.set(Refuse, tagVeto.Veto, fmt.Sprintf("Its tags call it %q. %s", album, tagVeto.Reason)) + case !strong && ev.Identified == 0 && ev.Lengths*2 < n: + return v.set(Refuse, CodeWrongAlbum, fmt.Sprintf( + "Only %d of its %d tracks are the length of %s's, and nothing else says it is the album.", ev.Lengths, n, w.Title)) + case strong && ev.Lengths*10 >= n*6, + !strong && ev.Lengths*10 >= n*8 && (ev.Titles*10 >= n*6 || albumAgrees): + v.Outcome, v.Code = Accept, CodeMatched + v.Reason = fmt.Sprintf("It is %s. %s", describe(e), ev.sentence()) + return v + } + return v.set(Review, CodeWeak, fmt.Sprintf( + "It may be %s, but the evidence is thin: %s A person decides.", describe(e), ev.sentence())) +} + +func (v Verdict) set(o Outcome, code, reason string) Verdict { + v.Outcome, v.Code, v.Reason = o, code, reason + return v +} + +// sentence says what agreed, for the verdict's reason. +func (ev Evidence) sentence() string { + var parts []string + if ev.GroupTag == "same" { + parts = append(parts, "its tags name this album") + } + if ev.Identified > 0 { + parts = append(parts, fmt.Sprintf("%d of %d identified tracks are its recordings", ev.Recordings, ev.Identified)) + } + parts = append(parts, fmt.Sprintf("%d of %d lengths fit", ev.Lengths, ev.Tracks), + fmt.Sprintf("%d of %d titles agree", ev.Titles, ev.Tracks)) + s := strings.Join(parts, ", ") + s = strings.ToUpper(s[:1]) + s[1:] + "." + if !ev.Fingerprinted { + s += " AcoustID was not asked (not set up or unreachable), so lengths and tags decided." + } + return s +} + +// describe names an edition for a person: "the 2013 CD release (13 tracks)". +func describe(e Edition) string { + var b strings.Builder + b.WriteString("the ") + if len(e.Date) >= 4 { + b.WriteString(e.Date[:4] + " ") + } + if f := e.format(); f != "" { + b.WriteString(f + " ") + } + fmt.Fprintf(&b, "edition (%d tracks)", len(e.Tracks)) + return b.String() +} + +func (e Edition) format() string { + if len(e.Tracks) == 0 { + return "" + } + f := e.Tracks[0].Format + for _, t := range e.Tracks { + if t.Format != f { + return "" + } + } + return f +} + +// pairUp matches files to tracks in order: by the tags' disc and track +// numbers when every file has them and no two share a place, by file path +// otherwise. With a person's "import anyway" the counts may differ; pairs +// run to the shorter. +func pairUp(files []File, tracks []Track) []Pair { + order := make([]int, len(files)) + for i := range order { + order[i] = i + } + if numbered(files) { + sort.SliceStable(order, func(a, b int) bool { + fa, fb := files[order[a]], files[order[b]] + if da, db := max(fa.Disc, 1), max(fb.Disc, 1); da != db { + return da < db + } + return fa.Track < fb.Track + }) + } else { + sort.SliceStable(order, func(a, b int) bool { return files[order[a]].Rel < files[order[b]].Rel }) + } + n := min(len(files), len(tracks)) + pairs := make([]Pair, n) + for i := range n { + f, t := files[order[i]], tracks[i] + pairs[i] = Pair{ + File: order[i], + Track: i, + Recording: t.RecordingMBID != "" && (f.RecordingMBID == t.RecordingMBID || slices.Contains(f.Recordings, t.RecordingMBID)), + Length: lengthFits(f.LengthMs, t.LengthMs), + Title: titleAgrees(f.Title, t.Title), + } + } + return pairs +} + +// numbered: every file carries a track number and no two share a disc and +// track. +func numbered(files []File) bool { + seen := map[[2]int]bool{} + for _, f := range files { + k := [2]int{max(f.Disc, 1), f.Track} + if f.Track <= 0 || seen[k] { + return false + } + seen[k] = true + } + return true +} + +// lengthFits allows five seconds or 3%, whichever is more: remasters and +// rips differ by a second or two, a different recording by far more. A +// track MusicBrainz has no length for says nothing against the file. +func lengthFits(fileMs, trackMs int) bool { + if trackMs <= 0 { + return true + } + if fileMs <= 0 { + return false + } + tol := max(5000, trackMs*3/100) + d := fileMs - trackMs + return d <= tol && -d <= tol +} + +// titleAgrees: the same words, or one title holding all of the other's +// ("Get Lucky (Radio Edit)" holds "Get Lucky"). +func titleAgrees(a, b string) bool { + ta, tb := release.Tokens(a), release.Tokens(b) + if len(ta) == 0 || len(tb) == 0 { + return false + } + return subset(ta, tb) || subset(tb, ta) +} + +func subset(small, big []string) bool { + for _, w := range small { + if !slices.Contains(big, w) { + return false + } + } + return true +} + +// score weighs recordings above lengths above titles: a fingerprint is the +// audio itself, a length is the audio's shape, a title is only a name. +func score(pairs []Pair) int { + s := 0 + for _, p := range pairs { + s += 3*b2i(p.Recording) + 2*b2i(p.Length) + b2i(p.Title) + } + return s +} + +// better breaks a tie between equally good editions: an official one, then +// the earlier. +func better(a, b Edition) bool { + if (a.Status == "Official") != (b.Status == "Official") { + return a.Status == "Official" + } + return a.Date != "" && (b.Date == "" || a.Date < b.Date) +} + +func majority(files []File, get func(File) string) string { + counts := map[string]int{} + best, n := "", 0 + for _, f := range files { + v := strings.TrimSpace(get(f)) + if v == "" { + continue + } + counts[strings.ToLower(v)]++ + if c := counts[strings.ToLower(v)]; c > n { + best, n = v, c + } + } + return best +} + +func countIf(files []File, f func(File) bool) int { + n := 0 + for _, x := range files { + if f(x) { + n++ + } + } + return n +} + +func span(a, b int) string { + if a == b { + return fmt.Sprint(a) + } + return fmt.Sprintf("%d to %d", a, b) +} + +func b2i(b bool) int { + if b { + return 1 + } + return 0 +} diff --git a/internal/importer/verify_test.go b/internal/importer/verify_test.go new file mode 100644 index 00000000..6bf8425c --- /dev/null +++ b/internal/importer/verify_test.go @@ -0,0 +1,243 @@ +package importer + +import ( + "fmt" + "path/filepath" + "strings" + "testing" +) + +var ram = Want{GroupMBID: "rg-ram", ArtistMBID: "mb-dp", Artist: "Daft Punk", Title: "Random Access Memories", + Year: 2013, PrimaryType: "Album"} + +var ramTitles = []string{"Give Life Back to Music", "The Game of Love", "Giorgio by Moroder", "Within", + "Instant Crush", "Lose Yourself to Dance", "Touch", "Get Lucky", "Beyond", "Motherboard", + "Fragments of Time", "Doin' It Right", "Contact"} + +// edition is RAM's 13-track CD, recordings rec-01.., lengths 4:00 + n seconds. +func edition(mbid string, n int) Edition { + e := Edition{MBID: mbid, Title: "Random Access Memories", Status: "Official", Date: "2013-05-17"} + for i := range n { + e.Tracks = append(e.Tracks, Track{Medium: 1, Format: "CD", Position: i + 1, Title: ramTitles[i%len(ramTitles)], + RecordingMBID: fmt.Sprintf("rec-%02d", i+1), LengthMs: 240000 + i*1000}) + } + return e +} + +// download is n files lined up with the edition: tagged positions, lengths +// within a second, AcoustID naming each file's recording. +func download(n int) []File { + var fs []File + for i := range n { + fs = append(fs, File{Path: fmt.Sprintf("/dl/%02d.flac", i+1), Rel: fmt.Sprintf("%02d.flac", i+1), + Disc: 1, Track: i + 1, Title: ramTitles[i%len(ramTitles)], Album: "Random Access Memories", + LengthMs: 240000 + i*1000 + 400, Recordings: []string{fmt.Sprintf("rec-%02d", i+1)}}) + } + return fs +} + +func TestVerify_TheAlbumAndWhatIsNot(t *testing.T) { + eds := []Edition{edition("rel-cd", 13), edition("rel-deluxe", 15)} + cases := []struct { + name string + files func() []File + printed bool + out Outcome + code string + edition string + }{ + {"the album, fingerprinted", func() []File { return download(13) }, true, Accept, CodeMatched, "rel-cd"}, + {"the deluxe by count", func() []File { return download(15) }, true, Accept, CodeMatched, "rel-deluxe"}, + {"no audio", func() []File { return nil }, true, Refuse, CodeNoAudio, ""}, + {"music videos", func() []File { + fs := download(13) + for i := range fs { + fs[i].Video = i < 10 + } + return fs + }, true, Refuse, CodeVideo, ""}, + {"tags name another group", func() []File { + fs := download(13) + for i := range fs { + fs[i].GroupMBID = "rg-homework" + } + return fs + }, true, Refuse, CodeWrongAlbum, ""}, + {"fingerprints say other recordings", func() []File { + fs := download(13) + for i := range fs { + fs[i].Recordings = []string{"rec-other"} + } + return fs + }, true, Refuse, CodeWrongAlbum, "rel-cd"}, + {"the drumless edition: right lengths, album tag says otherwise", func() []File { + fs := download(13) + for i := range fs { + fs[i].Recordings, fs[i].Album = nil, "Random Access Memories (Drumless Edition)" + } + return fs + }, false, Refuse, CodeOtherVersion, "rel-cd"}, + {"fewer files than any edition", func() []File { return download(9) }, true, Refuse, CodeIncomplete, ""}, + {"no edition this long", func() []File { return download(14) }, true, Review, CodeNoEdition, ""}, + {"no fingerprints, lengths and titles agree", func() []File { + fs := download(13) + for i := range fs { + fs[i].Recordings = nil + } + return fs + }, false, Accept, CodeMatched, "rel-cd"}, + {"no fingerprints, no titles, nothing to go on but lengths", func() []File { + fs := download(13) + for i := range fs { + fs[i].Recordings, fs[i].Title, fs[i].Album = nil, "", "" + } + return fs + }, false, Review, CodeWeak, "rel-cd"}, + {"no fingerprints, lengths all wrong", func() []File { + fs := download(13) + for i := range fs { + fs[i].Recordings, fs[i].LengthMs = nil, 600000 + } + return fs + }, false, Refuse, CodeWrongAlbum, "rel-cd"}, + {"tags name the group: strong even unprinted", func() []File { + fs := download(13) + for i := range fs { + fs[i].Recordings, fs[i].Title, fs[i].GroupMBID = nil, "", "rg-ram" + } + return fs + }, false, Accept, CodeMatched, "rel-cd"}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + v := Verify(ram, eds, c.files(), c.printed, false) + if v.Outcome != c.out || v.Code != c.code { + t.Fatalf("got %s/%s (%s), want %s/%s", v.Outcome, v.Code, v.Reason, c.out, c.code) + } + got := "" + if v.Edition >= 0 { + got = eds[v.Edition].MBID + } + if got != c.edition { + t.Errorf("edition %q, want %q", got, c.edition) + } + if v.Reason == "" { + t.Error("no reason given") + } + }) + } +} + +func TestVerify_UnnumberedFilesPairByPath(t *testing.T) { + fs := download(13) + for i := range fs { + fs[i].Track, fs[i].Disc = 0, 0 + } + fs[0], fs[5] = fs[5], fs[0] // listed out of order; their names sort right + v := Verify(ram, []Edition{edition("rel-cd", 13)}, fs, true, false) + if v.Outcome != Accept || v.Evidence.Recordings != 13 { + t.Fatalf("got %+v", v) + } +} + +func TestVerify_ImportAnywayTakesTheBestEdition(t *testing.T) { + v := Verify(ram, []Edition{edition("rel-cd", 13)}, download(14), true, true) + if v.Outcome != Accept || v.Code != CodeForced || v.Edition != 0 || len(v.Pairs) != 13 { + t.Fatalf("got %+v", v) + } + if v := Verify(ram, nil, nil, true, true); v.Outcome != Refuse { + t.Errorf("forced with no audio: %+v", v) + } +} + +func TestVerify_WithoutAcoustIDTheReasonSaysSo(t *testing.T) { + fs := download(13) + for i := range fs { + fs[i].Recordings = nil + } + v := Verify(ram, []Edition{edition("rel-cd", 13)}, fs, false, false) + if !strings.Contains(v.Reason, "AcoustID was not asked") { + t.Errorf("reason %q", v.Reason) + } +} + +func TestLengthFits(t *testing.T) { + for _, c := range []struct { + file, track int + ok bool + }{ + {240000, 244000, true}, {240000, 246000, false}, {600000, 615000, true}, {600000, 620000, false}, + {0, 240000, false}, {240000, 0, true}, + } { + if got := lengthFits(c.file, c.track); got != c.ok { + t.Errorf("lengthFits(%d, %d) = %v", c.file, c.track, got) + } + } +} + +func TestPlan_LidarrsNaming(t *testing.T) { + e := edition("rel-cd", 2) + e.Tracks[1].Title = "Within: Part 2/3?" + pl := Plan("/music", ram, &e, download(2), Verify(ram, []Edition{e}, download(2), true, false).Pairs) + want := []string{ + "/music/Daft Punk/Random Access Memories (2013)/Daft Punk - Random Access Memories - 01 - Give Life Back to Music.flac", + "/music/Daft Punk/Random Access Memories (2013)/Daft Punk - Random Access Memories - 02 - Within - Part 2+3!.flac", + } + for i, p := range pl { + if p.Dest != want[i] { + t.Errorf("dest %d = %q, want %q", i, p.Dest, want[i]) + } + } + if pl[0].Recording != "rec-01" || pl[0].Tracks != 2 { + t.Errorf("placement %+v", pl[0]) + } +} + +func TestPlan_MultiDiscGetsAMediumFolder(t *testing.T) { + e := edition("rel-2cd", 4) + e.Tracks[2].Medium, e.Tracks[2].Position = 2, 1 + e.Tracks[3].Medium, e.Tracks[3].Position = 2, 2 + fs := download(4) + fs[2].Disc, fs[2].Track, fs[3].Disc, fs[3].Track = 2, 1, 2, 2 + pl := Plan("/music", ram, &e, fs, Verify(ram, []Edition{e}, fs, true, false).Pairs) + if got := filepath.Dir(pl[3].Dest); got != "/music/Daft Punk/Random Access Memories (2013)/CD 02" { + t.Errorf("disc folder %q", got) + } + if !strings.HasSuffix(pl[3].Dest, " - 02 - Get Lucky.flac") && !strings.Contains(pl[3].Dest, " - 02 - ") { + t.Errorf("dest %q", pl[3].Dest) + } +} + +func TestSegment(t *testing.T) { + for in, want := range map[string]string{ + "AC/DC": "AC+DC", `What?`: "What!", "Title: Sub": "Title - Sub", "a:b": "a-b", " ..x.. ": "x", + "": "_", "..": "_", `"Quoted" *y* |z|`: "Quoted x -y- z", + } { + if got := Segment(in); got != want { + t.Errorf("Segment(%q) = %q, want %q", in, got, want) + } + } + if long := Segment(strings.Repeat("é", 200)); len(long) > segmentMax { + t.Errorf("long segment is %d bytes", len(long)) + } +} + +func TestTagsArgs(t *testing.T) { + tg := Tags{Title: "Get Lucky", Artist: "Daft Punk", Track: 8, Tracks: 13, Disc: 1, Discs: 1, Year: 2013, + RecordingMBID: "rec-08", GroupMBID: "rg-ram"} + flac := strings.Join(tg.args(".flac"), " ") + for _, s := range []string{"title=Get Lucky", "track=8/13", "date=2013", "MUSICBRAINZ_TRACKID=rec-08", "MUSICBRAINZ_RELEASEGROUPID=rg-ram"} { + if !strings.Contains(flac, s) { + t.Errorf("flac args lack %q: %s", s, flac) + } + } + if strings.Contains(flac, "MUSICBRAINZ_ALBUMID") { + t.Errorf("an empty release id was written: %s", flac) + } + if mp3 := strings.Join(tg.args(".mp3"), " "); !strings.Contains(mp3, "MusicBrainz Track Id=rec-08") { + t.Errorf("mp3 args %s", mp3) + } + if m4a := strings.Join(tg.args(".m4a"), " "); strings.Contains(m4a, "MusicBrainz") { + t.Errorf("m4a args %s", m4a) + } +} diff --git a/internal/library/fingerprint.go b/internal/library/fingerprint.go index 12890a78..8b69449a 100644 --- a/internal/library/fingerprint.go +++ b/internal/library/fingerprint.go @@ -422,3 +422,16 @@ func notURLBase64(r rune) bool { } return true } + +// LookupFingerprint is the compressed fingerprint AcoustID's lookup takes and +// the file's whole length in seconds, for a file not in the library yet: the +// import checks a download with it (M502 #5522). IsInconclusive tells a +// missing or stalled fpcalc from a file it rejected. +func LookupFingerprint(ctx context.Context, path string) (fingerprint string, durationSec int, err error) { + fp, err := computeLookupFingerprint(ctx, path) + return fp.fingerprint, fp.durationSec, err +} + +// IsInconclusive reports that a fingerprint tool could not give an answer +// (missing, timed out), as opposed to rejecting the file. +func IsInconclusive(err error) bool { return isInconclusive(err) } diff --git a/internal/library/scanner.go b/internal/library/scanner.go index 4fcd73c9..2c986caa 100644 --- a/internal/library/scanner.go +++ b/internal/library/scanner.go @@ -674,3 +674,16 @@ func probeDurationMs(ctx context.Context, path string) (int32, error) { } return int32(ms), nil } + +// ProbeDurationMs is a file's length in milliseconds, read by ffprobe. +func ProbeDurationMs(ctx context.Context, path string) (int32, error) { + return probeDurationMs(ctx, path) +} + +// IsAudioFile reports whether the scanner reads a file with this name. +func IsAudioFile(path string) bool { + return audioExtensions[strings.ToLower(filepath.Ext(path))] +} + +// Roots are the folders the scanner reads: the library. +func (s *Scanner) Roots() []string { return append([]string(nil), s.paths...) } diff --git a/internal/wanted/search.go b/internal/wanted/search.go index e13b6e8a..c8aa7885 100644 --- a/internal/wanted/search.go +++ b/internal/wanted/search.go @@ -180,6 +180,7 @@ func (s *Searcher) pass(ctx context.Context) Report { rep.Nothing++ default: rep.Failed++ + s.logger.Warn("wanted: search failed", "want", want.Artist+" - "+want.Title, "err", out.Error) } } return rep diff --git a/web/src/lib/api/queries.ts b/web/src/lib/api/queries.ts index 8abd41d4..2877ca37 100644 --- a/web/src/lib/api/queries.ts +++ b/web/src/lib/api/queries.ts @@ -48,6 +48,9 @@ export const qk = { // Search and grab (M502 #5521). wantCandidates: (wantId: string) => ['wantCandidates', wantId] as const, searchSettings: () => ['searchSettings'] as const, + // The Wanted view's Activity and Blocked tabs (#5522). + downloads: () => ['adminWants', 'downloads'] as const, + releaseBlocks: () => ['adminWants', 'releaseBlocks'] as const, myQuarantine: () => ['myQuarantine'] as const, adminQuarantine: () => ['adminQuarantine'] as const, adminQuarantineActions: (limit?: number) => diff --git a/web/src/lib/api/types.ts b/web/src/lib/api/types.ts index 783a5985..c0cc5728 100644 --- a/web/src/lib/api/types.ts +++ b/web/src/lib/api/types.ts @@ -606,11 +606,32 @@ export type WantSearch = { error: string; }; +// What an import made of a finished download (M502 #5522). +export type ImportState = + | '' + | 'downloading' + | 'waiting' + | 'importing' + | 'imported' + | 'refused' + | 'review' + | 'held' + | 'failed'; + +// A want's latest download, running or ended. export type WantGrab = { + id: string; name: string; grabbed_at: string; origin: 'schedule' | 'operator'; quality: string; + live: boolean; + end_reason: string; + progress: number; + import_state: ImportState; + import_reason: string; + imported_at: string | null; + imported_files: number; }; // Search and grab (M502 #5521) ---------------------------------------------- @@ -685,9 +706,77 @@ export type SearchSettings = { tiers: QualityTier[]; last_pass_at: string | null; last_pass: SearchPass; + // The library folder new music is filed into ("" = the first of + // scan_paths), and the library's folders (#5522). + import_root: string; + scan_paths: string[]; }; -export type WantCounts = { wanted: number; missing: number; held: number; following: number }; +export type WantCounts = { + wanted: number; + missing: number; + held: number; + following: number; + // Downloads running, those of them waiting on a person, releases set aside. + activity: number; + attention: number; + blocked: number; +}; + +// What an import's check counted (#5522). +export type ImportEvidence = { + files: number; + tracks: number; + identified: number; + recordings: number; + lengths: number; + titles: number; + group_tag?: string; + album_tag?: string; + fingerprinted: boolean; +}; + +// One download on the Activity tab. +export type Download = { + id: string; + want_id: string | null; + want_title: string; + want_artist: string; + name: string; + indexer: string; + client: string; + quality: string; + origin: 'schedule' | 'operator'; + grabbed_at: string; + ended_at: string | null; + end_reason: string; + progress: number; + import_state: ImportState; + import_reason: string; + imported_at: string | null; + imported_files: number; + destination: string; + edition?: string; + evidence?: ImportEvidence; + files: string[]; + skipped: number; + rehomed_from?: string; +}; + +export type DownloadsPage = { items: Download[]; limit: number; offset: number }; + +// A release set aside: by a person, by its files, or by an import's check. +export type ReleaseBlock = { + info_hash: string; + name: string; + reason: 'operator' | 'files' | 'import'; + detail: string; + blocked_at: string; + want_title: string; + want_artist: string; +}; + +export type ReleaseBlocksPage = { items: ReleaseBlock[]; limit: number; offset: number }; export type WantsPage = Page & { counts: WantCounts }; diff --git a/web/src/lib/api/wanted.ts b/web/src/lib/api/wanted.ts index 4d201d45..f09aa9aa 100644 --- a/web/src/lib/api/wanted.ts +++ b/web/src/lib/api/wanted.ts @@ -3,7 +3,9 @@ import { api } from './client'; import { qk } from './queries'; import type { Candidate, + DownloadsPage, Follow, + ReleaseBlocksPage, FollowState, ReleaseProfile, ReleaseProfileInput, @@ -157,6 +159,56 @@ export async function putSearchSettings(body: { auto_grab: boolean; search_interval_hours: number; allowed_qualities: string[]; + import_root: string; }): Promise { return api.put('/api/admin/acquisition/settings', body); } + +// Downloads and their imports (M502 #5522) ---------------------------------- + +// A page holding fewer than it asked for is the last. +function nextOffset(last: { items: unknown[]; limit: number; offset: number }): number | undefined { + return last.items.length < last.limit ? undefined : last.offset + last.items.length; +} + +export async function listDownloads(offset: number = 0): Promise { + return api.get(`/api/admin/downloads?offset=${offset}`); +} + +// Running downloads first; refreshed every few seconds while one runs. +export function createDownloadsQuery() { + return createInfiniteQuery({ + queryKey: qk.downloads(), + queryFn: ({ pageParam }) => listDownloads(pageParam as number), + initialPageParam: 0, + getNextPageParam: nextOffset, + refetchInterval: (q) => + q.state.data?.pages[0]?.items.some((d) => !d.ended_at) ? 5_000 : false + }); +} + +export async function importAnyway(id: string): Promise { + await api.post(`/api/admin/downloads/${id}/import`, {}); +} + +export async function retryImport(id: string): Promise { + await api.post(`/api/admin/downloads/${id}/retry`, {}); +} + +export async function refuseDownload(id: string): Promise { + await api.post(`/api/admin/downloads/${id}/refuse`, {}); +} + +export async function listReleaseBlocks(offset: number = 0): Promise { + return api.get(`/api/admin/release-blocks?offset=${offset}`); +} + +export function createReleaseBlocksQuery() { + return createInfiniteQuery({ + queryKey: qk.releaseBlocks(), + queryFn: ({ pageParam }) => listReleaseBlocks(pageParam as number), + initialPageParam: 0, + getNextPageParam: nextOffset, + staleTime: 30_000 + }); +} diff --git a/web/src/lib/components/AdminTabs.svelte b/web/src/lib/components/AdminTabs.svelte index 8c51bc8e..e75b3e54 100644 --- a/web/src/lib/components/AdminTabs.svelte +++ b/web/src/lib/components/AdminTabs.svelte @@ -7,7 +7,6 @@ { href: '/admin', label: 'Overview' }, { href: '/admin/integrations', label: 'Integrations' }, { href: '/admin/requests', label: 'Requests' }, - { href: '/admin/wanted', label: 'Wanted' }, { href: '/admin/quarantine', label: 'Quarantine' }, { href: '/admin/missing-files', label: 'Missing files' }, { href: '/admin/duplicates', label: 'Duplicates' }, diff --git a/web/src/lib/components/AdminTabs.test.ts b/web/src/lib/components/AdminTabs.test.ts index 8f180ea5..205a257a 100644 --- a/web/src/lib/components/AdminTabs.test.ts +++ b/web/src/lib/components/AdminTabs.test.ts @@ -52,7 +52,7 @@ describe('AdminTabs', () => { ); }); - test('renders all twelve tabs in order', () => { + test('renders all eleven tabs in order', () => { state.pageUrl = new URL('http://localhost/admin'); render(AdminTabs); const links = screen.getAllByRole('link'); @@ -60,8 +60,6 @@ describe('AdminTabs', () => { 'Overview', 'Integrations', 'Requests', - // Wanted follows Requests: an approved request goes on the wanted list. - 'Wanted', 'Quarantine', // Missing files sits with Quarantine and Playback errors: the three // surfaces that show tracks needing an operator's attention. diff --git a/web/src/lib/components/ReacquisitionSettingsCard.svelte b/web/src/lib/components/ReacquisitionSettingsCard.svelte index 3c210dbc..490b7664 100644 --- a/web/src/lib/components/ReacquisitionSettingsCard.svelte +++ b/web/src/lib/components/ReacquisitionSettingsCard.svelte @@ -74,7 +74,7 @@

Automatic re-acquisition

When a file has been missing for a while, Minstrel can put its album back on the - wanted list + wanted list by itself. It's wanted per album, not per track: a whole folder going missing is one want, not forty.

diff --git a/web/src/lib/components/ReleaseProfilesCard.svelte b/web/src/lib/components/ReleaseProfilesCard.svelte index 47af14e7..58a3e292 100644 --- a/web/src/lib/components/ReleaseProfilesCard.svelte +++ b/web/src/lib/components/ReleaseProfilesCard.svelte @@ -167,8 +167,7 @@

Release-type profiles

Each followed artist is on one profile, which says which of their new releases are - wanted. Requests for a - specific album aren't filtered. + wanted. Requests for a specific album aren't filtered.

diff --git a/web/src/lib/components/SearchSettingsCard.svelte b/web/src/lib/components/SearchSettingsCard.svelte index fb97fdcf..f7831722 100644 --- a/web/src/lib/components/SearchSettingsCard.svelte +++ b/web/src/lib/components/SearchSettingsCard.svelte @@ -10,13 +10,14 @@ // Searching and grabbing (M502 #5521): how often each want is searched // for, whether the best release is taken without asking, and which // qualities are taken at all. The schedule wakes every ten minutes and - // keeps to each indexer's searches-per-hour. + // keeps to each indexer's searches-per-hour. A finished download is + // checked and filed into one of the library's folders (#5522). const client = useQueryClient(); const store = createSearchSettingsQuery(); const query = $derived($store); - type Form = { auto_grab: boolean; search_interval_hours: number; allowed: string[] }; + type Form = { auto_grab: boolean; search_interval_hours: number; allowed: string[]; import_root: string }; // The form is the server's settings until edited; saved holds what a save // returned until the query catches up. @@ -27,7 +28,12 @@ const form = $derived
(edited ?? (base ? fromSettings(base) : null)); function fromSettings(s: SearchSettings): Form { - return { auto_grab: s.auto_grab, search_interval_hours: s.search_interval_hours, allowed: [...s.allowed_qualities] }; + return { + auto_grab: s.auto_grab, + search_interval_hours: s.search_interval_hours, + allowed: [...s.allowed_qualities], + import_root: s.import_root + }; } const dirty = $derived(!!edited && !!base && JSON.stringify(edited) !== JSON.stringify(fromSettings(base))); @@ -48,7 +54,8 @@ saved = await putSearchSettings({ auto_grab: form.auto_grab, search_interval_hours: Number(form.search_interval_hours), - allowed_qualities: form.allowed + allowed_qualities: form.allowed, + import_root: form.import_root }); edited = null; await client.invalidateQueries({ queryKey: qk.searchSettings() }); @@ -81,7 +88,7 @@

Searching and grabbing

Minstrel searches your indexers for each album on the - wanted list, judges every + wanted list, judges every release it finds, and hands the best one to your download client. Each album's releases and the reason for every verdict are on the Wanted page.

@@ -146,6 +153,32 @@ + {#if query.data.scan_paths.length > 1} + + {:else if query.data.scan_paths.length === 1} +

+ A download that checks out is copied into {query.data.scan_paths[0]} + as Artist/Album (Year)/Artist - Album - NN - Title, tagged with its MusicBrainz ids. The + download itself keeps seeding. +

+ {/if} +

{lastPass(query.data)}

diff --git a/web/src/lib/components/SearchSettingsCard.test.ts b/web/src/lib/components/SearchSettingsCard.test.ts index c5b919be..b93a0c86 100644 --- a/web/src/lib/components/SearchSettingsCard.test.ts +++ b/web/src/lib/components/SearchSettingsCard.test.ts @@ -22,7 +22,9 @@ const settings: SearchSettings = { { tier: 'unknown', label: 'Unknown', what: 'Nothing says what format it is' } ], last_pass_at: new Date(Date.now() - 5 * 60_000).toISOString(), - last_pass: { looked: 12, grabbed: 3, waiting: 2, nothing: 7, failed: 0, paced: true } + last_pass: { looked: 12, grabbed: 3, waiting: 2, nothing: 7, failed: 0, paced: true }, + import_root: '', + scan_paths: ['/music', '/music2'] }; afterEach(() => vi.clearAllMocks()); @@ -57,11 +59,29 @@ describe('SearchSettingsCard', () => { expect(putSearchSettings).toHaveBeenCalledWith({ auto_grab: false, search_interval_hours: 24, - allowed_qualities: ['lossless', 'lossy_high', 'unknown'] + allowed_qualities: ['lossless', 'lossy_high', 'unknown'], + import_root: '' }) ); }); + test('new music is filed into the chosen library folder, the first by default', async () => { + setup(); + const folder = screen.getByLabelText(/file new music into/i) as HTMLSelectElement; + expect(folder.value).toBe('/music'); + await fireEvent.change(folder, { target: { value: '/music2' } }); + await fireEvent.click(screen.getByRole('button', { name: /^save$/i })); + await waitFor(() => + expect(putSearchSettings).toHaveBeenCalledWith(expect.objectContaining({ import_root: '/music2' })) + ); + }); + + test('with one library folder there is nothing to choose', () => { + setup({ ...settings, scan_paths: ['/music'] }); + expect(screen.queryByLabelText(/file new music into/i)).toBeNull(); + expect(screen.getByText('/music')).toBeTruthy(); + }); + test('before the first pass it says so', () => { setup({ ...settings, last_pass_at: null, last_pass: {} }); expect(screen.getByTestId('last-pass').textContent).toMatch(/hasn’t run yet/); diff --git a/web/src/lib/components/Shell.svelte b/web/src/lib/components/Shell.svelte index f534c562..5caf9946 100644 --- a/web/src/lib/components/Shell.svelte +++ b/web/src/lib/components/Shell.svelte @@ -1,7 +1,7 @@ e.key === 'Escape' && (menuOpen = false)} /> diff --git a/web/src/lib/components/Shell.test.ts b/web/src/lib/components/Shell.test.ts index f7b658df..92a63aab 100644 --- a/web/src/lib/components/Shell.test.ts +++ b/web/src/lib/components/Shell.test.ts @@ -67,6 +67,21 @@ describe('Shell', () => { expect(screen.getByRole('link', { name: 'Discover' })).toHaveAttribute('href', '/discover'); }); + test('admins get Wanted after Discover; others do not', () => { + const primary = () => + screen + .getAllByRole('link') + .filter((el) => el.closest('nav[aria-label="Primary"]') !== null) + .map((el) => el.textContent?.trim()); + const { unmount } = render(Shell); + expect(primary()).not.toContain('Wanted'); + unmount(); + userState.current = { id: '1', username: 'alice', is_admin: true }; + render(Shell); + expect(primary()).toEqual(['Home', 'Library', 'Discover', 'Wanted']); + expect(screen.getByRole('link', { name: 'Wanted' })).toHaveAttribute('href', '/wanted'); + }); + test('main nav omits Artists/Albums/Liked/History/Playlists/Search/Settings/Admin', () => { userState.current = { id: '1', username: 'alice', is_admin: true }; render(Shell); diff --git a/web/src/lib/components/WantedTabs.svelte b/web/src/lib/components/WantedTabs.svelte new file mode 100644 index 00000000..3a9bf1a4 --- /dev/null +++ b/web/src/lib/components/WantedTabs.svelte @@ -0,0 +1,54 @@ + + + diff --git a/web/src/lib/components/wanted/BlockList.svelte b/web/src/lib/components/wanted/BlockList.svelte new file mode 100644 index 00000000..cd41332a --- /dev/null +++ b/web/src/lib/components/wanted/BlockList.svelte @@ -0,0 +1,88 @@ + + +{#if blocks.isPending} +

Reading the blocked releases…

+{:else if blocks.isError && !blocks.isFetchNextPageError} +

Couldn't load the blocked releases.

+{:else if items.length === 0} +
+ +

No releases are blocked.

+

+ A release you set aside, or one an import found wasn't the album, lands here so it's never + taken again. +

+
+{:else} +
    + {#each items as b (b.info_hash)} +
  • +
    +
    {b.name || b.info_hash}
    + {#if b.want_title} +
    For {b.want_artist} – {b.want_title}
    + {/if} +
    {by(b)} {relativeTime(b.blocked_at)}
    + {#if b.detail} +

    {b.detail}

    + {/if} +
    + +
    • + {/each} +
+ + blocks.fetchNextPage()} + /> +{/if} diff --git a/web/src/lib/components/wanted/BlockList.test.ts b/web/src/lib/components/wanted/BlockList.test.ts new file mode 100644 index 00000000..bd3574c3 --- /dev/null +++ b/web/src/lib/components/wanted/BlockList.test.ts @@ -0,0 +1,50 @@ +import { afterEach, describe, expect, test, vi } from 'vitest'; +import { fireEvent, render, screen, waitFor } from '@testing-library/svelte'; +import { mockInfiniteQuery } from '../../../test-utils/query'; +import type { ReleaseBlock } from '#lib/api/types.js'; + +vi.mock('#lib/api/wanted.js', () => ({ + createReleaseBlocksQuery: vi.fn(), + unblockRelease: vi.fn().mockResolvedValue(undefined) +})); +vi.mock('#lib/stores/toast.svelte.js', () => ({ pushToast: vi.fn() })); + +import BlockList from './BlockList.svelte'; +import { createReleaseBlocksQuery, unblockRelease } from '#lib/api/wanted.js'; + +const block: ReleaseBlock = { + info_hash: 'abc123', + name: 'Daft Punk - Random Access Memories (Drumless Edition)', + reason: 'import', + detail: 'The album tag says another version.', + blocked_at: new Date(Date.now() - 3_600_000).toISOString(), + want_title: 'Random Access Memories', + want_artist: 'Daft Punk' +}; + +afterEach(() => vi.clearAllMocks()); + +function setup(items: ReleaseBlock[]) { + vi.mocked(createReleaseBlocksQuery).mockReturnValue( + mockInfiniteQuery({ pages: [{ items, total: items.length, limit: 50, offset: 0 }] }) as unknown as ReturnType< + typeof createReleaseBlocksQuery + > + ); + return render(BlockList); +} + +describe('BlockList', () => { + test('a blocked release says why and can be allowed again', async () => { + setup([block]); + expect(screen.getByText(/refused on import/i)).toBeTruthy(); + expect(screen.getByText(/another version/)).toBeTruthy(); + expect(screen.getByText(/for daft punk – random access memories/i)).toBeTruthy(); + await fireEvent.click(screen.getByRole('button', { name: /allow .* again/i })); + await waitFor(() => expect(unblockRelease).toHaveBeenCalledWith('abc123')); + }); + + test('with nothing blocked it says so', () => { + setup([]); + expect(screen.getByText(/no releases are blocked/i)).toBeTruthy(); + }); +}); diff --git a/web/src/lib/components/wanted/DownloadList.svelte b/web/src/lib/components/wanted/DownloadList.svelte new file mode 100644 index 00000000..183babbb --- /dev/null +++ b/web/src/lib/components/wanted/DownloadList.svelte @@ -0,0 +1,192 @@ + + +{#if downloads.isPending} +

Reading the downloads…

+{:else if downloads.isError && !downloads.isFetchNextPageError} +

Couldn't load the downloads.

+{:else if items.length === 0} +
+ +

Nothing has been downloaded yet.

+

+ A release the search takes, or one you take from a want's Releases, shows here while it + downloads and once it's imported. +

+
+{:else} +
    + {#each items as d (d.id)} +
  • +
    +
    +
    + {d.want_artist ? `${d.want_artist} – ${d.want_title}` : d.name} +
    +
    + {d.name}{#if d.quality} · {d.quality}{/if} +
    +
    + + {label(d)} + +
    + + {#if !d.ended_at && (d.import_state === 'downloading' || d.import_state === '')} +
    +
    +
    + {/if} + + {#if d.import_reason} +

    + {d.import_reason} +

    + {/if} + {#if d.evidence} +

    {counted(d.evidence)}

    + {/if} + {#if filed(d)} +

    {filed(d)}

    + {/if} + {#if d.rehomed_from} +

    Taken for {d.rehomed_from}; it turned out to be this album.

    + {/if} +

    {when(d)}

    + + {#if needsYou(d)} +
    + {#if d.import_state === 'review'} + + {:else} + + {/if} + +
    + {/if} +
    • + {/each} +
+ + downloads.fetchNextPage()} + /> +{/if} diff --git a/web/src/lib/components/wanted/DownloadList.test.ts b/web/src/lib/components/wanted/DownloadList.test.ts new file mode 100644 index 00000000..812353c2 --- /dev/null +++ b/web/src/lib/components/wanted/DownloadList.test.ts @@ -0,0 +1,109 @@ +import { afterEach, describe, expect, test, vi } from 'vitest'; +import { fireEvent, render, screen, waitFor } from '@testing-library/svelte'; +import { mockInfiniteQuery } from '../../../test-utils/query'; +import type { Download } from '#lib/api/types.js'; + +vi.mock('#lib/api/wanted.js', () => ({ + createDownloadsQuery: vi.fn(), + importAnyway: vi.fn().mockResolvedValue(undefined), + retryImport: vi.fn().mockResolvedValue(undefined), + refuseDownload: vi.fn().mockResolvedValue(undefined) +})); +vi.mock('#lib/stores/toast.svelte.js', () => ({ pushToast: vi.fn() })); + +import DownloadList from './DownloadList.svelte'; +import { createDownloadsQuery, importAnyway, refuseDownload, retryImport } from '#lib/api/wanted.js'; + +const ago = (ms: number) => new Date(Date.now() - ms).toISOString(); + +function download(id: string, extra: Partial = {}): Download { + return { + id, + want_id: 'w1', + want_title: 'Random Access Memories', + want_artist: 'Daft Punk', + name: 'Daft Punk - Random Access Memories (2013) [FLAC]', + indexer: 'Prowlarr', + client: 'qbittorrent', + quality: 'lossless', + origin: 'schedule', + grabbed_at: ago(3_600_000), + ended_at: null, + end_reason: '', + progress: 0, + import_state: 'downloading', + import_reason: '', + imported_at: null, + imported_files: 0, + destination: '', + files: [], + skipped: 0, + ...extra + }; +} + +afterEach(() => vi.clearAllMocks()); + +function setup(items: Download[]) { + vi.mocked(createDownloadsQuery).mockReturnValue( + mockInfiniteQuery({ pages: [{ items, total: items.length, limit: 50, offset: 0 }] }) as unknown as ReturnType< + typeof createDownloadsQuery + > + ); + return render(DownloadList); +} + +describe('DownloadList', () => { + test('a running download shows how far it has got', () => { + setup([download('d1', { progress: 0.375 })]); + expect(screen.getByTestId('download-state').textContent).toMatch(/downloading/i); + expect(screen.getByRole('progressbar', { name: /downloaded/i }).getAttribute('aria-valuenow')).toBe('37'); + expect(screen.queryByRole('button')).toBeNull(); + }); + + test('a doubtful download says why and can be imported anyway or refused', async () => { + setup([ + download('d1', { + import_state: 'review', + import_reason: 'Only the lengths agree. A person decides.', + evidence: { files: 13, tracks: 13, identified: 0, recordings: 0, lengths: 13, titles: 0, fingerprinted: false } + }) + ]); + expect(screen.getByTestId('download-state').textContent).toMatch(/needs you/i); + expect(screen.getByTestId('download-reason').textContent).toMatch(/a person decides/i); + expect(screen.getByText(/13 files against 13 tracks · no AcoustID · 13 lengths agree/)).toBeTruthy(); + await fireEvent.click(screen.getByRole('button', { name: /import anyway/i })); + await waitFor(() => expect(importAnyway).toHaveBeenCalledWith('d1')); + await fireEvent.click(screen.getByRole('button', { name: /refuse/i })); + await waitFor(() => expect(refuseDownload).toHaveBeenCalledWith('d1')); + }); + + test('a failed import can be tried again', async () => { + setup([download('d1', { import_state: 'failed', import_reason: 'The library folder is not writable.' })]); + expect(screen.queryByRole('button', { name: /import anyway/i })).toBeNull(); + await fireEvent.click(screen.getByRole('button', { name: /try again/i })); + await waitFor(() => expect(retryImport).toHaveBeenCalledWith('d1')); + }); + + test('an imported download says what was filed where', () => { + setup([ + download('d1', { + ended_at: ago(60_000), + end_reason: 'imported', + import_state: 'imported', + imported_at: ago(60_000), + imported_files: 12, + skipped: 1, + destination: '/music/Daft Punk/Random Access Memories (2013)' + }) + ]); + expect(screen.getByTestId('download-state').textContent).toMatch(/imported/i); + expect(screen.getByText(/12 tracks filed, 1 already held into \/music\/Daft Punk/)).toBeTruthy(); + expect(screen.queryByRole('button')).toBeNull(); + }); + + test('with nothing downloaded it says what lands here', () => { + setup([]); + expect(screen.getByText(/nothing has been downloaded yet/i)).toBeTruthy(); + }); +}); diff --git a/web/src/lib/components/wanted/FollowList.svelte b/web/src/lib/components/wanted/FollowList.svelte new file mode 100644 index 00000000..bd1e3fbd --- /dev/null +++ b/web/src/lib/components/wanted/FollowList.svelte @@ -0,0 +1,117 @@ + + +{#if follows.isPending} +

Reading who's followed…

+{:else if follows.isError} +

Couldn't load the followed artists.

+{:else if (follows.data ?? []).length === 0} +

No artists are followed. Follow one from its artist page.

+{:else} +
    + {#each follows.data ?? [] as f (f.artist_mbid)} +
  • + +
    +
    + {#if f.artist_id} + {f.artist_name} + {:else} + {f.artist_name} + {/if} +
    +
    + {f.open_wants === 1 ? '1 album wanted' : `${f.open_wants} albums wanted`} · + {f.checked_at ? `checked ${relativeTime(f.checked_at)}` : 'not checked yet'} +
    +
    + + +
    • + {/each} +
+{/if} diff --git a/web/src/lib/components/wanted/FollowList.test.ts b/web/src/lib/components/wanted/FollowList.test.ts new file mode 100644 index 00000000..7ae23c05 --- /dev/null +++ b/web/src/lib/components/wanted/FollowList.test.ts @@ -0,0 +1,73 @@ +import { afterEach, describe, expect, test, vi } from 'vitest'; +import { fireEvent, render, screen, waitFor } from '@testing-library/svelte'; +import { mockQuery } from '../../../test-utils/query'; +import type { Follow, ReleaseProfilesResponse } from '#lib/api/types.js'; + +vi.mock('#lib/api/wanted.js', () => ({ + createFollowsQuery: vi.fn(), + createReleaseProfilesQuery: vi.fn(), + putFollow: vi.fn().mockResolvedValue({ following: true }), + deleteFollow: vi.fn().mockResolvedValue(undefined) +})); +vi.mock('#lib/stores/toast.svelte.js', () => ({ pushToast: vi.fn() })); + +import FollowList from './FollowList.svelte'; +import { createFollowsQuery, createReleaseProfilesQuery, deleteFollow, putFollow } from '#lib/api/wanted.js'; + +const ago = (ms: number) => new Date(Date.now() - ms).toISOString(); + +const follow: Follow = { + artist_mbid: 'blur', + artist_name: 'Blur', + artist_id: 'art-1', + profile_id: 'p-std', + profile_name: 'Standard', + followed_at: ago(10 * 86_400_000), + checked_at: ago(3 * 3_600_000), + open_wants: 2, + image_url: '' +}; + +const profiles: ReleaseProfilesResponse = { + profiles: [ + { id: 'p-std', name: 'Standard', primary_types: ['Album'], secondary_types: [], is_default: true, follow_count: 1 }, + { id: 'p-live', name: 'With live', primary_types: ['Album'], secondary_types: ['Live'], is_default: false, follow_count: 0 } + ], + primary_types: ['Album'], + secondary_types: ['Live'] +}; + +afterEach(() => vi.clearAllMocks()); + +function setup(follows: Follow[] = [follow]) { + vi.mocked(createFollowsQuery).mockReturnValue( + mockQuery({ data: follows }) as unknown as ReturnType + ); + vi.mocked(createReleaseProfilesQuery).mockReturnValue( + mockQuery({ data: profiles }) as unknown as ReturnType + ); + return render(FollowList); +} + +describe('FollowList', () => { + test('lists artists; the profile can change and the artist be unfollowed', async () => { + setup(); + expect(screen.getAllByTestId('follow-row')).toHaveLength(1); + expect(screen.getByText(/2 albums wanted/)).toBeTruthy(); + + const select = screen.getByLabelText(/profile for blur/i) as HTMLSelectElement; + expect(select.value).toBe('p-std'); + await fireEvent.change(select, { target: { value: 'p-live' } }); + await waitFor(() => + expect(putFollow).toHaveBeenCalledWith('blur', { artist_name: 'Blur', profile_id: 'p-live' }) + ); + + await fireEvent.click(screen.getByRole('button', { name: /unfollow blur/i })); + await waitFor(() => expect(deleteFollow).toHaveBeenCalledWith('blur')); + }); + + test('with no follows it says where to follow from', () => { + setup([]); + expect(screen.getByText(/follow one from its artist page/i)).toBeTruthy(); + }); +}); diff --git a/web/src/lib/components/wanted/WantList.svelte b/web/src/lib/components/wanted/WantList.svelte new file mode 100644 index 00000000..d13e390f --- /dev/null +++ b/web/src/lib/components/wanted/WantList.svelte @@ -0,0 +1,263 @@ + + +{#if wants.isPending} +

Reading the wanted list…

+{:else if wants.isError && !wants.isFetchNextPageError} +

Couldn't load the wanted list.

+{:else if items.length === 0} +
+ +

+ {#if state === 'held'}Nothing on the list has been held yet. + {:else if state === 'missing'}No missing albums are wanted back. + {:else}Nothing is wanted.{/if} +

+

+ {#if state === 'missing'} + An album whose files stay missing past the grace window on the Missing files page lands here. + {:else} + A request you approve, a new release from a followed artist, or an album whose files went + missing lands here. + {/if} +

+
+{:else} +
    + {#each items as w (w.id)} +
  • +
    + +
    +
    + {#if w.album_id && w.held_at} + {w.title} + {:else} + {w.title} + {/if} +
    +
    + {#if w.artist_id} + {w.artist_name} + {:else} + {w.artist_name} + {/if} + {#if kind(w)} · {kind(w)}{/if} +
    +
    + {why(w)} · {w.held_at ? `held ${relativeTime(w.held_at)}` : `wanted ${relativeTime(w.wanted_at)}`} +
    + {#if !w.held_at} +
    + {status(w)} + {#if attention(w)} + Decide on Activity + {/if} +
    + {:else if arrived(w)} +
    {arrived(w)}
    + {/if} +
    + {#if !w.held_at} +
    + + + +
    + {/if} +
    + {#if open === w.id && !w.held_at} +
    + +
    + {/if} +
    • + {/each} +
+ + wants.fetchNextPage()} + /> +{/if} diff --git a/web/src/routes/admin/wanted/wanted.test.ts b/web/src/lib/components/wanted/WantList.test.ts similarity index 58% rename from web/src/routes/admin/wanted/wanted.test.ts rename to web/src/lib/components/wanted/WantList.test.ts index 410505ae..47e6ae09 100644 --- a/web/src/routes/admin/wanted/wanted.test.ts +++ b/web/src/lib/components/wanted/WantList.test.ts @@ -1,15 +1,11 @@ import { afterEach, describe, expect, test, vi } from 'vitest'; import { fireEvent, render, screen, waitFor } from '@testing-library/svelte'; import { mockInfiniteQuery, mockQuery } from '../../../test-utils/query'; -import type { Follow, ReleaseProfilesResponse, Want, WantsPage } from '#lib/api/types.js'; +import type { Want, WantGrab, WantsPage } from '#lib/api/types.js'; vi.mock('#lib/api/wanted.js', () => ({ createWantsQuery: vi.fn(), - createFollowsQuery: vi.fn(), - createReleaseProfilesQuery: vi.fn(), removeWant: vi.fn().mockResolvedValue(undefined), - putFollow: vi.fn().mockResolvedValue({ following: true }), - deleteFollow: vi.fn().mockResolvedValue(undefined), searchWant: vi.fn().mockResolvedValue({ outcome: 'nothing', considered: 0, best_release: '', best_verdict: '', narrowed: [] }), @@ -20,18 +16,10 @@ vi.mock('#lib/api/wanted.js', () => ({ })); vi.mock('#lib/stores/toast.svelte.js', () => ({ pushToast: vi.fn() })); -import AdminWantedPage from './+page.svelte'; +import WantList from './WantList.svelte'; import { pushToast } from '#lib/stores/toast.svelte.js'; -import { - createCandidatesQuery, - createFollowsQuery, - createReleaseProfilesQuery, - createWantsQuery, - deleteFollow, - putFollow, - removeWant, - searchWant -} from '#lib/api/wanted.js'; +import { createCandidatesQuery, createWantsQuery, removeWant, searchWant } from '#lib/api/wanted.js'; +import type { WantState } from '#lib/api/types.js'; const DAY = 24 * 3_600_000; const ago = (ms: number) => new Date(Date.now() - ms).toISOString(); @@ -61,54 +49,48 @@ function page(items: Want[]): WantsPage { total: items.length, limit: 50, offset: 0, - counts: { wanted: 2, missing: 1, held: 4, following: 1 } + counts: { wanted: 2, missing: 1, held: 4, following: 1, activity: 1, attention: 0, blocked: 0 } }; } -const follow: Follow = { - artist_mbid: 'blur', - artist_name: 'Blur', - artist_id: 'art-1', - profile_id: 'p-std', - profile_name: 'Standard', - followed_at: ago(10 * DAY), - checked_at: ago(3 * 3_600_000), - open_wants: 2, - image_url: '' -}; - -const profiles: ReleaseProfilesResponse = { - profiles: [ - { id: 'p-std', name: 'Standard', primary_types: ['Album'], secondary_types: [], is_default: true, follow_count: 1 }, - { id: 'p-live', name: 'With live', primary_types: ['Album'], secondary_types: ['Live'], is_default: false, follow_count: 0 } - ], - primary_types: ['Album'], - secondary_types: ['Live'] -}; +function grab(extra: Partial = {}): WantGrab { + return { + id: 'g1', + name: 'Blur - Blur (1997) [FLAC]', + grabbed_at: ago(60_000), + origin: 'schedule', + quality: 'lossless', + live: true, + end_reason: '', + progress: 0, + import_state: '', + import_reason: '', + imported_at: null, + imported_files: 0, + ...extra + }; +} afterEach(() => vi.clearAllMocks()); -function setup(items: Want[] = [want('w1'), want('w2', { reason: 'request', requested_by: 'sam', title: 'Parklife' })]) { +function setup( + items: Want[] = [want('w1'), want('w2', { reason: 'request', requested_by: 'sam', title: 'Parklife' })], + state: WantState = 'wanted' +) { vi.mocked(createWantsQuery).mockReturnValue( mockInfiniteQuery({ pages: [page(items)] }) as unknown as ReturnType ); - vi.mocked(createFollowsQuery).mockReturnValue( - mockQuery({ data: [follow] }) as unknown as ReturnType - ); - vi.mocked(createReleaseProfilesQuery).mockReturnValue( - mockQuery({ data: profiles }) as unknown as ReturnType - ); vi.mocked(createCandidatesQuery).mockReturnValue( mockQuery({ data: [] }) as unknown as ReturnType ); - return render(AdminWantedPage); + return render(WantList, { state }); } -describe('/admin/wanted', () => { - test('each tab shows its count', () => { - setup(); - expect(screen.getByRole('tab', { name: /wanted/i }).textContent).toMatch(/2/); - expect(screen.getByRole('tab', { name: /held/i }).textContent).toMatch(/4/); +describe('WantList', () => { + test('asks for its own state', () => { + setup([], 'missing'); + expect(createWantsQuery).toHaveBeenCalledWith('missing'); + expect(screen.getByText(/no missing albums are wanted back/i)).toBeTruthy(); }); test('a want says why it is wanted', () => { @@ -118,12 +100,6 @@ describe('/admin/wanted', () => { expect(why[1]).toMatch(/requested by sam/i); }); - test('switching tab asks for that state', async () => { - setup(); - await fireEvent.click(screen.getByRole('tab', { name: /missing files/i })); - expect(createWantsQuery).toHaveBeenCalledWith('missing'); - }); - test('remove takes a want off the list', async () => { setup(); await fireEvent.click(screen.getByRole('button', { name: /stop wanting parklife/i })); @@ -136,22 +112,6 @@ describe('/admin/wanted', () => { expect(screen.getByRole('link', { name: /the ballad of darren/i }).getAttribute('href')).toBe('/albums/al-1'); }); - test('following lists artists; the profile can change and the artist be unfollowed', async () => { - setup(); - await fireEvent.click(screen.getByRole('tab', { name: /following/i })); - expect(screen.getAllByTestId('follow-row')).toHaveLength(1); - - const select = screen.getByLabelText(/profile for blur/i) as HTMLSelectElement; - expect(select.value).toBe('p-std'); - await fireEvent.change(select, { target: { value: 'p-live' } }); - await waitFor(() => - expect(putFollow).toHaveBeenCalledWith('blur', { artist_name: 'Blur', profile_id: 'p-live' }) - ); - - await fireEvent.click(screen.getByRole('button', { name: /unfollow blur/i })); - await waitFor(() => expect(deleteFollow).toHaveBeenCalledWith('blur')); - }); - test('an empty list explains what lands there', () => { setup([]); expect(screen.getByText(/nothing is wanted/i)).toBeTruthy(); @@ -164,15 +124,49 @@ describe('/admin/wanted', () => { title: 'Parklife', search: { searched_at: ago(3_600_000), outcome: 'nothing', considered: 4, best_release: 'x', best_verdict: 'rejected', error: '' } }), - want('w3', { - title: 'Blur', - grab: { name: 'Blur - Blur (1997) [FLAC]', grabbed_at: ago(60_000), origin: 'schedule', quality: 'lossless' } - }) + want('w3', { title: 'Blur', grab: grab() }), + want('w4', { title: '13', grab: grab({ import_state: 'downloading', progress: 0.42 }) }) ]); const lines = screen.getAllByTestId('want-search').map((e) => e.textContent ?? ''); expect(lines[0]).toMatch(/not searched yet/i); expect(lines[1]).toMatch(/4 found, none of them this album/i); expect(lines[2]).toMatch(/downloading blur - blur \(1997\) \[flac\], taken by the schedule/i); + expect(lines[3]).toMatch(/downloading 42%/i); + }); + + test('a download waiting on a person says why and points to Activity', () => { + setup([ + want('w1', { grab: grab({ import_state: 'review', import_reason: 'Only the lengths agree. A person decides.' }) }) + ]); + const line = screen.getByTestId('want-search'); + expect(line.textContent).toMatch(/needs you: only the lengths agree/i); + expect(screen.getByRole('link', { name: /decide on activity/i }).getAttribute('href')).toBe('/wanted/activity'); + // A live download can't be searched for again. + expect((screen.getByRole('button', { name: /search for the ballad of darren now/i }) as HTMLButtonElement).disabled).toBe(true); + }); + + test('a refused download is named beside the search that follows it', () => { + setup([ + want('w1', { + search: { searched_at: ago(3_600_000), outcome: 'nothing', considered: 0, best_release: '', best_verdict: '', error: '' }, + grab: grab({ live: false, end_reason: 'refused', import_state: 'refused', import_reason: 'Another album.' }) + }) + ]); + expect(screen.getByTestId('want-search').textContent).toMatch(/last download refused: another album/i); + }); + + test('a held album says how it arrived', () => { + setup( + [ + want('w1', { + held_at: ago(DAY), + album_id: 'al-1', + grab: grab({ live: false, end_reason: 'imported', import_state: 'imported', imported_at: ago(DAY), imported_files: 11 }) + }) + ], + 'held' + ); + expect(screen.getByTestId('want-arrived').textContent).toMatch(/imported 11 tracks .* from blur - blur/i); }); test('Search runs one now and says what came of it', async () => { diff --git a/web/src/lib/styles/error-copy.json b/web/src/lib/styles/error-copy.json index 90abec17..e9a072c6 100644 --- a/web/src/lib/styles/error-copy.json +++ b/web/src/lib/styles/error-copy.json @@ -26,6 +26,9 @@ "bad_body": "Couldn't read the request.", "bad_kind": "Invalid request type.", "bad_paging": "Invalid page size or offset.", + "invalid_paging": "Invalid page size or offset.", + "not_decidable": "That download isn't waiting on a decision any more. Refresh the list.", + "bad_import_root": "New music has to be filed into one of the library's folders.", "bad_reason": "Invalid quarantine reason.", "mbid_required": "An MBID is required for this lookup.", "system_playlist_readonly": "System playlists can't be edited directly.", diff --git a/web/src/routes/admin/integrations/+page.svelte b/web/src/routes/admin/integrations/+page.svelte index 94542a0d..3095e4cf 100644 --- a/web/src/routes/admin/integrations/+page.svelte +++ b/web/src/routes/admin/integrations/+page.svelte @@ -29,7 +29,6 @@ import Modal from '#lib/components/Modal.svelte'; import AcoustIDCard from '#lib/components/AcoustIDCard.svelte'; import AcquisitionCard from '#lib/components/AcquisitionCard.svelte'; - import ReleaseProfilesCard from '#lib/components/ReleaseProfilesCard.svelte'; import SearchSettingsCard from '#lib/components/SearchSettingsCard.svelte'; import NetworkSettingsCard from '#lib/components/NetworkSettingsCard.svelte'; import PublicAddressCard from '#lib/components/PublicAddressCard.svelte'; @@ -427,8 +426,6 @@ - -
diff --git a/web/src/routes/admin/integrations/integrations.test.ts b/web/src/routes/admin/integrations/integrations.test.ts index 26b475d7..5bd63952 100644 --- a/web/src/routes/admin/integrations/integrations.test.ts +++ b/web/src/routes/admin/integrations/integrations.test.ts @@ -43,25 +43,10 @@ vi.mock('#lib/api/admin.js', () => ({ addAcquisitionService: vi.fn() })); -// ReleaseProfilesCard reads its own query; settled, so it adds no "Loading…". +// SearchSettingsCard reads its own query; settled, so it adds no "Loading…". vi.mock('#lib/api/wanted.js', async () => { const { mockQuery } = await import('../../../test-utils/query'); return { - createReleaseProfilesQuery: vi.fn(() => - mockQuery({ - data: { - profiles: [ - { id: 'p-std', name: 'Standard', primary_types: ['Album', 'EP', 'Single'], secondary_types: [], is_default: true, follow_count: 3 } - ], - primary_types: ['Album', 'EP', 'Single'], - secondary_types: ['Live'] - } - }) - ), - createReleaseProfile: vi.fn(), - updateReleaseProfile: vi.fn(), - deleteReleaseProfile: vi.fn(), - // SearchSettingsCard, settled the same way. createSearchSettingsQuery: vi.fn(() => mockQuery({ data: { @@ -71,7 +56,9 @@ vi.mock('#lib/api/wanted.js', async () => { allowed_qualities: ['lossy_high'], tiers: [{ tier: 'lossy_high', label: 'High quality lossy', what: 'MP3 320' }], last_pass_at: null, - last_pass: {} + last_pass: {}, + import_root: '', + scan_paths: ['/music'] } }) ), diff --git a/web/src/routes/admin/missing-files/+page.svelte b/web/src/routes/admin/missing-files/+page.svelte index f64636f5..fb89e25d 100644 --- a/web/src/routes/admin/missing-files/+page.svelte +++ b/web/src/routes/admin/missing-files/+page.svelte @@ -107,7 +107,7 @@ {#if r.held_at} Held again {relativeTime(r.held_at)}. The rows here clear on the next scan. {:else if r.wanted_at} - On the wanted list since {relativeTime(r.wanted_at)}. {/if} diff --git a/web/src/routes/admin/missing-files/missing-files.test.ts b/web/src/routes/admin/missing-files/missing-files.test.ts index e8adf9b7..cb804653 100644 --- a/web/src/routes/admin/missing-files/missing-files.test.ts +++ b/web/src/routes/admin/missing-files/missing-files.test.ts @@ -140,7 +140,7 @@ describe('admin missing files', () => { expect( within(screen.getByTestId('reacquisition-state')).getByRole('link', { name: /wanted list/i }).getAttribute('href') ).toBe( - '/admin/wanted?tab=missing' + '/wanted/missing' ); }); diff --git a/web/src/routes/admin/requests/+page.svelte b/web/src/routes/admin/requests/+page.svelte index 06395cd9..b2952c24 100644 --- a/web/src/routes/admin/requests/+page.svelte +++ b/web/src/routes/admin/requests/+page.svelte @@ -146,7 +146,7 @@

Requests

Approve or set aside what users have asked for. An approved album or track goes on the - wanted list; an approved + wanted list; an approved artist is followed, wanting what they've released and what they release next.

diff --git a/web/src/routes/admin/wanted/+page.svelte b/web/src/routes/admin/wanted/+page.svelte deleted file mode 100644 index d791c2d5..00000000 --- a/web/src/routes/admin/wanted/+page.svelte +++ /dev/null @@ -1,363 +0,0 @@ - - -{pageTitle('Admin · Wanted')} - -
-
-

Wanted

-

- Albums Minstrel is to get: ones users requested, new releases from the artists it follows, - and albums whose files went missing. Minstrel searches your indexers for each one on a - schedule and hands the best release to your download client; searching and quality are set - under Integrations. - An album leaves the list once the library holds it. -

-
- - - - {#if tab === 'following'} -

- Each artist is followed on a release-type profile, which says which kinds of release are - wanted. Profiles are edited under - Integrations. -

- {#if follows.isPending} -

Reading who's followed…

- {:else if follows.isError} -

Couldn't load the followed artists.

- {:else if (follows.data ?? []).length === 0} -

- No artists are followed. Follow one from its artist page. -

- {:else} -
    - {#each follows.data ?? [] as f (f.artist_mbid)} -
  • - -
    -
    - {#if f.artist_id} - {f.artist_name} - {:else} - {f.artist_name} - {/if} -
    -
    - {f.open_wants === 1 ? '1 album wanted' : `${f.open_wants} albums wanted`} · - {f.checked_at ? `checked ${relativeTime(f.checked_at)}` : 'not checked yet'} -
    -
    - - -
    • - {/each} -
- {/if} - {:else if wants.isPending} -

Reading the wanted list…

- {:else if wants.isError && !wants.isFetchNextPageError} -

Couldn't load the wanted list.

- {:else if items.length === 0} -
- -

- {#if tab === 'held'}Nothing on the list has been held yet. - {:else if tab === 'missing'}No missing albums are wanted back. - {:else}Nothing is wanted.{/if} -

-

- {#if tab === 'missing'} - An album whose files stay missing past the grace window on the Missing files page lands here. - {:else} - A request you approve, a new release from a followed artist, or an album whose files went - missing lands here. - {/if} -

-
- {:else} -
    - {#each items as w (w.id)} -
  • -
    - -
    -
    - {#if w.album_id && w.held_at} - {w.title} - {:else} - {w.title} - {/if} -
    -
    - {#if w.artist_id} - {w.artist_name} - {:else} - {w.artist_name} - {/if} - {#if kind(w)} · {kind(w)}{/if} -
    -
    - {why(w)} · {w.held_at ? `held ${relativeTime(w.held_at)}` : `wanted ${relativeTime(w.wanted_at)}`} -
    - {#if !w.held_at} -
    {status(w)}
    - {/if} -
    - {#if !w.held_at} -
    - - - -
    - {/if} -
    - {#if open === w.id && !w.held_at} -
    - -
    - {/if} -
    • - {/each} -
- - wants.fetchNextPage()} - /> - {/if} -
diff --git a/web/src/routes/wanted/+layout.svelte b/web/src/routes/wanted/+layout.svelte new file mode 100644 index 00000000..0e2116bf --- /dev/null +++ b/web/src/routes/wanted/+layout.svelte @@ -0,0 +1,48 @@ + + +
+
+

Wanted

+

+ Albums Minstrel is getting: ones users requested, new releases from the artists it follows, + and albums whose files went missing. It searches your indexers on a schedule, checks each + download against MusicBrainz, and files what's right into the library. Searching and quality + are set under Integrations. +

+ {#if verdict} +

+ {verdict} + {#if counts?.attention} + See Activity + {/if} +

+ {/if} +
+ + + + {@render children?.()} +
diff --git a/web/src/routes/wanted/+layout.ts b/web/src/routes/wanted/+layout.ts new file mode 100644 index 00000000..f22d5109 --- /dev/null +++ b/web/src/routes/wanted/+layout.ts @@ -0,0 +1,13 @@ +import { redirect } from '@sveltejs/kit'; +import { user } from '#lib/auth/store.svelte.js'; +import type { LayoutLoad } from './$types'; + +// Hard route gate: the Wanted view is an admin's (M502). Runs before the +// layout renders; auth is bootstrapped by the root +layout.ts, so user.value is +// either a settled User object or null. +export const load: LayoutLoad = () => { + if (!user.value || !user.value.is_admin) { + throw redirect(302, '/'); + } + return {}; +}; diff --git a/web/src/routes/wanted/+page.svelte b/web/src/routes/wanted/+page.svelte new file mode 100644 index 00000000..96b2637e --- /dev/null +++ b/web/src/routes/wanted/+page.svelte @@ -0,0 +1,8 @@ + + +{pageTitle('Wanted')} + + diff --git a/web/src/routes/wanted/activity/+page.svelte b/web/src/routes/wanted/activity/+page.svelte new file mode 100644 index 00000000..c98588ee --- /dev/null +++ b/web/src/routes/wanted/activity/+page.svelte @@ -0,0 +1,8 @@ + + +{pageTitle('Wanted · Activity')} + + diff --git a/web/src/routes/wanted/blocked/+page.svelte b/web/src/routes/wanted/blocked/+page.svelte new file mode 100644 index 00000000..3c4ea0d0 --- /dev/null +++ b/web/src/routes/wanted/blocked/+page.svelte @@ -0,0 +1,8 @@ + + +{pageTitle('Wanted · Blocked')} + + diff --git a/web/src/routes/wanted/following/+page.svelte b/web/src/routes/wanted/following/+page.svelte new file mode 100644 index 00000000..7303fe95 --- /dev/null +++ b/web/src/routes/wanted/following/+page.svelte @@ -0,0 +1,12 @@ + + +{pageTitle('Wanted · Following')} + +
+ + +
diff --git a/web/src/routes/wanted/held/+page.svelte b/web/src/routes/wanted/held/+page.svelte new file mode 100644 index 00000000..dd44902b --- /dev/null +++ b/web/src/routes/wanted/held/+page.svelte @@ -0,0 +1,8 @@ + + +{pageTitle('Wanted · Held')} + + diff --git a/web/src/routes/wanted/missing/+page.svelte b/web/src/routes/wanted/missing/+page.svelte new file mode 100644 index 00000000..f2b7e6df --- /dev/null +++ b/web/src/routes/wanted/missing/+page.svelte @@ -0,0 +1,8 @@ + + +{pageTitle('Wanted · Missing files')} + + diff --git a/web/src/routes/wanted/wanted.test.ts b/web/src/routes/wanted/wanted.test.ts new file mode 100644 index 00000000..9787f91b --- /dev/null +++ b/web/src/routes/wanted/wanted.test.ts @@ -0,0 +1,61 @@ +import { afterEach, describe, expect, test, vi } from 'vitest'; +import { render, screen } from '@testing-library/svelte'; +import { createRawSnippet } from 'svelte'; +import { mockInfiniteQuery } from '../../test-utils/query'; +import { pageUrlModule } from '#test-utils/mocks/appState.js'; +import type { WantCounts, WantsPage } from '#lib/api/types.js'; + +const state = vi.hoisted(() => ({ pageUrl: new URL('http://localhost/wanted') })); +vi.mock('$app/state', () => pageUrlModule(state)); +vi.mock('#lib/api/wanted.js', () => ({ createWantsQuery: vi.fn() })); + +import WantedLayout from './+layout.svelte'; +import { createWantsQuery } from '#lib/api/wanted.js'; + +const counts: WantCounts = { wanted: 5, missing: 1, held: 9, following: 2, activity: 3, attention: 0, blocked: 4 }; +const children = createRawSnippet(() => ({ render: () => `

body

` })); + +afterEach(() => vi.clearAllMocks()); + +function setup(c: WantCounts = counts, path = '/wanted') { + state.pageUrl = new URL(`http://localhost${path}`); + const page: WantsPage = { items: [], total: 0, limit: 50, offset: 0, counts: c }; + vi.mocked(createWantsQuery).mockReturnValue( + mockInfiniteQuery({ pages: [page] }) as unknown as ReturnType + ); + // eslint-disable-next-line @typescript-eslint/no-explicit-any + return render(WantedLayout, { children } as any); +} + +describe('/wanted layout', () => { + test('tabs are links carrying their counts, the current one marked', () => { + setup(counts, '/wanted/activity'); + const nav = screen.getByRole('navigation', { name: /wanted sections/i }); + const links = Array.from(nav.querySelectorAll('a')).map((a) => [a.getAttribute('href'), a.textContent?.replace(/\s+/g, ' ').trim()]); + expect(links).toEqual([ + ['/wanted', 'Wanted 5'], + ['/wanted/activity', 'Activity 3'], + ['/wanted/missing', 'Missing files 1'], + ['/wanted/held', 'Held 9'], + ['/wanted/following', 'Following 2'], + ['/wanted/blocked', 'Blocked 4'] + ]); + expect(screen.getByRole('link', { name: /activity/i })).toHaveAttribute('aria-current', 'page'); + expect(screen.getByRole('link', { name: /^wanted/i })).not.toHaveAttribute('aria-current'); + expect(screen.getByTestId('tab-body')).toBeTruthy(); + }); + + test('when nothing needs a person the verdict says so', () => { + setup(); + expect(screen.getByTestId('wanted-verdict').textContent).toMatch(/5 albums wanted, 3 downloading\. Nothing needs you\./); + expect(screen.queryByTestId('tab-attention')).toBeNull(); + }); + + test('downloads waiting on a person lead the page and mark Activity', () => { + setup({ ...counts, attention: 2 }); + const verdict = screen.getByTestId('wanted-verdict'); + expect(verdict.textContent).toMatch(/2 downloads need you/); + expect(screen.getByRole('link', { name: /see activity/i }).getAttribute('href')).toBe('/wanted/activity'); + expect(screen.getByTestId('tab-attention').textContent).toBe('2'); + }); +});