feat: import with verification, and a Wanted view of its own (M502 #5522)

A finished download is checked against MusicBrainz before anything is
filed: the files are paired with an edition's tracks by position or path,
and AcoustID, tag ids, lengths and titles say whether it is the album.
What is the album is copied into the library as
Artist/Album (Year)/Artist - Album - NN - Title, tagged with Picard's
MusicBrainz ids, and scanned; the download keeps seeding. What is not is
refused: set aside, removed from the client, and the album searched for
again. A download the check cannot settle waits for a person, who imports
it anyway or refuses it. Tracks the library already holds are skipped.

The wanted list leaves Admin for a top-level Wanted view, admin-only and
web-only, with tabs for Wanted, Activity (downloads and their imports,
the ones needing a person first), Missing files, Held, Following (with the
release-type profiles, moved off Integrations) and Blocked. The search
settings gain "File new music into" when the library has several folders.

The search pass logs why each failed search failed.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

internal/api/admin_import.go

@@ -0,0 +1,205 @@
+package api
+
+import (
+	"encoding/json"
+	"errors"
+	"net/http"
+
+	"github.com/jackc/pgx/v5"
+
+	"git.fabledsword.com/bvandeusen/minstrel/internal/apierror"
+	"git.fabledsword.com/bvandeusen/minstrel/internal/db/dbq"
+	"git.fabledsword.com/bvandeusen/minstrel/internal/importer"
+)
+
+// Downloads and their imports (M502 #5522): the Wanted view's Activity tab,
+// a person's "import anyway", "refuse" and "retry", and the Blocked tab.
+
+// downloadView is one grab and what its import made of it.
+type downloadView struct {
+	ID            string             `json:"id"`
+	WantID        *string            `json:"want_id"`
+	WantTitle     string             `json:"want_title"`
+	WantArtist    string             `json:"want_artist"`
+	Name          string             `json:"name"`
+	Indexer       string             `json:"indexer"`
+	Client        string             `json:"client"`
+	Quality       string             `json:"quality"`
+	Origin        string             `json:"origin"`
+	GrabbedAt     string             `json:"grabbed_at"`
+	EndedAt       *string            `json:"ended_at"`
+	EndReason     string             `json:"end_reason"`
+	Progress      float32            `json:"progress"`
+	ImportState   string             `json:"import_state"`
+	ImportReason  string             `json:"import_reason"`
+	ImportedAt    *string            `json:"imported_at"`
+	ImportedFiles int32              `json:"imported_files"`
+	Destination   string             `json:"destination"`
+	Edition       string             `json:"edition,omitempty"`
+	Evidence      *importer.Evidence `json:"evidence,omitempty"`
+	Files         []string           `json:"files"`
+	Skipped       int                `json:"skipped"`
+	RehomedFrom   string             `json:"rehomed_from,omitempty"`
+}
+
+func downloadFrom(g dbq.ListGrabsRow) downloadView {
+	v := downloadView{
+		ID: uuidToString(g.ID), Name: g.Name, Indexer: g.Indexer, Client: g.Client, Quality: g.Quality,
+		Origin: g.Origin, GrabbedAt: formatTimestamp(g.GrabbedAt), EndReason: g.EndReason,
+		Progress: g.Progress, ImportState: g.ImportState, ImportReason: g.ImportReason,
+		ImportedFiles: g.ImportedFiles, Destination: g.Destination, Files: []string{},
+	}
+	if g.WantID.Valid {
+		id := uuidToString(g.WantID)
+		v.WantID = &id
+	}
+	if g.WantTitle != nil {
+		v.WantTitle = *g.WantTitle
+	}
+	if g.WantArtist != nil {
+		v.WantArtist = *g.WantArtist
+	}
+	if g.EndedAt.Valid {
+		at := formatTimestamp(g.EndedAt)
+		v.EndedAt = &at
+	}
+	if g.ImportedAt.Valid {
+		at := formatTimestamp(g.ImportedAt)
+		v.ImportedAt = &at
+	}
+	var det importer.Detail
+	if len(g.ImportDetail) > 0 && json.Unmarshal(g.ImportDetail, &det) == nil && det.Verdict.Outcome != "" {
+		ev := det.Verdict.Evidence
+		v.Evidence, v.Edition, v.Skipped, v.RehomedFrom = &ev, det.Edition, det.Skipped, det.RehomedFrom
+		if det.Files != nil {
+			v.Files = det.Files
+		}
+	}
+	return v
+}
+
+type downloadsPage struct {
+	Items  []downloadView `json:"items"`
+	Limit  int            `json:"limit"`
+	Offset int            `json:"offset"`
+}
+
+// handleListDownloads implements GET /api/admin/downloads?offset=: running
+// downloads first, then the rest newest first.
+func (h *handlers) handleListDownloads(w http.ResponseWriter, r *http.Request) {
+	limit, offset, err := parsePaging(r.URL.Query())
+	if err != nil {
+		writeErr(w, apierror.BadRequest("invalid_paging", "limit and offset must be numbers"))
+		return
+	}
+	rows, err := dbq.New(h.pool).ListGrabs(r.Context(), dbq.ListGrabsParams{MaxRows: int32(limit), Skip: int32(offset)})
+	if err != nil {
+		writeErrWithLog(w, h.logger, "admin downloads: list", apierror.Internal(err))
+		return
+	}
+	out := downloadsPage{Items: make([]downloadView, 0, len(rows)), Limit: limit, Offset: offset}
+	for _, g := range rows {
+		out.Items = append(out.Items, downloadFrom(g))
+	}
+	writeJSON(w, http.StatusOK, out)
+}
+
+// handleImportAnyway implements POST /api/admin/downloads/{id}/import: a
+// download the check held for a person (or one that failed) is imported on
+// the next pass whatever the check says.
+func (h *handlers) handleImportAnyway(w http.ResponseWriter, r *http.Request) {
+	id, ok := requireURLUUID(w, r, "id")
+	if !ok {
+		return
+	}
+	if _, err := dbq.New(h.pool).ForceGrabImport(r.Context(), id); err != nil {
+		h.writeImportErr(w, "import anyway", err)
+		return
+	}
+	w.WriteHeader(http.StatusNoContent)
+}
+
+// handleRetryImport implements POST /api/admin/downloads/{id}/retry.
+func (h *handlers) handleRetryImport(w http.ResponseWriter, r *http.Request) {
+	id, ok := requireURLUUID(w, r, "id")
+	if !ok {
+		return
+	}
+	if _, err := dbq.New(h.pool).RetryGrabImport(r.Context(), id); err != nil {
+		h.writeImportErr(w, "retry", err)
+		return
+	}
+	w.WriteHeader(http.StatusNoContent)
+}
+
+// handleRefuseDownload implements POST /api/admin/downloads/{id}/refuse: the
+// release is blocked, the download removed and the album searched again.
+func (h *handlers) handleRefuseDownload(w http.ResponseWriter, r *http.Request) {
+	admin, ok := requireUser(w, r)
+	if !ok {
+		return
+	}
+	id, ok := requireURLUUID(w, r, "id")
+	if !ok {
+		return
+	}
+	if _, err := h.importer.Refuse(r.Context(), id, admin.ID); err != nil {
+		h.writeImportErr(w, "refuse", err)
+		return
+	}
+	w.WriteHeader(http.StatusNoContent)
+}
+
+func (h *handlers) writeImportErr(w http.ResponseWriter, what string, err error) {
+	switch {
+	case errors.Is(err, pgx.ErrNoRows), errors.Is(err, importer.ErrNotDecidable):
+		writeErr(w, apierror.Conflict("not_decidable", "that download is not waiting on a decision"))
+	default:
+		writeErrWithLog(w, h.logger, "admin downloads: "+what, apierror.Internal(err))
+	}
+}
+
+type releaseBlockView struct {
+	InfoHash   string `json:"info_hash"`
+	Name       string `json:"name"`
+	Reason     string `json:"reason"`
+	Detail     string `json:"detail"`
+	BlockedAt  string `json:"blocked_at"`
+	WantTitle  string `json:"want_title"`
+	WantArtist string `json:"want_artist"`
+}
+
+type releaseBlocksPage struct {
+	Items  []releaseBlockView `json:"items"`
+	Limit  int                `json:"limit"`
+	Offset int                `json:"offset"`
+}
+
+// handleListReleaseBlocks implements GET /api/admin/release-blocks?offset=:
+// every release set aside, by a person or by a check, newest first.
+func (h *handlers) handleListReleaseBlocks(w http.ResponseWriter, r *http.Request) {
+	limit, offset, err := parsePaging(r.URL.Query())
+	if err != nil {
+		writeErr(w, apierror.BadRequest("invalid_paging", "limit and offset must be numbers"))
+		return
+	}
+	rows, err := dbq.New(h.pool).ListReleaseBlocksPage(r.Context(), dbq.ListReleaseBlocksPageParams{
+		MaxRows: int32(limit), Skip: int32(offset)})
+	if err != nil {
+		writeErrWithLog(w, h.logger, "admin release blocks: list", apierror.Internal(err))
+		return
+	}
+	out := releaseBlocksPage{Items: make([]releaseBlockView, 0, len(rows)), Limit: limit, Offset: offset}
+	for _, b := range rows {
+		v := releaseBlockView{InfoHash: b.InfoHash, Name: b.Name, Reason: b.Reason, Detail: b.Detail,
+			BlockedAt: formatTimestamp(b.BlockedAt)}
+		if b.WantTitle != nil {
+			v.WantTitle = *b.WantTitle
+		}
+		if b.WantArtist != nil {
+			v.WantArtist = *b.WantArtist
+		}
+		out.Items = append(out.Items, v)
+	}
+	writeJSON(w, http.StatusOK, out)
+}

internal/api/admin_search.go

@@ -5,6 +5,7 @@ import (
 	"encoding/json"
 	"errors"
 	"net/http"
+	"slices"
 	"time"
 
 	"github.com/go-chi/chi/v5"
@@ -32,12 +33,20 @@ type searchSettingsView struct {
 	Tiers               []release.TierInfo `json:"tiers"`
 	LastPassAt          *string            `json:"last_pass_at"`
 	LastPass            json.RawMessage    `json:"last_pass"`
+	// ImportRoot is the library folder new music is filed into (#5522); ""
+	// is the first of ScanPaths, the library's folders.
+	ImportRoot string   `json:"import_root"`
+	ScanPaths  []string `json:"scan_paths"`
 }
 
-func searchSettings(s dbq.AcquisitionSetting) searchSettingsView {
+func searchSettings(s dbq.AcquisitionSetting, roots []string) searchSettingsView {
 	v := searchSettingsView{
 		AutoGrab: s.AutoGrab, SearchIntervalHours: s.SearchIntervalHours,
 		AllowedQualities: s.AllowedQualities, Tiers: release.Tiers, LastPass: s.LastPass,
+		ImportRoot: s.ImportRoot, ScanPaths: roots,
+	}
+	if v.ScanPaths == nil {
+		v.ScanPaths = []string{}
 	}
 	if v.AllowedQualities == nil {
 		v.AllowedQualities = []string{}
@@ -59,13 +68,14 @@ func (h *handlers) handleGetSearchSettings(w http.ResponseWriter, r *http.Reques
 		writeErrWithLog(w, h.logger, "admin search settings: get", apierror.Internal(err))
 		return
 	}
-	writeJSON(w, http.StatusOK, searchSettings(s))
+	writeJSON(w, http.StatusOK, searchSettings(s, h.importer.Roots))
 }
 
 type putSearchSettingsBody struct {
 	AutoGrab            bool     `json:"auto_grab"`
 	SearchIntervalHours int32    `json:"search_interval_hours"`
 	AllowedQualities    []string `json:"allowed_qualities"`
+	ImportRoot          string   `json:"import_root"`
 }
 
 // handlePutSearchSettings implements PUT /api/admin/acquisition/settings.
@@ -88,14 +98,19 @@ func (h *handlers) handlePutSearchSettings(w http.ResponseWriter, r *http.Reques
 			return
 		}
 	}
+	if body.ImportRoot != "" && !slices.Contains(h.importer.Roots, body.ImportRoot) {
+		writeErr(w, apierror.BadRequest("bad_import_root", "file new music into one of the library's folders"))
+		return
+	}
 	s, err := dbq.New(h.pool).UpdateAcquisitionSettings(r.Context(), dbq.UpdateAcquisitionSettingsParams{
 		AutoGrab: body.AutoGrab, SearchIntervalHours: body.SearchIntervalHours, AllowedQualities: body.AllowedQualities,
+		ImportRoot: body.ImportRoot,
 	})
 	if err != nil {
 		writeErrWithLog(w, h.logger, "admin search settings: put", apierror.Internal(err))
 		return
 	}
-	writeJSON(w, http.StatusOK, searchSettings(s))
+	writeJSON(w, http.StatusOK, searchSettings(s, h.importer.Roots))
 }
 
 // candidateView is one release a want's last search found.

internal/api/admin_wanted.go

@@ -49,11 +49,21 @@ type wantSearchView struct {
 	Error       string `json:"error"`
 }
 
+// wantGrabView is the want's latest download, running or ended, and what
+// its import made of it (#5522).
 type wantGrabView struct {
-	Name      string `json:"name"`
-	GrabbedAt string `json:"grabbed_at"`
-	Origin    string `json:"origin"`
-	Quality   string `json:"quality"`
+	ID            string  `json:"id"`
+	Name          string  `json:"name"`
+	GrabbedAt     string  `json:"grabbed_at"`
+	Origin        string  `json:"origin"`
+	Quality       string  `json:"quality"`
+	Live          bool    `json:"live"`
+	EndReason     string  `json:"end_reason"`
+	Progress      float32 `json:"progress"`
+	ImportState   string  `json:"import_state"`
+	ImportReason  string  `json:"import_reason"`
+	ImportedAt    *string `json:"imported_at"`
+	ImportedFiles int32   `json:"imported_files"`
 }
 
 // wantCounts sizes each tab of the Wanted page.
@@ -62,6 +72,11 @@ type wantCounts struct {
 	Missing   int64 `json:"missing"`
 	Held      int64 `json:"held"`
 	Following int64 `json:"following"`
+	// Activity: downloads running; Attention: of those, the ones waiting on
+	// a person; Blocked: releases set aside (#5522).
+	Activity  int64 `json:"activity"`
+	Attention int64 `json:"attention"`
+	Blocked   int64 `json:"blocked"`
 }
 
 type wantsPage struct {
@@ -102,7 +117,8 @@ func (h *handlers) handleListWants(w http.ResponseWriter, r *http.Request) {
 	}
 	out := wantsPage{
 		Items: make([]wantView, 0, len(rows)), Limit: wantsPageSize, Offset: offset,
-		Counts: wantCounts{Wanted: counts.Wanted, Missing: counts.Missing, Held: counts.Held, Following: counts.Following},
+		Counts: wantCounts{Wanted: counts.Wanted, Missing: counts.Missing, Held: counts.Held, Following: counts.Following,
+			Activity: counts.Activity, Attention: counts.Attention, Blocked: counts.Blocked},
 	}
 	switch state {
 	case "missing":
@@ -145,12 +161,19 @@ func (h *handlers) handleListWants(w http.ResponseWriter, r *http.Request) {
 			}
 		}
 		if row.GrabName != nil {
-			v.Grab = &wantGrabView{Name: *row.GrabName, GrabbedAt: formatTimestamp(row.GrabbedAt)}
-			if row.GrabOrigin != nil {
-				v.Grab.Origin = *row.GrabOrigin
+			v.Grab = &wantGrabView{ID: uuidToString(row.GrabID), Name: *row.GrabName,
+				GrabbedAt: formatTimestamp(row.GrabbedAt), Live: !row.GrabEndedAt.Valid,
+				Origin: deref(row.GrabOrigin), Quality: deref(row.GrabQuality), EndReason: deref(row.GrabEndReason),
+				ImportState: deref(row.GrabImportState), ImportReason: deref(row.GrabImportReason)}
+			if row.GrabProgress != nil {
+				v.Grab.Progress = *row.GrabProgress
 			}
-			if row.GrabQuality != nil {
-				v.Grab.Quality = *row.GrabQuality
+			if row.GrabImportedFiles != nil {
+				v.Grab.ImportedFiles = *row.GrabImportedFiles
+			}
+			if row.GrabImportedAt.Valid {
+				at := formatTimestamp(row.GrabImportedAt)
+				v.Grab.ImportedAt = &at
 			}
 		}
 		out.Items = append(out.Items, v)
@@ -425,3 +448,11 @@ func (h *handlers) writeProfileErr(w http.ResponseWriter, err error) {
 		writeErrWithLog(w, h.logger, "admin release profiles", apierror.Internal(err))
 	}
 }
+
+// deref is a nullable column's text, or "".
+func deref(s *string) string {
+	if s == nil {
+		return ""
+	}
+	return *s
+}

internal/api/api.go

@@ -19,6 +19,7 @@ import (
 	"git.fabledsword.com/bvandeusen/minstrel/internal/config"
 	"git.fabledsword.com/bvandeusen/minstrel/internal/coverart"
 	"git.fabledsword.com/bvandeusen/minstrel/internal/eventbus"
+	"git.fabledsword.com/bvandeusen/minstrel/internal/importer"
 	"git.fabledsword.com/bvandeusen/minstrel/internal/library"
 	"git.fabledsword.com/bvandeusen/minstrel/internal/lidarrconfig"
 	"git.fabledsword.com/bvandeusen/minstrel/internal/lidarrquarantine"
@@ -78,6 +79,7 @@ func Mount(r chi.Router, pool *pgxpool.Pool, logger *slog.Logger, events *playev
 		catalogue:           cat,
 		wanted:              wanted.New(pool, cat, logger),
 		searcher:            wanted.NewSearcher(pool, cat, logger),
+		importer:            importer.New(pool, scanner, cat, nil, scannerRoots(scanner), logger),
 		catalogueSearches:   newSearchCache(),
 		catalogueArt:        catalogue.NewArt(dataDir, artSources(coverSettings)),
 		leveled:             newLeveledRenderer(dataDir, loudSettings, logger),
@@ -257,6 +259,11 @@ func Mount(r chi.Router, pool *pgxpool.Pool, logger *slog.Logger, events *playev
 				admin.Post("/wanted/{id}/grab", h.handleGrabCandidate)
 				admin.Post("/wanted/{id}/block", h.handleBlockCandidate)
 				admin.Delete("/release-blocks/{hash}", h.handleUnblockRelease)
+				admin.Get("/release-blocks", h.handleListReleaseBlocks)
+				admin.Get("/downloads", h.handleListDownloads)
+				admin.Post("/downloads/{id}/import", h.handleImportAnyway)
+				admin.Post("/downloads/{id}/retry", h.handleRetryImport)
+				admin.Post("/downloads/{id}/refuse", h.handleRefuseDownload)
 				admin.Get("/follows", h.handleListFollows)
 				admin.Get("/follows/{mbid}", h.handleGetFollow)
 				admin.Put("/follows/{mbid}", h.handlePutFollow)
@@ -427,6 +434,9 @@ type handlers struct {
 	wanted *wanted.Service
 	// searcher searches for wants and grabs releases for them (#5521).
 	searcher *wanted.Searcher
+	// importer refuses a download at a person's word (#5522). The import
+	// loop itself runs in main; this one only acts on a request.
+	importer *importer.Importer
 	// catalogueSearches reuses recent MusicBrainz answers; nil asks every
 	// time, which is what handler tests get.
 	catalogueSearches *searchCache
@@ -482,3 +492,12 @@ func artSources(s *coverart.SettingsService) catalogue.ArtSources {
 	}
 	return s
 }
+
+// scannerRoots are the library folders, or none when the API runs without a
+// scanner (tests).
+func scannerRoots(s *library.Scanner) []string {
+	if s == nil {
+		return nil
+	}
+	return s.Roots()
+}