feat: the library's folders are set in Admin › Library, and changing a folder moves its tracks with it, keeping their plays, likes and playlists (#6129)
release / govulncheck (push) Successful in 15s
release / web (push) Successful in 1m20s
release / go (push) Failing after 1m37s
release / integration (push) Failing after 4m56s
release / android (push) Successful in 5m1s
release / Build signed APK (releases and dev) (push) Successful in 5m13s
release / Attach APK to the Release (tag releases only) (push) Skipped
release / Build + push container image (push) Skipped
release / Verify release artifacts (tag releases only) (push) Skipped
release / govulncheck (push) Successful in 15s
release / web (push) Successful in 1m20s
release / go (push) Failing after 1m37s
release / integration (push) Failing after 4m56s
release / android (push) Successful in 5m1s
release / Build signed APK (releases and dev) (push) Successful in 5m13s
release / Attach APK to the Release (tag releases only) (push) Skipped
release / Build + push container image (push) Skipped
release / Verify release artifacts (tag releases only) (push) Skipped
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,111 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"net/http"
|
||||
"os"
|
||||
|
||||
"git.fabledsword.com/bvandeusen/minstrel/internal/apierror"
|
||||
"git.fabledsword.com/bvandeusen/minstrel/internal/db/dbq"
|
||||
"git.fabledsword.com/bvandeusen/minstrel/internal/library"
|
||||
)
|
||||
|
||||
// The library's folders (#6129): where the scan, the watcher and the import
|
||||
// look for music, set here rather than only by MINSTREL_LIBRARY_SCAN_PATHS.
|
||||
|
||||
type libraryFolderView struct {
|
||||
Path string `json:"path"`
|
||||
// Present is whether the server can see the folder right now; a folder
|
||||
// whose mount went away reads false here before anything else notices.
|
||||
Present bool `json:"present"`
|
||||
Tracks int64 `json:"tracks"`
|
||||
}
|
||||
|
||||
type libraryFoldersView struct {
|
||||
Folders []libraryFolderView `json:"folders"`
|
||||
// Saved is false while the folders still come from
|
||||
// MINSTREL_LIBRARY_SCAN_PATHS.
|
||||
Saved bool `json:"saved"`
|
||||
// Moved is how many tracks a save moved to a new folder.
|
||||
Moved int64 `json:"moved,omitempty"`
|
||||
}
|
||||
|
||||
func (h *handlers) libraryFolders(ctx context.Context) (libraryFoldersView, error) {
|
||||
q := dbq.New(h.pool)
|
||||
row, err := q.GetLibrarySettings(ctx)
|
||||
if err != nil {
|
||||
return libraryFoldersView{}, err
|
||||
}
|
||||
v := libraryFoldersView{Folders: []libraryFolderView{}, Saved: len(row.ScanPaths) > 0}
|
||||
for _, root := range h.scanner.Roots() {
|
||||
n, err := q.CountTracksUnder(ctx, root)
|
||||
if err != nil {
|
||||
return libraryFoldersView{}, err
|
||||
}
|
||||
info, statErr := os.Stat(root)
|
||||
v.Folders = append(v.Folders, libraryFolderView{Path: root, Present: statErr == nil && info.IsDir(), Tracks: n})
|
||||
}
|
||||
return v, nil
|
||||
}
|
||||
|
||||
// handleGetLibraryFolders implements GET /api/admin/library/folders.
|
||||
func (h *handlers) handleGetLibraryFolders(w http.ResponseWriter, r *http.Request) {
|
||||
v, err := h.libraryFolders(r.Context())
|
||||
if err != nil {
|
||||
writeErrWithLog(w, h.logger, "admin library folders: get", apierror.Internal(err))
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, v)
|
||||
}
|
||||
|
||||
type putLibraryFoldersBody struct {
|
||||
Folders []struct {
|
||||
Path string `json:"path"`
|
||||
// Was is the folder this one replaces, whose files are now at Path;
|
||||
// "" is a folder added.
|
||||
Was string `json:"was"`
|
||||
} `json:"folders"`
|
||||
}
|
||||
|
||||
// handlePutLibraryFolders implements PUT /api/admin/library/folders.
|
||||
//
|
||||
// The whole list. A folder whose path changed moves its tracks with it, so
|
||||
// their plays, likes and playlists survive; one left out is no longer read,
|
||||
// and its tracks are marked missing by the next scan. After a save a scan
|
||||
// starts, to pick up anything the old folders did not hold.
|
||||
func (h *handlers) handlePutLibraryFolders(w http.ResponseWriter, r *http.Request) {
|
||||
var body putLibraryFoldersBody
|
||||
if !decodeBody(w, r, &body) {
|
||||
return
|
||||
}
|
||||
if h.scanner == nil {
|
||||
writeErr(w, apierror.Internal(errors.New("no scanner")))
|
||||
return
|
||||
}
|
||||
changes := make([]library.RootChange, 0, len(body.Folders))
|
||||
for _, f := range body.Folders {
|
||||
changes = append(changes, library.RootChange{Path: f.Path, Was: f.Was})
|
||||
}
|
||||
res, err := h.scanner.SetRoots(r.Context(), changes)
|
||||
if err != nil {
|
||||
var re *library.RootsError
|
||||
if errors.As(err, &re) {
|
||||
writeErr(w, apierror.BadRequest("bad_folders", re.Error()))
|
||||
return
|
||||
}
|
||||
writeErrWithLog(w, h.logger, "admin library folders: put", apierror.Internal(err))
|
||||
return
|
||||
}
|
||||
if _, _, err := library.TryStartScan(context.Background(), h.pool, h.scanner, h.coverart,
|
||||
h.logger.With("source", "library_folders"), h.scanCfg); err != nil {
|
||||
h.logger.Warn("admin library folders: scan after save", "err", err)
|
||||
}
|
||||
v, err := h.libraryFolders(r.Context())
|
||||
if err != nil {
|
||||
writeErrWithLog(w, h.logger, "admin library folders: get", apierror.Internal(err))
|
||||
return
|
||||
}
|
||||
v.Moved = res.Tracks
|
||||
writeJSON(w, http.StatusOK, v)
|
||||
}
|
||||
@@ -79,7 +79,7 @@ func (h *handlers) handleGetSearchSettings(w http.ResponseWriter, r *http.Reques
|
||||
writeErrWithLog(w, h.logger, "admin search settings: get", apierror.Internal(err))
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, searchSettings(s, h.importer.Roots))
|
||||
writeJSON(w, http.StatusOK, searchSettings(s, h.importer.Roots()))
|
||||
}
|
||||
|
||||
type putSearchSettingsBody struct {
|
||||
@@ -117,7 +117,7 @@ func (h *handlers) handlePutSearchSettings(w http.ResponseWriter, r *http.Reques
|
||||
writeErr(w, apierror.BadRequest("bad_upgrade_until", "unknown quality "+body.UpgradeUntil))
|
||||
return
|
||||
}
|
||||
if body.ImportRoot != "" && !slices.Contains(h.importer.Roots, body.ImportRoot) {
|
||||
if body.ImportRoot != "" && !slices.Contains(h.importer.Roots(), body.ImportRoot) {
|
||||
writeErr(w, apierror.BadRequest("bad_import_root", "file new music into one of the library's folders"))
|
||||
return
|
||||
}
|
||||
@@ -136,7 +136,7 @@ func (h *handlers) handlePutSearchSettings(w http.ResponseWriter, r *http.Reques
|
||||
writeErrWithLog(w, h.logger, "admin search settings: put", apierror.Internal(err))
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, searchSettings(s, h.importer.Roots))
|
||||
writeJSON(w, http.StatusOK, searchSettings(s, h.importer.Roots()))
|
||||
}
|
||||
|
||||
// candidateView is one release a want's last search found.
|
||||
|
||||
@@ -151,7 +151,7 @@ func (h *handlers) handleGetAdminStatus(w http.ResponseWriter, r *http.Request)
|
||||
out.Library = libraryStatusView{Tracks: lib.Tracks, Albums: lib.Albums, Artists: lib.Artists,
|
||||
Bytes: lib.Bytes, Disks: []diskView{}}
|
||||
if h.importer != nil {
|
||||
for _, root := range h.importer.Roots {
|
||||
for _, root := range h.importer.Roots() {
|
||||
out.Library.Disks = append(out.Library.Disks, diskOf(root))
|
||||
}
|
||||
}
|
||||
|
||||
+13
-6
@@ -299,6 +299,11 @@ func Mount(r chi.Router, pool *pgxpool.Pool, logger *slog.Logger, events *playev
|
||||
admin.Get("/library/reacquisition", h.handleGetReacquisitionSettings)
|
||||
admin.Put("/library/reacquisition", h.handleUpdateReacquisitionSettings)
|
||||
|
||||
// Where the library lives (#6129). A changed folder moves its
|
||||
// tracks rather than re-adding them.
|
||||
admin.Get("/library/folders", h.handleGetLibraryFolders)
|
||||
admin.Put("/library/folders", h.handlePutLibraryFolders)
|
||||
|
||||
admin.Get("/scan/status", h.handleGetScanStatus)
|
||||
admin.Post("/scan/run", h.handleTriggerScan)
|
||||
// Sits under /library rather than /tracks because what it
|
||||
@@ -498,11 +503,13 @@ func artSources(s *coverart.SettingsService) catalogue.ArtSources {
|
||||
return s
|
||||
}
|
||||
|
||||
// scannerRoots are the library folders, or none when the API runs without a
|
||||
// scanner (tests).
|
||||
func scannerRoots(s *library.Scanner) []string {
|
||||
if s == nil {
|
||||
return nil
|
||||
// scannerRoots reads the library folders, or none when the API runs without
|
||||
// a scanner (tests).
|
||||
func scannerRoots(s *library.Scanner) func() []string {
|
||||
return func() []string {
|
||||
if s == nil {
|
||||
return nil
|
||||
}
|
||||
return s.Roots()
|
||||
}
|
||||
return s.Roots()
|
||||
}
|
||||
|
||||
@@ -0,0 +1,162 @@
|
||||
// Code generated by sqlc. DO NOT EDIT.
|
||||
// versions:
|
||||
// sqlc v1.31.1
|
||||
// source: library_settings.sql
|
||||
|
||||
package dbq
|
||||
|
||||
import (
|
||||
"context"
|
||||
)
|
||||
|
||||
const countTracksUnder = `-- name: CountTracksUnder :one
|
||||
SELECT count(*) FROM tracks WHERE starts_with(file_path, $1::text || '/')
|
||||
`
|
||||
|
||||
func (q *Queries) CountTracksUnder(ctx context.Context, root string) (int64, error) {
|
||||
row := q.db.QueryRow(ctx, countTracksUnder, root)
|
||||
var count int64
|
||||
err := row.Scan(&count)
|
||||
return count, err
|
||||
}
|
||||
|
||||
const getLibrarySettings = `-- name: GetLibrarySettings :one
|
||||
SELECT id, scan_paths, updated_at FROM library_settings WHERE id = true
|
||||
`
|
||||
|
||||
func (q *Queries) GetLibrarySettings(ctx context.Context) (LibrarySetting, error) {
|
||||
row := q.db.QueryRow(ctx, getLibrarySettings)
|
||||
var i LibrarySetting
|
||||
err := row.Scan(&i.ID, &i.ScanPaths, &i.UpdatedAt)
|
||||
return i, err
|
||||
}
|
||||
|
||||
const rerootAlbumCoverPaths = `-- name: RerootAlbumCoverPaths :execrows
|
||||
UPDATE albums
|
||||
SET cover_art_path = $1::text || substr(cover_art_path, length($2::text) + 1)
|
||||
WHERE starts_with(cover_art_path, $2::text || '/')
|
||||
`
|
||||
|
||||
type RerootAlbumCoverPathsParams struct {
|
||||
NewRoot string
|
||||
OldRoot string
|
||||
}
|
||||
|
||||
func (q *Queries) RerootAlbumCoverPaths(ctx context.Context, arg RerootAlbumCoverPathsParams) (int64, error) {
|
||||
result, err := q.db.Exec(ctx, rerootAlbumCoverPaths, arg.NewRoot, arg.OldRoot)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return result.RowsAffected(), nil
|
||||
}
|
||||
|
||||
const rerootGrabDestinations = `-- name: RerootGrabDestinations :execrows
|
||||
UPDATE grabs
|
||||
SET destination = $1::text || substr(destination, length($2::text) + 1)
|
||||
WHERE destination = $2::text
|
||||
OR starts_with(destination, $2::text || '/')
|
||||
`
|
||||
|
||||
type RerootGrabDestinationsParams struct {
|
||||
NewRoot string
|
||||
OldRoot string
|
||||
}
|
||||
|
||||
func (q *Queries) RerootGrabDestinations(ctx context.Context, arg RerootGrabDestinationsParams) (int64, error) {
|
||||
result, err := q.db.Exec(ctx, rerootGrabDestinations, arg.NewRoot, arg.OldRoot)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return result.RowsAffected(), nil
|
||||
}
|
||||
|
||||
const rerootImportRoot = `-- name: RerootImportRoot :execrows
|
||||
UPDATE acquisition_settings
|
||||
SET import_root = $1::text
|
||||
WHERE import_root = $2::text
|
||||
`
|
||||
|
||||
type RerootImportRootParams struct {
|
||||
NewRoot string
|
||||
OldRoot string
|
||||
}
|
||||
|
||||
func (q *Queries) RerootImportRoot(ctx context.Context, arg RerootImportRootParams) (int64, error) {
|
||||
result, err := q.db.Exec(ctx, rerootImportRoot, arg.NewRoot, arg.OldRoot)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return result.RowsAffected(), nil
|
||||
}
|
||||
|
||||
const rerootTrackPaths = `-- name: RerootTrackPaths :execrows
|
||||
|
||||
UPDATE tracks
|
||||
SET file_path = $1::text || substr(file_path, length($2::text) + 1)
|
||||
WHERE starts_with(file_path, $2::text || '/')
|
||||
`
|
||||
|
||||
type RerootTrackPathsParams struct {
|
||||
NewRoot string
|
||||
OldRoot string
|
||||
}
|
||||
|
||||
// A library folder that moved (#6129): every path under old_root is rewritten
|
||||
// to the same place under new_root, so the tracks keep their ids and with them
|
||||
// their plays, likes and playlists. updated_at is left alone on purpose: the
|
||||
// scan compares it with each file's mtime, and the files have not changed.
|
||||
func (q *Queries) RerootTrackPaths(ctx context.Context, arg RerootTrackPathsParams) (int64, error) {
|
||||
result, err := q.db.Exec(ctx, rerootTrackPaths, arg.NewRoot, arg.OldRoot)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return result.RowsAffected(), nil
|
||||
}
|
||||
|
||||
const sampleTrackPathsUnder = `-- name: SampleTrackPathsUnder :many
|
||||
SELECT file_path FROM tracks
|
||||
WHERE starts_with(file_path, $1::text || '/') AND missing_since IS NULL
|
||||
ORDER BY random()
|
||||
LIMIT $2::int
|
||||
`
|
||||
|
||||
type SampleTrackPathsUnderParams struct {
|
||||
Root string
|
||||
N int32
|
||||
}
|
||||
|
||||
// A few present tracks under a folder, to check a move before rewriting it.
|
||||
func (q *Queries) SampleTrackPathsUnder(ctx context.Context, arg SampleTrackPathsUnderParams) ([]string, error) {
|
||||
rows, err := q.db.Query(ctx, sampleTrackPathsUnder, arg.Root, arg.N)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
var items []string
|
||||
for rows.Next() {
|
||||
var file_path string
|
||||
if err := rows.Scan(&file_path); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
items = append(items, file_path)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return items, nil
|
||||
}
|
||||
|
||||
const updateLibraryScanPaths = `-- name: UpdateLibraryScanPaths :one
|
||||
UPDATE library_settings
|
||||
SET scan_paths = $1::text[],
|
||||
updated_at = now()
|
||||
WHERE id = true
|
||||
RETURNING id, scan_paths, updated_at
|
||||
`
|
||||
|
||||
func (q *Queries) UpdateLibraryScanPaths(ctx context.Context, scanPaths []string) (LibrarySetting, error) {
|
||||
row := q.db.QueryRow(ctx, updateLibraryScanPaths, scanPaths)
|
||||
var i LibrarySetting
|
||||
err := row.Scan(&i.ID, &i.ScanPaths, &i.UpdatedAt)
|
||||
return i, err
|
||||
}
|
||||
@@ -506,6 +506,12 @@ type LibraryChange struct {
|
||||
ChangedAt pgtype.Timestamptz
|
||||
}
|
||||
|
||||
type LibrarySetting struct {
|
||||
ID bool
|
||||
ScanPaths []string
|
||||
UpdatedAt pgtype.Timestamptz
|
||||
}
|
||||
|
||||
type ListenbrainzSimilarRecording struct {
|
||||
SeedTrackID pgtype.UUID
|
||||
RecordingMbid string
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
DROP TABLE IF EXISTS library_settings;
|
||||
@@ -0,0 +1,11 @@
|
||||
-- The library's folders, set in Settings (#6129). Until someone saves them
|
||||
-- there, scan_paths is NULL and MINSTREL_LIBRARY_SCAN_PATHS names the folders,
|
||||
-- as it always has; from the first save the row is what the server reads.
|
||||
CREATE TABLE library_settings (
|
||||
id boolean PRIMARY KEY DEFAULT true,
|
||||
scan_paths text[],
|
||||
updated_at timestamptz NOT NULL DEFAULT now(),
|
||||
|
||||
CONSTRAINT library_settings_singleton CHECK (id = true)
|
||||
);
|
||||
INSERT INTO library_settings (id) VALUES (true) ON CONFLICT (id) DO NOTHING;
|
||||
@@ -0,0 +1,45 @@
|
||||
-- name: GetLibrarySettings :one
|
||||
SELECT * FROM library_settings WHERE id = true;
|
||||
|
||||
-- name: UpdateLibraryScanPaths :one
|
||||
UPDATE library_settings
|
||||
SET scan_paths = sqlc.arg(scan_paths)::text[],
|
||||
updated_at = now()
|
||||
WHERE id = true
|
||||
RETURNING *;
|
||||
|
||||
-- A library folder that moved (#6129): every path under old_root is rewritten
|
||||
-- to the same place under new_root, so the tracks keep their ids and with them
|
||||
-- their plays, likes and playlists. updated_at is left alone on purpose: the
|
||||
-- scan compares it with each file's mtime, and the files have not changed.
|
||||
|
||||
-- name: RerootTrackPaths :execrows
|
||||
UPDATE tracks
|
||||
SET file_path = sqlc.arg(new_root)::text || substr(file_path, length(sqlc.arg(old_root)::text) + 1)
|
||||
WHERE starts_with(file_path, sqlc.arg(old_root)::text || '/');
|
||||
|
||||
-- name: RerootAlbumCoverPaths :execrows
|
||||
UPDATE albums
|
||||
SET cover_art_path = sqlc.arg(new_root)::text || substr(cover_art_path, length(sqlc.arg(old_root)::text) + 1)
|
||||
WHERE starts_with(cover_art_path, sqlc.arg(old_root)::text || '/');
|
||||
|
||||
-- name: RerootGrabDestinations :execrows
|
||||
UPDATE grabs
|
||||
SET destination = sqlc.arg(new_root)::text || substr(destination, length(sqlc.arg(old_root)::text) + 1)
|
||||
WHERE destination = sqlc.arg(old_root)::text
|
||||
OR starts_with(destination, sqlc.arg(old_root)::text || '/');
|
||||
|
||||
-- name: RerootImportRoot :execrows
|
||||
UPDATE acquisition_settings
|
||||
SET import_root = sqlc.arg(new_root)::text
|
||||
WHERE import_root = sqlc.arg(old_root)::text;
|
||||
|
||||
-- name: SampleTrackPathsUnder :many
|
||||
-- A few present tracks under a folder, to check a move before rewriting it.
|
||||
SELECT file_path FROM tracks
|
||||
WHERE starts_with(file_path, sqlc.arg(root)::text || '/') AND missing_since IS NULL
|
||||
ORDER BY random()
|
||||
LIMIT sqlc.arg(n)::int;
|
||||
|
||||
-- name: CountTracksUnder :one
|
||||
SELECT count(*) FROM tracks WHERE starts_with(file_path, sqlc.arg(root)::text || '/');
|
||||
@@ -63,9 +63,10 @@ type Importer struct {
|
||||
Scanner Scanner
|
||||
Catalogue Catalogue
|
||||
Printer Printer
|
||||
// Roots are the library's folders (the scan paths). New music is filed
|
||||
// under the one the settings name, or the first.
|
||||
Roots []string
|
||||
// Roots are the library's folders (the scan paths), read on every use
|
||||
// because they change in Settings (#6129). New music is filed under the
|
||||
// one the settings name, or the first.
|
||||
Roots func() []string
|
||||
// DataDir is where cached art lives, for tidying an artist an upgrade
|
||||
// leaves with nothing (#5524). Empty skips it.
|
||||
DataDir string
|
||||
@@ -79,7 +80,7 @@ type Importer struct {
|
||||
|
||||
// New returns an Importer. printer may be nil: the check then rests on
|
||||
// lengths and tags.
|
||||
func New(pool *pgxpool.Pool, scanner Scanner, cat Catalogue, printer Printer, roots []string, logger *slog.Logger) *Importer {
|
||||
func New(pool *pgxpool.Pool, scanner Scanner, cat Catalogue, printer Printer, roots func() []string, logger *slog.Logger) *Importer {
|
||||
if logger == nil {
|
||||
logger = slog.Default()
|
||||
}
|
||||
@@ -549,14 +550,15 @@ func (im *Importer) editions(ctx context.Context, group string) ([]Edition, erro
|
||||
|
||||
// root is the library folder new music is filed into.
|
||||
func (im *Importer) root(ctx context.Context, q *dbq.Queries) (string, error) {
|
||||
if len(im.Roots) == 0 {
|
||||
return "", errors.New("there is no library folder to file new music into (MINSTREL_LIBRARY_SCAN_PATHS)")
|
||||
roots := im.Roots()
|
||||
if len(roots) == 0 {
|
||||
return "", errors.New("there is no library folder to file new music into; add one in Settings → Library")
|
||||
}
|
||||
s, err := q.GetAcquisitionSettings(ctx)
|
||||
if err == nil && s.ImportRoot != "" && slices.Contains(im.Roots, s.ImportRoot) {
|
||||
if err == nil && s.ImportRoot != "" && slices.Contains(roots, s.ImportRoot) {
|
||||
return s.ImportRoot, nil
|
||||
}
|
||||
return im.Roots[0], nil
|
||||
return roots[0], nil
|
||||
}
|
||||
|
||||
// held reads which of the planned tracks the library already has: by
|
||||
|
||||
@@ -130,7 +130,7 @@ func newStack(t *testing.T, printer Printer) *stack {
|
||||
}
|
||||
s := &stack{pool: pool, q: q, client: &fakeClient{content: content}, scanner: &fakeScanner{},
|
||||
library: t.TempDir(), want: w, grab: g}
|
||||
s.im = New(pool, s.scanner, fakeCatalogue{}, printer, []string{s.library}, slog.New(slog.NewTextHandler(io.Discard, nil)))
|
||||
s.im = New(pool, s.scanner, fakeCatalogue{}, printer, func() []string { return []string{s.library} }, slog.New(slog.NewTextHandler(io.Discard, nil)))
|
||||
s.im.ClientFor = func(context.Context, string) (downloader.Driver, error) { return s.client, nil }
|
||||
return s
|
||||
}
|
||||
|
||||
@@ -0,0 +1,231 @@
|
||||
package library
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"strings"
|
||||
|
||||
"github.com/jackc/pgx/v5"
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
|
||||
"git.fabledsword.com/bvandeusen/minstrel/internal/db/dbq"
|
||||
)
|
||||
|
||||
// The library's folders (#6129). They are set in Settings and changed without
|
||||
// a restart; MINSTREL_LIBRARY_SCAN_PATHS names them until the first save. The
|
||||
// Scanner holds the live list, and the watcher and the importer read it there.
|
||||
//
|
||||
// Changing a folder's path is a move, not a new library: when the files that
|
||||
// were under /music are now under /data/music, every row under the old path is
|
||||
// rewritten to the new one, so the tracks keep their ids and their history.
|
||||
// The scan never sees the change as a mass deletion followed by a mass add.
|
||||
|
||||
// LoadRoots is the folders the server starts with: the saved ones, or env
|
||||
// when nothing has been saved. fromSettings says which.
|
||||
func LoadRoots(ctx context.Context, pool *pgxpool.Pool, env []string) (roots []string, fromSettings bool, err error) {
|
||||
row, err := dbq.New(pool).GetLibrarySettings(ctx)
|
||||
if err != nil {
|
||||
return env, false, fmt.Errorf("library settings: load: %w", err)
|
||||
}
|
||||
if len(row.ScanPaths) == 0 {
|
||||
return env, false, nil
|
||||
}
|
||||
return row.ScanPaths, true, nil
|
||||
}
|
||||
|
||||
// RootChange is one folder of a save. Was is the folder it replaces, whose
|
||||
// files have moved to Path; "" is a folder added.
|
||||
type RootChange struct {
|
||||
Path string
|
||||
Was string
|
||||
}
|
||||
|
||||
// RerootResult is what a save rewrote.
|
||||
type RerootResult struct {
|
||||
Roots []string
|
||||
Tracks int64
|
||||
}
|
||||
|
||||
// RootsError is a save the folders refuse. Its text says why, in a sentence
|
||||
// for the operator.
|
||||
type RootsError struct{ msg string }
|
||||
|
||||
func (e *RootsError) Error() string { return e.msg }
|
||||
|
||||
func badRoots(format string, args ...any) error {
|
||||
return &RootsError{msg: fmt.Sprintf(format, args...)}
|
||||
}
|
||||
|
||||
// moveSample is how many present tracks a move checks for at the new path,
|
||||
// and moveSampleMinFound how many of them must be there.
|
||||
const (
|
||||
moveSample = 20
|
||||
moveSampleMinFound = 0.5
|
||||
)
|
||||
|
||||
// SetRoots validates the folders, rewrites the paths of any that moved, saves
|
||||
// them and starts reading them. Nothing is written when any check fails.
|
||||
func (s *Scanner) SetRoots(ctx context.Context, changes []RootChange) (RerootResult, error) {
|
||||
s.rootsMu.Lock()
|
||||
defer s.rootsMu.Unlock()
|
||||
|
||||
roots, moves, err := planRoots(s.paths, changes)
|
||||
if err != nil {
|
||||
return RerootResult{}, err
|
||||
}
|
||||
q := dbq.New(s.pool)
|
||||
// A scan walking the old folders would put their paths back, or mark the
|
||||
// moved rows missing when it reconciles.
|
||||
_, err = q.GetInFlightScanRun(ctx)
|
||||
if err == nil {
|
||||
return RerootResult{}, badRoots("A library scan is running; save again once it finishes.")
|
||||
}
|
||||
if !errors.Is(err, pgx.ErrNoRows) {
|
||||
return RerootResult{}, fmt.Errorf("in-flight scan check: %w", err)
|
||||
}
|
||||
for _, m := range moves {
|
||||
if err := checkMove(ctx, q, m); err != nil {
|
||||
return RerootResult{}, err
|
||||
}
|
||||
}
|
||||
|
||||
var res RerootResult
|
||||
err = pgx.BeginFunc(ctx, s.pool, func(tx pgx.Tx) error {
|
||||
tq := dbq.New(tx)
|
||||
for _, m := range moves {
|
||||
n, err := tq.RerootTrackPaths(ctx, dbq.RerootTrackPathsParams{NewRoot: m.Path, OldRoot: m.Was})
|
||||
if err != nil {
|
||||
if isUniqueViolation(err) {
|
||||
return badRoots("Some tracks under %s are already in the library at %s, "+
|
||||
"so its tracks cannot move there.", m.Was, m.Path)
|
||||
}
|
||||
return fmt.Errorf("reroot tracks: %w", err)
|
||||
}
|
||||
res.Tracks += n
|
||||
if _, err := tq.RerootAlbumCoverPaths(ctx, dbq.RerootAlbumCoverPathsParams{NewRoot: m.Path, OldRoot: m.Was}); err != nil {
|
||||
return fmt.Errorf("reroot covers: %w", err)
|
||||
}
|
||||
if _, err := tq.RerootGrabDestinations(ctx, dbq.RerootGrabDestinationsParams{NewRoot: m.Path, OldRoot: m.Was}); err != nil {
|
||||
return fmt.Errorf("reroot downloads: %w", err)
|
||||
}
|
||||
if _, err := tq.RerootImportRoot(ctx, dbq.RerootImportRootParams{NewRoot: m.Path, OldRoot: m.Was}); err != nil {
|
||||
return fmt.Errorf("reroot import folder: %w", err)
|
||||
}
|
||||
}
|
||||
_, err := tq.UpdateLibraryScanPaths(ctx, roots)
|
||||
return err
|
||||
})
|
||||
if err != nil {
|
||||
return RerootResult{}, err
|
||||
}
|
||||
for _, m := range moves {
|
||||
s.logger.Info("library folder moved", "from", m.Was, "to", m.Path)
|
||||
}
|
||||
s.paths = roots
|
||||
select {
|
||||
case s.rootsChanged <- struct{}{}:
|
||||
default:
|
||||
}
|
||||
res.Roots = slices.Clone(roots)
|
||||
return res, nil
|
||||
}
|
||||
|
||||
// RootsChanged is signalled after a save changes the folders. A Scanner built
|
||||
// without New never signals.
|
||||
func (s *Scanner) RootsChanged() <-chan struct{} { return s.rootsChanged }
|
||||
|
||||
// planRoots checks a save against the current folders and returns the new
|
||||
// list and the folders that moved.
|
||||
func planRoots(current []string, changes []RootChange) ([]string, []RootChange, error) {
|
||||
if len(changes) == 0 {
|
||||
return nil, nil, badRoots("Name at least one folder.")
|
||||
}
|
||||
roots := make([]string, 0, len(changes))
|
||||
var moves []RootChange
|
||||
replaced := map[string]bool{}
|
||||
for _, c := range changes {
|
||||
p := strings.TrimSpace(c.Path)
|
||||
if !filepath.IsAbs(p) {
|
||||
return nil, nil, badRoots("%q is not a full path; it starts with /.", c.Path)
|
||||
}
|
||||
p = filepath.Clean(p)
|
||||
if p == "/" {
|
||||
return nil, nil, badRoots("The library cannot be the whole filesystem.")
|
||||
}
|
||||
if err := checkDir(p); err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
for _, r := range roots {
|
||||
if r == p {
|
||||
return nil, nil, badRoots("%s is listed twice.", p)
|
||||
}
|
||||
if within(p, r) || within(r, p) {
|
||||
return nil, nil, badRoots("%s and %s overlap; name one of them.", r, p)
|
||||
}
|
||||
}
|
||||
roots = append(roots, p)
|
||||
if c.Was == "" || c.Was == p {
|
||||
continue
|
||||
}
|
||||
if !slices.Contains(current, c.Was) {
|
||||
return nil, nil, badRoots("%s is not one of the library's folders.", c.Was)
|
||||
}
|
||||
if replaced[c.Was] {
|
||||
return nil, nil, badRoots("%s can only move to one place.", c.Was)
|
||||
}
|
||||
replaced[c.Was] = true
|
||||
moves = append(moves, RootChange{Path: p, Was: c.Was})
|
||||
}
|
||||
return roots, moves, nil
|
||||
}
|
||||
|
||||
// checkDir refuses a folder the server cannot read: a typo, or a mount the
|
||||
// container does not have.
|
||||
func checkDir(p string) error {
|
||||
info, err := os.Stat(p)
|
||||
if err != nil {
|
||||
if errors.Is(err, os.ErrNotExist) {
|
||||
return badRoots("%s does not exist on the server. Is it mounted into the container?", p)
|
||||
}
|
||||
return badRoots("%s cannot be read: %v.", p, err)
|
||||
}
|
||||
if !info.IsDir() {
|
||||
return badRoots("%s is a file, not a folder.", p)
|
||||
}
|
||||
if _, err := os.ReadDir(p); err != nil {
|
||||
return badRoots("%s cannot be read: %v.", p, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// checkMove looks for a sample of the folder's tracks at the new path, so a
|
||||
// move to the wrong folder is refused before any row is rewritten.
|
||||
func checkMove(ctx context.Context, q *dbq.Queries, m RootChange) error {
|
||||
sample, err := q.SampleTrackPathsUnder(ctx, dbq.SampleTrackPathsUnderParams{Root: m.Was, N: moveSample})
|
||||
if err != nil {
|
||||
return fmt.Errorf("sample tracks: %w", err)
|
||||
}
|
||||
if len(sample) == 0 {
|
||||
return nil
|
||||
}
|
||||
found := 0
|
||||
for _, p := range sample {
|
||||
if _, err := os.Stat(m.Path + strings.TrimPrefix(p, m.Was)); err == nil {
|
||||
found++
|
||||
}
|
||||
}
|
||||
if float64(found) < moveSampleMinFound*float64(len(sample)) {
|
||||
return badRoots("Only %d of %d tracks checked from %s are at the same place under %s. "+
|
||||
"Is that the folder they moved to?", found, len(sample), m.Was, m.Path)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// within reports whether p is inside dir.
|
||||
func within(p, dir string) bool {
|
||||
return strings.HasPrefix(p, dir+"/")
|
||||
}
|
||||
@@ -0,0 +1,148 @@
|
||||
package library
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"io"
|
||||
"log/slog"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
|
||||
"git.fabledsword.com/bvandeusen/minstrel/internal/db/dbq"
|
||||
)
|
||||
|
||||
func mkdir(t *testing.T, parts ...string) string {
|
||||
t.Helper()
|
||||
p := filepath.Join(parts...)
|
||||
if err := os.MkdirAll(p, 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return p
|
||||
}
|
||||
|
||||
func TestPlanRoots(t *testing.T) {
|
||||
base := t.TempDir()
|
||||
music := mkdir(t, base, "music")
|
||||
data := mkdir(t, base, "data", "music")
|
||||
more := mkdir(t, base, "more")
|
||||
file := filepath.Join(base, "a.mp3")
|
||||
if err := os.WriteFile(file, []byte("x"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
current := []string{music, more}
|
||||
|
||||
roots, moves, err := planRoots(current, []RootChange{{Path: data + "/", Was: music}, {Path: more, Was: more}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !slices.Equal(roots, []string{data, more}) {
|
||||
t.Errorf("roots %v", roots)
|
||||
}
|
||||
if len(moves) != 1 || moves[0] != (RootChange{Path: data, Was: music}) {
|
||||
t.Errorf("moves %v; want only %s → %s", moves, music, data)
|
||||
}
|
||||
|
||||
for name, tc := range map[string]struct {
|
||||
changes []RootChange
|
||||
mentions string
|
||||
}{
|
||||
"none": {nil, "At least one"},
|
||||
"relative": {[]RootChange{{Path: "music"}}, "full path"},
|
||||
"root": {[]RootChange{{Path: "/"}}, "whole filesystem"},
|
||||
"absent": {[]RootChange{{Path: filepath.Join(base, "nope")}}, "does not exist"},
|
||||
"a file": {[]RootChange{{Path: file}}, "not a folder"},
|
||||
"twice": {[]RootChange{{Path: music}, {Path: music}}, "twice"},
|
||||
"nested": {[]RootChange{{Path: base}, {Path: music}}, "overlap"},
|
||||
"unknown was": {[]RootChange{{Path: data, Was: "/elsewhere"}}, "not one of"},
|
||||
"split a move": {[]RootChange{{Path: data, Was: music}, {Path: more, Was: music}}, "one place"},
|
||||
} {
|
||||
_, _, err := planRoots(current, tc.changes)
|
||||
var re *RootsError
|
||||
if !errors.As(err, &re) || !strings.Contains(err.Error(), tc.mentions) {
|
||||
t.Errorf("%s: err %v; want a RootsError mentioning %q", name, err, tc.mentions)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// clearSavedRoots puts the folders back to env afterwards. It also finishes any
|
||||
// scan an earlier test left running: dbtest.ResetDB keeps scan_runs, and a
|
||||
// save refuses while a scan is in flight.
|
||||
func clearSavedRoots(t *testing.T, pool *pgxpool.Pool) {
|
||||
t.Helper()
|
||||
if _, err := pool.Exec(context.Background(),
|
||||
`UPDATE scan_runs SET finished_at = now() WHERE finished_at IS NULL`); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
_, _ = pool.Exec(context.Background(), `UPDATE library_settings SET scan_paths = NULL`)
|
||||
})
|
||||
}
|
||||
|
||||
func TestSetRoots_TracksMoveWithTheirFolder_Integration(t *testing.T) {
|
||||
pool := newPool(t)
|
||||
clearSavedRoots(t, pool)
|
||||
ctx := context.Background()
|
||||
base := t.TempDir()
|
||||
old := filepath.Join(base, "music") // gone: the mount moved
|
||||
moved := mkdir(t, base, "data", "music", "Artist")
|
||||
if err := os.WriteFile(filepath.Join(moved, "a.mp3"), []byte("x"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
newRoot := filepath.Dir(moved)
|
||||
track, _, _ := seedTrack(t, pool, filepath.Join(old, "Artist", "a.mp3"))
|
||||
|
||||
s := New(pool, slog.New(slog.NewTextHandler(io.Discard, nil)), []string{old}, nil)
|
||||
res, err := s.SetRoots(ctx, []RootChange{{Path: newRoot, Was: old}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if res.Tracks != 1 {
|
||||
t.Errorf("moved %d tracks; want 1", res.Tracks)
|
||||
}
|
||||
got, err := dbq.New(pool).GetTrackByPath(ctx, filepath.Join(newRoot, "Artist", "a.mp3"))
|
||||
if err != nil {
|
||||
t.Fatalf("the track is not at its new path: %v", err)
|
||||
}
|
||||
if got.ID != track.ID {
|
||||
t.Error("the move made a new track instead of keeping the old one")
|
||||
}
|
||||
if !slices.Equal(s.Roots(), []string{newRoot}) {
|
||||
t.Errorf("roots %v", s.Roots())
|
||||
}
|
||||
select {
|
||||
case <-s.RootsChanged():
|
||||
default:
|
||||
t.Error("the watcher was not told the folders changed")
|
||||
}
|
||||
roots, saved, err := LoadRoots(ctx, pool, []string{old})
|
||||
if err != nil || !saved || !slices.Equal(roots, []string{newRoot}) {
|
||||
t.Errorf("after a restart: %v saved=%v err=%v; want the saved folder over env", roots, saved, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSetRoots_RefusesAMoveToAFolderWithoutTheFiles_Integration(t *testing.T) {
|
||||
pool := newPool(t)
|
||||
clearSavedRoots(t, pool)
|
||||
ctx := context.Background()
|
||||
base := t.TempDir()
|
||||
old := filepath.Join(base, "music")
|
||||
wrong := mkdir(t, base, "elsewhere")
|
||||
seedTrack(t, pool, filepath.Join(old, "a.mp3"))
|
||||
|
||||
s := New(pool, slog.New(slog.NewTextHandler(io.Discard, nil)), []string{old}, nil)
|
||||
var re *RootsError
|
||||
if _, err := s.SetRoots(ctx, []RootChange{{Path: wrong, Was: old}}); !errors.As(err, &re) {
|
||||
t.Fatalf("err %v; want a RootsError", err)
|
||||
}
|
||||
if _, err := dbq.New(pool).GetTrackByPath(ctx, filepath.Join(old, "a.mp3")); err != nil {
|
||||
t.Errorf("a refused move rewrote the track: %v", err)
|
||||
}
|
||||
if !slices.Equal(s.Roots(), []string{old}) {
|
||||
t.Errorf("a refused move changed the folders to %v", s.Roots())
|
||||
}
|
||||
}
|
||||
@@ -176,10 +176,11 @@ func logTrackChanges(ctx context.Context, q trackReconciler, ids []pgtype.UUID)
|
||||
// obvious case, and it is common enough — a container restart racing its volume
|
||||
// mount does exactly this.
|
||||
func (s *Scanner) verifyRootsPresent() error {
|
||||
if len(s.paths) == 0 {
|
||||
roots := s.Roots()
|
||||
if len(roots) == 0 {
|
||||
return errors.New("no scan roots configured")
|
||||
}
|
||||
for _, root := range s.paths {
|
||||
for _, root := range roots {
|
||||
info, err := os.Stat(root)
|
||||
if err != nil {
|
||||
return fmt.Errorf("scan root %q unavailable: %w", root, err)
|
||||
|
||||
@@ -12,6 +12,7 @@ import (
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/dhowden/tag"
|
||||
@@ -87,7 +88,11 @@ type Stats struct {
|
||||
type Scanner struct {
|
||||
pool *pgxpool.Pool
|
||||
logger *slog.Logger
|
||||
paths []string
|
||||
// paths are the library's folders, changed by SetRoots (#6129). Read them
|
||||
// through Roots.
|
||||
paths []string
|
||||
rootsMu sync.RWMutex
|
||||
rootsChanged chan struct{}
|
||||
// fingerprint derives a file's acoustic identity (M400). A field so an
|
||||
// integration test can substitute a deterministic one: CI has no real audio
|
||||
// to fingerprint, and what the test pins is WHEN the scan fingerprints, not
|
||||
@@ -99,7 +104,8 @@ type Scanner struct {
|
||||
}
|
||||
|
||||
func New(pool *pgxpool.Pool, logger *slog.Logger, paths []string, settings *FingerprintSettingsService) *Scanner {
|
||||
return &Scanner{pool: pool, logger: logger, paths: paths, fingerprint: computeFingerprint, settings: settings}
|
||||
return &Scanner{pool: pool, logger: logger, paths: paths, rootsChanged: make(chan struct{}, 1),
|
||||
fingerprint: computeFingerprint, settings: settings}
|
||||
}
|
||||
|
||||
// Scan walks every configured root and upserts any audio file whose mtime is
|
||||
@@ -206,7 +212,7 @@ func (s *Scanner) enumerate(
|
||||
) ([]string, int) {
|
||||
paths := make([]string, 0, 8192)
|
||||
errs := 0
|
||||
for _, root := range s.paths {
|
||||
for _, root := range s.Roots() {
|
||||
// WalkDir's own error return is folded into the per-entry handler below,
|
||||
// so a bad root is counted rather than aborting the whole scan — one
|
||||
// unreadable root shouldn't discard the others' results.
|
||||
@@ -691,4 +697,11 @@ func IsAudioFile(path string) bool {
|
||||
}
|
||||
|
||||
// Roots are the folders the scanner reads: the library.
|
||||
func (s *Scanner) Roots() []string { return append([]string(nil), s.paths...) }
|
||||
func (s *Scanner) Roots() []string {
|
||||
if s == nil {
|
||||
return nil
|
||||
}
|
||||
s.rootsMu.RLock()
|
||||
defer s.rootsMu.RUnlock()
|
||||
return append([]string(nil), s.paths...)
|
||||
}
|
||||
|
||||
@@ -67,18 +67,16 @@ type Watcher struct {
|
||||
scanner *Scanner
|
||||
enricher *coverart.Enricher
|
||||
logger *slog.Logger
|
||||
roots []string
|
||||
debounce time.Duration
|
||||
}
|
||||
|
||||
func NewWatcher(
|
||||
scanner *Scanner, enricher *coverart.Enricher, logger *slog.Logger, roots []string,
|
||||
) *Watcher {
|
||||
// NewWatcher watches the scanner's folders, and moves to the new ones when
|
||||
// they change in Settings (#6129).
|
||||
func NewWatcher(scanner *Scanner, enricher *coverart.Enricher, logger *slog.Logger) *Watcher {
|
||||
return &Watcher{
|
||||
scanner: scanner,
|
||||
enricher: enricher,
|
||||
logger: logger,
|
||||
roots: roots,
|
||||
debounce: watcherDebounce,
|
||||
}
|
||||
}
|
||||
@@ -87,8 +85,32 @@ func NewWatcher(
|
||||
// fsnotify watcher cannot be created at all; per-path watch failures and
|
||||
// fsnotify errors are logged and tolerated.
|
||||
func (w *Watcher) Run(ctx context.Context) error {
|
||||
if len(w.roots) == 0 {
|
||||
w.logger.Info("library watcher: no scan paths configured, watcher disabled")
|
||||
for {
|
||||
if err := w.watch(ctx, w.scanner.Roots()); err != nil {
|
||||
return err
|
||||
}
|
||||
if ctx.Err() != nil {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// watch watches roots until ctx is cancelled or the folders change.
|
||||
func (w *Watcher) watch(ctx context.Context, roots []string) error {
|
||||
ctx, cancel := context.WithCancel(ctx)
|
||||
defer cancel()
|
||||
go func() {
|
||||
select {
|
||||
case <-w.scanner.RootsChanged():
|
||||
w.logger.Info("library watcher: folders changed, restarting")
|
||||
cancel()
|
||||
case <-ctx.Done():
|
||||
}
|
||||
}()
|
||||
|
||||
if len(roots) == 0 {
|
||||
w.logger.Info("library watcher: no scan paths configured, watcher idle")
|
||||
<-ctx.Done()
|
||||
return nil
|
||||
}
|
||||
fsw, err := fsnotify.NewWatcher()
|
||||
@@ -98,10 +120,10 @@ func (w *Watcher) Run(ctx context.Context) error {
|
||||
}
|
||||
defer func() { _ = fsw.Close() }()
|
||||
|
||||
for _, root := range w.roots {
|
||||
for _, root := range roots {
|
||||
w.addRecursive(fsw, root)
|
||||
}
|
||||
w.logger.Info("library watcher: started", "roots", len(w.roots))
|
||||
w.logger.Info("library watcher: started", "roots", len(roots))
|
||||
|
||||
batches := make(chan []string, batchQueueDepth)
|
||||
go w.processLoop(ctx, batches)
|
||||
|
||||
Reference in New Issue
Block a user