Merge pull request 'M1/#293: library scanner (walk + tag-parse + upsert incremental) + admin scan endpoint' (#6) from dev into main

This commit was merged in pull request #6.
This commit is contained in:
2026-04-19 15:21:10 +00:00
12 changed files with 577 additions and 6 deletions
+11 -1
View File
@@ -14,6 +14,7 @@ import (
"git.fabledsword.com/bvandeusen/minstrel/internal/auth"
"git.fabledsword.com/bvandeusen/minstrel/internal/config"
"git.fabledsword.com/bvandeusen/minstrel/internal/db"
"git.fabledsword.com/bvandeusen/minstrel/internal/library"
"git.fabledsword.com/bvandeusen/minstrel/internal/logging"
"git.fabledsword.com/bvandeusen/minstrel/internal/server"
)
@@ -62,7 +63,16 @@ func run() error {
return fmt.Errorf("bootstrap admin: %w", err)
}
srv := server.New(logger)
scanner := library.New(pool, logger, cfg.Library.ScanPaths)
if cfg.Library.ScanOnStartup && len(cfg.Library.ScanPaths) > 0 {
go func() {
if _, err := scanner.Scan(ctx); err != nil {
logger.Error("startup scan failed", "err", err)
}
}()
}
srv := server.New(logger, pool, scanner)
httpServer := &http.Server{
Addr: cfg.Server.Address,
Handler: srv.Router(),
+1
View File
@@ -3,6 +3,7 @@ module git.fabledsword.com/bvandeusen/minstrel
go 1.23.0
require (
github.com/dhowden/tag v0.0.0-20240417053706-3d75831295e8
github.com/go-chi/chi/v5 v5.2.5
github.com/golang-migrate/migrate/v4 v4.18.2
github.com/jackc/pgx/v5 v5.7.4
+2
View File
@@ -5,6 +5,8 @@ github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/dhowden/tag v0.0.0-20240417053706-3d75831295e8 h1:OtSeLS5y0Uy01jaKK4mA/WVIYtpzVm63vLVAPzJXigg=
github.com/dhowden/tag v0.0.0-20240417053706-3d75831295e8/go.mod h1:apkPC/CR3s48O2D7Y++n1XWEpgPNNCjXYga3PPbJe2E=
github.com/dhui/dktest v0.4.4 h1:+I4s6JRE1yGuqflzwqG+aIaMdgXIorCf5P98JnaAWa8=
github.com/dhui/dktest v0.4.4/go.mod h1:4+22R4lgsdAXrDyaH4Nqx2JEz2hLp49MqQmm9HLCQhM=
github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk=
+54
View File
@@ -0,0 +1,54 @@
package auth
import (
"context"
"errors"
"net/http"
"github.com/jackc/pgx/v5"
"github.com/jackc/pgx/v5/pgxpool"
"git.fabledsword.com/bvandeusen/minstrel/internal/db/dbq"
)
type ctxKey int
const userCtxKey ctxKey = 1
// RequireAdmin gates a handler on X-API-Token matching an admin user. This is
// the Minstrel-native token path (`/api/*`); Subsonic-compatible auth under
// `/rest/*` lands with the Subsonic server.
func RequireAdmin(pool *pgxpool.Pool) func(http.Handler) http.Handler {
return func(next http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
token := r.Header.Get("X-API-Token")
if token == "" {
http.Error(w, "missing X-API-Token", http.StatusUnauthorized)
return
}
q := dbq.New(pool)
user, err := q.GetUserByAPIToken(r.Context(), token)
if err != nil {
if errors.Is(err, pgx.ErrNoRows) {
http.Error(w, "invalid token", http.StatusUnauthorized)
return
}
http.Error(w, "auth lookup failed", http.StatusInternalServerError)
return
}
if !user.IsAdmin {
http.Error(w, "admin required", http.StatusForbidden)
return
}
ctx := context.WithValue(r.Context(), userCtxKey, user)
next.ServeHTTP(w, r.WithContext(ctx))
})
}
}
// UserFromContext returns the authenticated user when the request passed
// through RequireAdmin (or a future RequireUser middleware).
func UserFromContext(ctx context.Context) (dbq.User, bool) {
u, ok := ctx.Value(userCtxKey).(dbq.User)
return u, ok
}
+28
View File
@@ -3,6 +3,7 @@ package config
import (
"fmt"
"os"
"strconv"
"strings"
"gopkg.in/yaml.v3"
@@ -13,6 +14,7 @@ type Config struct {
Database DatabaseConfig `yaml:"database"`
Log LogConfig `yaml:"log"`
Auth AuthConfig `yaml:"auth"`
Library LibraryConfig `yaml:"library"`
}
type ServerConfig struct {
@@ -40,6 +42,11 @@ type AdminBootstrapConfig struct {
Password string `yaml:"password"`
}
type LibraryConfig struct {
ScanPaths []string `yaml:"scan_paths"`
ScanOnStartup bool `yaml:"scan_on_startup"`
}
func Default() Config {
return Config{
Server: ServerConfig{Address: ":4533"},
@@ -83,4 +90,25 @@ func applyEnv(cfg *Config) {
if v, ok := os.LookupEnv("SMARTMUSIC_AUTH_ADMIN_PASSWORD"); ok {
cfg.Auth.AdminBootstrap.Password = v
}
if v, ok := os.LookupEnv("SMARTMUSIC_LIBRARY_SCAN_PATHS"); ok {
cfg.Library.ScanPaths = splitPaths(v)
}
if v, ok := os.LookupEnv("SMARTMUSIC_LIBRARY_SCAN_ON_STARTUP"); ok {
if b, err := strconv.ParseBool(v); err == nil {
cfg.Library.ScanOnStartup = b
}
}
}
// splitPaths splits a colon-separated path list, dropping empty entries.
// Colon matches PATH-style Unix conventions; Minstrel ships Linux-only.
func splitPaths(s string) []string {
parts := strings.Split(s, ":")
out := parts[:0]
for _, p := range parts {
if p != "" {
out = append(out, p)
}
}
return out
}
+46
View File
@@ -94,3 +94,49 @@ func TestAdminBootstrapEnvOverrides(t *testing.T) {
t.Errorf("admin password = %q", cfg.Auth.AdminBootstrap.Password)
}
}
func TestLibraryEnvOverrides(t *testing.T) {
t.Setenv("SMARTMUSIC_LIBRARY_SCAN_PATHS", "/music:/other::/third")
t.Setenv("SMARTMUSIC_LIBRARY_SCAN_ON_STARTUP", "true")
cfg, err := Load("")
if err != nil {
t.Fatalf("Load: %v", err)
}
want := []string{"/music", "/other", "/third"}
if len(cfg.Library.ScanPaths) != len(want) {
t.Fatalf("scan_paths = %v, want %v", cfg.Library.ScanPaths, want)
}
for i, p := range want {
if cfg.Library.ScanPaths[i] != p {
t.Errorf("scan_paths[%d] = %q, want %q", i, cfg.Library.ScanPaths[i], p)
}
}
if !cfg.Library.ScanOnStartup {
t.Error("scan_on_startup = false, want true")
}
}
func TestLibraryYAMLLoads(t *testing.T) {
dir := t.TempDir()
path := filepath.Join(dir, "config.yaml")
content := []byte(`library:
scan_paths:
- /srv/music
- /mnt/archive
scan_on_startup: true
`)
if err := os.WriteFile(path, content, 0o644); err != nil {
t.Fatal(err)
}
cfg, err := Load(path)
if err != nil {
t.Fatalf("Load: %v", err)
}
if len(cfg.Library.ScanPaths) != 2 || cfg.Library.ScanPaths[0] != "/srv/music" {
t.Errorf("scan_paths = %v", cfg.Library.ScanPaths)
}
if !cfg.Library.ScanOnStartup {
t.Error("scan_on_startup = false")
}
}
+27
View File
@@ -11,6 +11,33 @@ import (
"github.com/jackc/pgx/v5/pgtype"
)
const getAlbumByArtistAndTitle = `-- name: GetAlbumByArtistAndTitle :one
SELECT id, title, sort_title, artist_id, release_date, mbid, cover_art_path, created_at, updated_at FROM albums WHERE artist_id = $1 AND title = $2 LIMIT 1
`
type GetAlbumByArtistAndTitleParams struct {
ArtistID pgtype.UUID
Title string
}
// Scanner uses this for the no-mbid dedupe path: resolve-or-create.
func (q *Queries) GetAlbumByArtistAndTitle(ctx context.Context, arg GetAlbumByArtistAndTitleParams) (Album, error) {
row := q.db.QueryRow(ctx, getAlbumByArtistAndTitle, arg.ArtistID, arg.Title)
var i Album
err := row.Scan(
&i.ID,
&i.Title,
&i.SortTitle,
&i.ArtistID,
&i.ReleaseDate,
&i.Mbid,
&i.CoverArtPath,
&i.CreatedAt,
&i.UpdatedAt,
)
return i, err
}
const getAlbumByID = `-- name: GetAlbumByID :one
SELECT id, title, sort_title, artist_id, release_date, mbid, cover_art_path, created_at, updated_at FROM albums WHERE id = $1
`
+4
View File
@@ -14,5 +14,9 @@ RETURNING *;
-- name: GetAlbumByID :one
SELECT * FROM albums WHERE id = $1;
-- name: GetAlbumByArtistAndTitle :one
-- Scanner uses this for the no-mbid dedupe path: resolve-or-create.
SELECT * FROM albums WHERE artist_id = $1 AND title = $2 LIMIT 1;
-- name: ListAlbumsByArtist :many
SELECT * FROM albums WHERE artist_id = $1 ORDER BY release_date NULLS LAST, sort_title;
+227
View File
@@ -0,0 +1,227 @@
package library
import (
"context"
"errors"
"fmt"
"io/fs"
"log/slog"
"os"
"path/filepath"
"strings"
"time"
"github.com/dhowden/tag"
"github.com/jackc/pgx/v5"
"github.com/jackc/pgx/v5/pgtype"
"github.com/jackc/pgx/v5/pgxpool"
"git.fabledsword.com/bvandeusen/minstrel/internal/db/dbq"
)
// audioExtensions is the v1 set. Duration extraction is not wired, so all of
// these get duration_ms=0 until an ffprobe / native-decoder pass lands.
var audioExtensions = map[string]bool{
".mp3": true,
".m4a": true,
".flac": true,
".ogg": true,
}
type Stats struct {
Scanned int `json:"scanned"`
Added int `json:"added"`
Updated int `json:"updated"`
Skipped int `json:"skipped"`
Errored int `json:"errored"`
}
type Scanner struct {
pool *pgxpool.Pool
logger *slog.Logger
paths []string
}
func New(pool *pgxpool.Pool, logger *slog.Logger, paths []string) *Scanner {
return &Scanner{pool: pool, logger: logger, paths: paths}
}
// Scan walks every configured root and upserts any audio file whose mtime is
// newer than the existing row's updated_at. Walk errors and per-file errors
// are logged + counted; the scan keeps going.
func (s *Scanner) Scan(ctx context.Context) (Stats, error) {
var stats Stats
q := dbq.New(s.pool)
start := time.Now()
for _, root := range s.paths {
if err := filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error {
if ctx.Err() != nil {
return fs.SkipAll
}
if err != nil {
s.logger.Warn("library scan walk error", "path", path, "err", err)
stats.Errored++
return nil
}
if d.IsDir() {
return nil
}
if !audioExtensions[strings.ToLower(filepath.Ext(path))] {
return nil
}
if err := s.scanFile(ctx, q, path, &stats); err != nil {
s.logger.Warn("library scan file error", "path", path, "err", err)
stats.Errored++
}
return nil
}); err != nil {
return stats, fmt.Errorf("library: walk %q: %w", root, err)
}
}
s.logger.Info("library scan complete",
"scanned", stats.Scanned,
"added", stats.Added,
"updated", stats.Updated,
"skipped", stats.Skipped,
"errored", stats.Errored,
"duration_ms", time.Since(start).Milliseconds(),
)
if err := ctx.Err(); err != nil {
return stats, err
}
return stats, nil
}
func (s *Scanner) scanFile(ctx context.Context, q *dbq.Queries, path string, stats *Stats) error {
stats.Scanned++
info, err := os.Stat(path)
if err != nil {
return fmt.Errorf("stat: %w", err)
}
mtime := info.ModTime()
existing, err := q.GetTrackByPath(ctx, path)
knownTrack := err == nil
if err != nil && !errors.Is(err, pgx.ErrNoRows) {
return fmt.Errorf("lookup: %w", err)
}
if knownTrack && !existing.UpdatedAt.Time.Before(mtime) {
stats.Skipped++
return nil
}
f, err := os.Open(path)
if err != nil {
return fmt.Errorf("open: %w", err)
}
defer func() { _ = f.Close() }()
meta, err := tag.ReadFrom(f)
if err != nil {
return fmt.Errorf("tag read: %w", err)
}
artistName := meta.Artist()
if artistName == "" {
artistName = "Unknown Artist"
}
albumTitle := meta.Album()
if albumTitle == "" {
albumTitle = "Unknown Album"
}
trackTitle := meta.Title()
if trackTitle == "" {
trackTitle = strings.TrimSuffix(filepath.Base(path), filepath.Ext(path))
}
artist, err := s.resolveArtist(ctx, q, artistName)
if err != nil {
return fmt.Errorf("artist: %w", err)
}
album, err := s.resolveAlbum(ctx, q, artist.ID, albumTitle, meta.Year())
if err != nil {
return fmt.Errorf("album: %w", err)
}
trackNum, _ := meta.Track()
discNum, _ := meta.Disc()
params := dbq.UpsertTrackParams{
Title: trackTitle,
AlbumID: album.ID,
ArtistID: artist.ID,
DurationMs: 0,
FilePath: path,
FileSize: info.Size(),
FileFormat: strings.TrimPrefix(strings.ToLower(filepath.Ext(path)), "."),
}
if trackNum > 0 {
v := int32(trackNum)
params.TrackNumber = &v
}
if discNum > 0 {
v := int32(discNum)
params.DiscNumber = &v
}
if g := meta.Genre(); g != "" {
params.Genre = &g
}
if _, err := q.UpsertTrack(ctx, params); err != nil {
return fmt.Errorf("upsert track: %w", err)
}
if knownTrack {
stats.Updated++
} else {
stats.Added++
}
return nil
}
func (s *Scanner) resolveArtist(ctx context.Context, q *dbq.Queries, name string) (dbq.Artist, error) {
existing, err := q.GetArtistByName(ctx, name)
if err == nil {
return existing, nil
}
if !errors.Is(err, pgx.ErrNoRows) {
return dbq.Artist{}, err
}
return q.UpsertArtist(ctx, dbq.UpsertArtistParams{
Name: name,
SortName: sortKey(name),
})
}
func (s *Scanner) resolveAlbum(ctx context.Context, q *dbq.Queries, artistID pgtype.UUID, title string, year int) (dbq.Album, error) {
existing, err := q.GetAlbumByArtistAndTitle(ctx, dbq.GetAlbumByArtistAndTitleParams{ArtistID: artistID, Title: title})
if err == nil {
return existing, nil
}
if !errors.Is(err, pgx.ErrNoRows) {
return dbq.Album{}, err
}
params := dbq.UpsertAlbumParams{
Title: title,
SortTitle: sortKey(title),
ArtistID: artistID,
}
if year > 0 {
params.ReleaseDate = pgtype.Date{
Time: time.Date(year, 1, 1, 0, 0, 0, 0, time.UTC),
Valid: true,
}
}
return q.UpsertAlbum(ctx, params)
}
// sortKey drops a leading "The " for sortable ordering. Non-English articles
// (Los, Die, Les) can be added if users ask — keeping the rule obvious for now.
func sortKey(s string) string {
if len(s) >= 4 && strings.EqualFold(s[:4], "the ") {
return s[4:]
}
return s
}
+132
View File
@@ -0,0 +1,132 @@
package library
import (
"bytes"
"context"
"encoding/binary"
"io"
"log/slog"
"os"
"path/filepath"
"testing"
"github.com/jackc/pgx/v5/pgxpool"
"git.fabledsword.com/bvandeusen/minstrel/internal/db"
"git.fabledsword.com/bvandeusen/minstrel/internal/db/dbq"
)
// TestScanner_Integration exercises walk → tag-parse → upsert → incremental
// skip against a real Postgres. Gated on MINSTREL_TEST_DATABASE_URL.
func TestScanner_Integration(t *testing.T) {
if testing.Short() {
t.Skip("skipping scanner integration in -short mode")
}
dsn := os.Getenv("MINSTREL_TEST_DATABASE_URL")
if dsn == "" {
t.Skip("MINSTREL_TEST_DATABASE_URL not set")
}
ctx := context.Background()
logger := slog.New(slog.NewTextHandler(io.Discard, nil))
if err := db.Migrate(dsn, logger); err != nil {
t.Fatalf("migrate: %v", err)
}
pool, err := pgxpool.New(ctx, dsn)
if err != nil {
t.Fatalf("pool: %v", err)
}
t.Cleanup(pool.Close)
if _, err := pool.Exec(ctx, "TRUNCATE tracks, albums, artists RESTART IDENTITY CASCADE"); err != nil {
t.Fatalf("truncate: %v", err)
}
root := t.TempDir()
writeTestMP3(t, filepath.Join(root, "artistX/albumA/01.mp3"), map[string]string{
"TIT2": "Track One", "TPE1": "Artist X", "TALB": "Album A", "TRCK": "1", "TYER": "2020",
})
writeTestMP3(t, filepath.Join(root, "artistX/albumA/02.mp3"), map[string]string{
"TIT2": "Track Two", "TPE1": "Artist X", "TALB": "Album A", "TRCK": "2", "TYER": "2020",
})
writeTestMP3(t, filepath.Join(root, "artistX/albumB/01.mp3"), map[string]string{
"TIT2": "B Song", "TPE1": "Artist X", "TALB": "Album B", "TRCK": "1",
})
writeTestMP3(t, filepath.Join(root, "artistY/01.mp3"), map[string]string{
"TIT2": "Solo", "TPE1": "The Artist Y", "TALB": "Y Album", "TRCK": "1",
})
scanner := New(pool, logger, []string{root})
stats, err := scanner.Scan(ctx)
if err != nil {
t.Fatalf("first scan: %v", err)
}
if stats.Scanned != 4 || stats.Added != 4 {
t.Errorf("first scan stats = %+v, want Scanned=4 Added=4", stats)
}
q := dbq.New(pool)
artists, err := q.ListArtists(ctx)
if err != nil {
t.Fatalf("ListArtists: %v", err)
}
if len(artists) != 2 {
t.Fatalf("artist rows = %d (%+v), want 2", len(artists), artists)
}
// "The Artist Y" should sort under "Artist Y" (article stripped).
if artists[0].SortName != "Artist X" || artists[1].SortName != "Artist Y" {
t.Errorf("artist sort = [%q, %q], want [Artist X, Artist Y]", artists[0].SortName, artists[1].SortName)
}
stats2, err := scanner.Scan(ctx)
if err != nil {
t.Fatalf("second scan: %v", err)
}
if stats2.Added != 0 || stats2.Updated != 0 || stats2.Skipped != 4 {
t.Errorf("incremental scan stats = %+v, want Skipped=4", stats2)
}
}
// writeTestMP3 emits a file with only an ID3v2.3 tag payload plus a few
// bytes of MPEG-sync trailer. dhowden/tag reads tags without decoding audio,
// so this is enough for scanner round-trip tests without shipping binaries.
func writeTestMP3(t *testing.T, path string, frames map[string]string) {
t.Helper()
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
t.Fatal(err)
}
var framesBuf bytes.Buffer
for id, value := range frames {
if len(id) != 4 {
t.Fatalf("frame id %q not 4 bytes", id)
}
payload := append([]byte{0x03}, []byte(value)...)
framesBuf.WriteString(id)
_ = binary.Write(&framesBuf, binary.BigEndian, uint32(len(payload)))
framesBuf.WriteByte(0x00)
framesBuf.WriteByte(0x00)
framesBuf.Write(payload)
}
total := framesBuf.Len()
header := []byte{
'I', 'D', '3', 0x03, 0x00, 0x00,
byte((total >> 21) & 0x7F),
byte((total >> 14) & 0x7F),
byte((total >> 7) & 0x7F),
byte(total & 0x7F),
}
f, err := os.Create(path)
if err != nil {
t.Fatal(err)
}
defer func() { _ = f.Close() }()
if _, err := f.Write(header); err != nil {
t.Fatal(err)
}
if _, err := f.Write(framesBuf.Bytes()); err != nil {
t.Fatal(err)
}
if _, err := f.Write([]byte{0xFF, 0xFB, 0x90, 0x00}); err != nil {
t.Fatal(err)
}
}
+44 -4
View File
@@ -1,20 +1,33 @@
package server
import (
"context"
"encoding/json"
"log/slog"
"net/http"
"github.com/go-chi/chi/v5"
"github.com/go-chi/chi/v5/middleware"
"github.com/jackc/pgx/v5/pgxpool"
"git.fabledsword.com/bvandeusen/minstrel/internal/auth"
"git.fabledsword.com/bvandeusen/minstrel/internal/library"
)
type Server struct {
Logger *slog.Logger
// ScanTrigger is the subset of the scanner the HTTP handler needs. Kept as an
// interface so tests can stub it without touching the DB.
type ScanTrigger interface {
Scan(ctx context.Context) (library.Stats, error)
}
func New(logger *slog.Logger) *Server {
return &Server{Logger: logger}
type Server struct {
Logger *slog.Logger
Pool *pgxpool.Pool
Scanner ScanTrigger
}
func New(logger *slog.Logger, pool *pgxpool.Pool, scanner ScanTrigger) *Server {
return &Server{Logger: logger, Pool: pool, Scanner: scanner}
}
func (s *Server) Router() http.Handler {
@@ -23,6 +36,17 @@ func (s *Server) Router() http.Handler {
r.Use(middleware.Recoverer)
r.Get("/healthz", s.handleHealthz)
if s.Pool != nil {
r.Route("/api", func(api chi.Router) {
api.Route("/admin", func(admin chi.Router) {
admin.Use(auth.RequireAdmin(s.Pool))
if s.Scanner != nil {
admin.Post("/scan", s.handleAdminScan)
}
})
})
}
return r
}
@@ -31,3 +55,19 @@ func (s *Server) handleHealthz(w http.ResponseWriter, _ *http.Request) {
w.WriteHeader(http.StatusOK)
_ = json.NewEncoder(w).Encode(map[string]string{"status": "ok"})
}
// handleAdminScan runs a scan synchronously and returns the resulting stats.
// Kept synchronous for v1: libraries are small and the operator can tell when
// it's done. Async jobs with status polling are a later-milestone concern.
func (s *Server) handleAdminScan(w http.ResponseWriter, r *http.Request) {
stats, err := s.Scanner.Scan(r.Context())
w.Header().Set("Content-Type", "application/json")
if err != nil {
s.Logger.Error("admin scan failed", "err", err)
w.WriteHeader(http.StatusInternalServerError)
_ = json.NewEncoder(w).Encode(map[string]any{"error": err.Error(), "stats": stats})
return
}
w.WriteHeader(http.StatusOK)
_ = json.NewEncoder(w).Encode(stats)
}
+1 -1
View File
@@ -10,7 +10,7 @@ import (
)
func TestHealthz(t *testing.T) {
s := New(slog.New(slog.NewTextHandler(io.Discard, nil)))
s := New(slog.New(slog.NewTextHandler(io.Discard, nil)), nil, nil)
ts := httptest.NewServer(s.Router())
defer ts.Close()