package library import ( "context" "errors" "fmt" "io/fs" "log/slog" "math" "os" "os/exec" "path/filepath" "strconv" "strings" "time" "github.com/dhowden/tag" "github.com/jackc/pgx/v5" "github.com/jackc/pgx/v5/pgtype" "github.com/jackc/pgx/v5/pgxpool" "git.fabledsword.com/bvandeusen/minstrel/internal/db/dbq" syncpkg "git.fabledsword.com/bvandeusen/minstrel/internal/sync" ) // audioExtensions is the v1 set. Duration extraction is not wired, so all of // these get duration_ms=0 until an ffprobe / native-decoder pass lands. var audioExtensions = map[string]bool{ ".mp3": true, ".m4a": true, ".flac": true, ".ogg": true, } type Stats struct { Scanned int `json:"scanned"` Added int `json:"added"` Updated int `json:"updated"` Skipped int `json:"skipped"` Errored int `json:"errored"` } type Scanner struct { pool *pgxpool.Pool logger *slog.Logger paths []string } func New(pool *pgxpool.Pool, logger *slog.Logger, paths []string) *Scanner { return &Scanner{pool: pool, logger: logger, paths: paths} } // Scan walks every configured root and upserts any audio file whose mtime is // newer than the existing row's updated_at. Walk errors and per-file errors // are logged + counted; the scan keeps going. // // progressCb (may be nil) receives the current Stats snapshot after each // processed file. Used by the orchestrator to drive partial-tally writes. func (s *Scanner) Scan(ctx context.Context, progressCb func(Stats)) (Stats, error) { var stats Stats q := dbq.New(s.pool) start := time.Now() for _, root := range s.paths { if err := filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error { if ctx.Err() != nil { return fs.SkipAll } if err != nil { s.logger.Warn("library scan walk error", "path", path, "err", err) stats.Errored++ if progressCb != nil { progressCb(stats) } return nil } if d.IsDir() { return nil } if !audioExtensions[strings.ToLower(filepath.Ext(path))] { return nil } if err := s.scanFile(ctx, q, path, &stats); err != nil { s.logger.Warn("library scan file error", "path", path, "err", err) stats.Errored++ } if progressCb != nil { progressCb(stats) } return nil }); err != nil { return stats, fmt.Errorf("library: walk %q: %w", root, err) } } s.logger.Info("library scan complete", "scanned", stats.Scanned, "added", stats.Added, "updated", stats.Updated, "skipped", stats.Skipped, "errored", stats.Errored, "duration_ms", time.Since(start).Milliseconds(), ) if err := ctx.Err(); err != nil { return stats, err } return stats, nil } func (s *Scanner) scanFile(ctx context.Context, q *dbq.Queries, path string, stats *Stats) error { stats.Scanned++ info, err := os.Stat(path) if err != nil { return fmt.Errorf("stat: %w", err) } mtime := info.ModTime() existing, err := q.GetTrackByPath(ctx, path) knownTrack := err == nil if err != nil && !errors.Is(err, pgx.ErrNoRows) { return fmt.Errorf("lookup: %w", err) } // Incremental skip: only when the file hasn't changed AND we already have // a real duration. The second clause lets older scans that recorded // duration_ms=0 (before ffprobe was wired) get backfilled without forcing // the operator to wipe the library. Once duration is set, subsequent // scans short-circuit as before. if knownTrack && !existing.UpdatedAt.Time.Before(mtime) && existing.DurationMs > 0 { stats.Skipped++ return nil } f, err := os.Open(path) if err != nil { return fmt.Errorf("open: %w", err) } defer func() { _ = f.Close() }() meta, err := tag.ReadFrom(f) if err != nil { return fmt.Errorf("tag read: %w", err) } albumMBID, artistMBID := extractMBIDs(meta) recordingMBID := extractRecordingMBID(meta) artistName := meta.Artist() if artistName == "" { artistName = "Unknown Artist" } albumTitle := meta.Album() if albumTitle == "" { albumTitle = "Unknown Album" } trackTitle := meta.Title() if trackTitle == "" { trackTitle = strings.TrimSuffix(filepath.Base(path), filepath.Ext(path)) } artist, err := s.resolveArtist(ctx, q, artistName, artistMBID) if err != nil { return fmt.Errorf("artist: %w", err) } album, err := s.resolveAlbum(ctx, q, artist.ID, albumTitle, meta.Year(), albumMBID) if err != nil { return fmt.Errorf("album: %w", err) } trackNum, _ := meta.Track() discNum, _ := meta.Disc() durationMs, err := probeDurationMs(ctx, path) if err != nil { // Missing duration is degraded UX (clients can't scrub) but not a // blocker for ingestion. Record the file with 0ms; the next scan // will retry via the backfill clause in the skip check above. s.logger.Warn("library scan: ffprobe failed", "path", path, "err", err) durationMs = 0 } params := dbq.UpsertTrackParams{ Title: trackTitle, AlbumID: album.ID, ArtistID: artist.ID, DurationMs: durationMs, FilePath: path, FileSize: info.Size(), FileFormat: strings.TrimPrefix(strings.ToLower(filepath.Ext(path)), "."), } if trackNum > 0 { v := int32(trackNum) params.TrackNumber = &v } if discNum > 0 { v := int32(discNum) params.DiscNumber = &v } if g := meta.Genre(); g != "" { params.Genre = &g } // Recording MBID feeds the ListenBrainz similarity pipeline. // UpsertTrack heals mbid on the file_path conflict, so a re-scan // of a previously-untagged-into-DB track backfills it for free. if recordingMBID != "" { m := recordingMBID params.Mbid = &m } track, err := q.UpsertTrack(ctx, params) if err != nil { return fmt.Errorf("upsert track: %w", err) } if err := syncpkg.LogChange(ctx, s.pool, syncpkg.EntityTrack, syncpkg.FormatUUID(track.ID), syncpkg.OpUpsert); err != nil { // Best-effort: log but don't fail the scan. The next scan that // touches this track will re-emit the change. s.logger.Warn("library scan: LogChange track upsert failed", "track_id", track.ID, "err", err) } if knownTrack { stats.Updated++ } else { stats.Added++ } return nil } func (s *Scanner) resolveArtist(ctx context.Context, q *dbq.Queries, name, mbid string) (dbq.Artist, error) { existing, err := q.GetArtistByName(ctx, name) if err == nil { // Heal: backfill mbid on a previously-imported row if we have one now. if mbid != "" && (existing.Mbid == nil || *existing.Mbid == "") { m := mbid if uerr := q.SetArtistMbidIfNull(ctx, dbq.SetArtistMbidIfNullParams{ ID: existing.ID, Mbid: &m, }); uerr != nil { s.logger.Warn("library scan: heal artist mbid failed", "artist_id", existing.ID, "err", uerr) } else { existing.Mbid = &m } } return existing, nil } if !errors.Is(err, pgx.ErrNoRows) { return dbq.Artist{}, err } params := dbq.UpsertArtistParams{ Name: name, SortName: sortKey(name), } if mbid != "" { m := mbid params.Mbid = &m } artist, err := q.UpsertArtist(ctx, params) if err != nil { return dbq.Artist{}, err } if err := syncpkg.LogChange(ctx, s.pool, syncpkg.EntityArtist, syncpkg.FormatUUID(artist.ID), syncpkg.OpUpsert); err != nil { s.logger.Warn("library scan: LogChange artist upsert failed", "artist_id", artist.ID, "err", err) } return artist, nil } func (s *Scanner) resolveAlbum(ctx context.Context, q *dbq.Queries, artistID pgtype.UUID, title string, year int, mbid string) (dbq.Album, error) { existing, err := q.GetAlbumByArtistAndTitle(ctx, dbq.GetAlbumByArtistAndTitleParams{ArtistID: artistID, Title: title}) if err == nil { // Heal: backfill mbid on a previously-imported row if we have one now. if mbid != "" && (existing.Mbid == nil || *existing.Mbid == "") { m := mbid if uerr := q.SetAlbumMbidIfNull(ctx, dbq.SetAlbumMbidIfNullParams{ ID: existing.ID, Mbid: &m, }); uerr != nil { if isUniqueViolation(uerr) { // Another album row already owns this MBID — duplicate // release in the DB. Leave NULL; operator merges later. s.logger.Info("library scan: duplicate album mbid (canonical row already owns it)", "album_id", existing.ID, "mbid", mbid) } else { s.logger.Warn("library scan: heal album mbid failed", "album_id", existing.ID, "err", uerr) } } else { existing.Mbid = &m } } return existing, nil } if !errors.Is(err, pgx.ErrNoRows) { return dbq.Album{}, err } params := dbq.UpsertAlbumParams{ Title: title, SortTitle: sortKey(title), ArtistID: artistID, } if d, ok := releaseDateFromYear(year); ok { params.ReleaseDate = d } else if year != 0 { // year=0 is the no-tag case; anything else getting rejected is a tag // we couldn't trust (typo, OOB number, …). Log it so users can chase // down the file but don't fail the album insert over a soft field. s.logger.Warn("library scan: dropping invalid release year", "year", year, "album", title) } if mbid != "" { m := mbid params.Mbid = &m } album, err := q.UpsertAlbum(ctx, params) if err != nil { return dbq.Album{}, err } if err := syncpkg.LogChange(ctx, s.pool, syncpkg.EntityAlbum, syncpkg.FormatUUID(album.ID), syncpkg.OpUpsert); err != nil { s.logger.Warn("library scan: LogChange album upsert failed", "album_id", album.ID, "err", err) } return album, nil } // releaseDateFromYear converts a tag-supplied year into a Postgres date, // returning ok=false if the year is outside what we'll accept. We're // deliberately strict (1..9999): Postgres' date type goes much wider, but // 5-digit years from ID3 tags are always garbage in practice and trigger // SQLSTATE 22008 (datetime field overflow) at insert time. func releaseDateFromYear(year int) (pgtype.Date, bool) { if year < 1 || year > 9999 { return pgtype.Date{}, false } return pgtype.Date{ Time: time.Date(year, 1, 1, 0, 0, 0, 0, time.UTC), Valid: true, }, true } // sortKey drops a leading "The " for sortable ordering. Non-English articles // (Los, Die, Les) can be added if users ask — keeping the rule obvious for now. func sortKey(s string) string { if len(s) >= 4 && strings.EqualFold(s[:4], "the ") { return s[4:] } return s } // probeTimeout bounds how long ffprobe is allowed to inspect a single file. // 10s is generous for local mp3/flac — hits are usually <100ms — but caps // blast radius if a pathological file or a slow network mount stalls a scan. const probeTimeout = 10 * time.Second // probeDurationMs shells out to ffprobe to extract a track's duration. We // rely on ffmpeg being in the image (see Dockerfile). The CLI is slow per // call (fork+exec) but scans are batch-mode; this is simpler than pulling // a Go-side decoder library and handles every format ffmpeg does. func probeDurationMs(ctx context.Context, path string) (int32, error) { probeCtx, cancel := context.WithTimeout(ctx, probeTimeout) defer cancel() cmd := exec.CommandContext(probeCtx, "ffprobe", "-v", "error", "-show_entries", "format=duration", "-of", "default=noprint_wrappers=1:nokey=1", path, ) out, err := cmd.Output() if err != nil { return 0, fmt.Errorf("ffprobe: %w", err) } seconds, err := strconv.ParseFloat(strings.TrimSpace(string(out)), 64) if err != nil { return 0, fmt.Errorf("parse ffprobe output %q: %w", out, err) } if seconds <= 0 || math.IsNaN(seconds) || math.IsInf(seconds, 0) { return 0, fmt.Errorf("invalid duration: %v", seconds) } ms := seconds * 1000 if ms > math.MaxInt32 { ms = math.MaxInt32 } return int32(ms), nil }