feat(library): adopt moved files instead of forking their history — #2528
Track identity was file_path, so a file that came back renamed or in a different directory looked like a deletion plus an unrelated new track: the old row kept the like and every play_event while a fresh zero-history row appeared, and nothing connected them. A liked song read as unliked, its play count reset, and Rediscover could offer it as a discovery — silently. Renumbering an album was enough, which is what happened to the operator's copy of Minutes to Midnight. Adoption re-points the existing row's file_path at the new location and clears its missing mark. The normal UpsertTrack then conflicts on file_path and updates THAT row, so the track id survives and likes, plays and playlist memberships travel with it — and clients see an update rather than a delete-and-create, so no cache churn either. Matching is MBID first (identifies the recording, so it survives a re-encode), then file_size + duration_ms for untagged files. Both fingerprint components must be non-zero: duration_ms is 0 when ffprobe failed, and matching 0 against 0 would pair up unrelated broken files. Only rows already marked missing are eligible — a row whose file is present elsewhere is a duplicate, not a move, and re-pointing it would corrupt the copy that still exists. An ambiguous match inserts fresh rather than adopting one arbitrarily: a fork is recoverable later, a wrong merge isn't. Scan is now three phases, and the order is the point. Adoption can only claim a row that is ALREADY marked missing, but reconcile previously ran after processing — so a rename performed while the server was down surfaced the deletion and the addition in the same scan, the new path inserted first, and the fork became permanent. Enumeration is therefore separated from processing so reconcile can run between them: walk (paths only, no tag reads or probes) -> reconcile -> process in walk order. Consequence worth knowing: when reconcile refuses (an absent root, or a reorganisation exceeding the 25% mark cap) adoption cannot fire and renamed files fork as before. That's the pre-#2528 behaviour rather than a new failure, and the warning now names it. The old outer walk-error branch was unreachable — the callback always returned nil, so WalkDir never surfaced an error — and verifyRootsPresent is the real protection, so enumerate counts walk errors instead of pretending to abort on them.
This commit is contained in:
+103
-49
@@ -94,59 +94,56 @@ func (s *Scanner) Scan(ctx context.Context, progressCb func(Stats)) (Stats, erro
|
||||
q := dbq.New(s.pool)
|
||||
start := time.Now()
|
||||
|
||||
// Every audio path the walk visited. Reconcile diffs this against the table,
|
||||
// so it costs no extra filesystem I/O — the walk already established which
|
||||
// files exist. ~100 bytes/path, so a 250k-track library is ~25MB, which is
|
||||
// worth it to avoid a second stat pass over the whole library.
|
||||
seen := make(map[string]struct{}, 8192)
|
||||
|
||||
for _, root := range s.paths {
|
||||
if err := filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error {
|
||||
if ctx.Err() != nil {
|
||||
return fs.SkipAll
|
||||
}
|
||||
if err != nil {
|
||||
s.logger.Warn("library scan walk error", "path", path, "err", err)
|
||||
stats.Errored++
|
||||
if progressCb != nil {
|
||||
progressCb(stats)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
if d.IsDir() {
|
||||
return nil
|
||||
}
|
||||
if !audioExtensions[strings.ToLower(filepath.Ext(path))] {
|
||||
return nil
|
||||
}
|
||||
// Recorded before scanFile so a file that exists but fails to parse
|
||||
// still counts as present. It's a broken file, not a missing one,
|
||||
// and marking it missing would hide it from the operator behind the
|
||||
// wrong explanation.
|
||||
seen[path] = struct{}{}
|
||||
if _, _, err := s.scanFile(ctx, q, path, &stats); err != nil {
|
||||
s.logger.Warn("library scan file error", "path", path, "err", err)
|
||||
stats.Errored++
|
||||
}
|
||||
if progressCb != nil {
|
||||
progressCb(stats)
|
||||
}
|
||||
return nil
|
||||
}); err != nil {
|
||||
return stats, fmt.Errorf("library: walk %q: %w", root, err)
|
||||
}
|
||||
}
|
||||
|
||||
// Reconcile only after a COMPLETE walk. A cancelled scan has a partial
|
||||
// `seen` set, which would mark everything it hadn't reached yet.
|
||||
// PHASE 1 — enumerate. Collect every audio path without touching tags or
|
||||
// ffprobe. Cheap: WalkDir already stats each entry, so this adds a directory
|
||||
// traversal and nothing else.
|
||||
//
|
||||
// The order matters and is the whole reason enumeration is separate.
|
||||
// Reconcile has to mark disappeared rows BEFORE any file is processed,
|
||||
// because move detection (#2528) can only adopt a row that is already marked
|
||||
// missing. A rename performed while the server was down surfaces the deletion
|
||||
// and the addition in the SAME scan — so if reconcile ran at the end, the new
|
||||
// path would insert a fresh row first and the fork would be permanent.
|
||||
paths, walkErrs := s.enumerate(ctx, progressCb, &stats)
|
||||
stats.Errored += walkErrs
|
||||
if err := ctx.Err(); err != nil {
|
||||
return stats, err
|
||||
}
|
||||
|
||||
// PHASE 2 — reconcile. Only ever on a COMPLETE enumeration: a cancelled walk
|
||||
// has a partial view and would mark everything it hadn't reached.
|
||||
seen := make(map[string]struct{}, len(paths))
|
||||
for _, p := range paths {
|
||||
seen[p] = struct{}{}
|
||||
}
|
||||
if err := s.reconcileMissing(ctx, q, seen, &stats); err != nil {
|
||||
// Not fatal: the walk's results are already persisted and useful. The
|
||||
// guards deliberately refuse to act on ambiguous evidence, and that
|
||||
// refusal arrives here as an error.
|
||||
s.logger.Warn("library scan: reconcile skipped", "err", err)
|
||||
// Not fatal. The guards deliberately refuse to act on ambiguous
|
||||
// evidence, and that refusal arrives here as an error.
|
||||
//
|
||||
// The consequence is named explicitly because it is not obvious: move
|
||||
// detection (#2528) can only adopt a row that is already marked missing,
|
||||
// so a refused reconcile also means renamed files insert fresh rows and
|
||||
// fork their history. That's the pre-#2528 behaviour rather than a new
|
||||
// failure, but it's worth knowing which scan it happened on. It bites
|
||||
// hardest when a large fraction of a small library is reorganised at
|
||||
// once, which trips the mark cap.
|
||||
s.logger.Warn("library scan: reconcile skipped — moved files will fork rather than adopt",
|
||||
"err", err)
|
||||
}
|
||||
|
||||
// PHASE 3 — process, in walk order so logs and cover-art batching stay
|
||||
// grouped by directory rather than following map iteration order.
|
||||
for _, path := range paths {
|
||||
if ctx.Err() != nil {
|
||||
break
|
||||
}
|
||||
if _, _, err := s.scanFile(ctx, q, path, &stats); err != nil {
|
||||
s.logger.Warn("library scan file error", "path", path, "err", err)
|
||||
stats.Errored++
|
||||
}
|
||||
if progressCb != nil {
|
||||
progressCb(stats)
|
||||
}
|
||||
}
|
||||
|
||||
s.logger.Info("library scan complete",
|
||||
@@ -165,6 +162,46 @@ func (s *Scanner) Scan(ctx context.Context, progressCb func(Stats)) (Stats, erro
|
||||
return stats, nil
|
||||
}
|
||||
|
||||
// enumerate walks every configured root and returns the audio paths found, in
|
||||
// walk order, plus a count of walk errors.
|
||||
//
|
||||
// A path is recorded even if it will later fail to parse: an unreadable file is a
|
||||
// broken file, not a missing one, and letting reconcile mark it missing would
|
||||
// hide it from the operator behind the wrong explanation.
|
||||
func (s *Scanner) enumerate(
|
||||
ctx context.Context, progressCb func(Stats), stats *Stats,
|
||||
) ([]string, int) {
|
||||
paths := make([]string, 0, 8192)
|
||||
errs := 0
|
||||
for _, root := range s.paths {
|
||||
// WalkDir's own error return is folded into the per-entry handler below,
|
||||
// so a bad root is counted rather than aborting the whole scan — one
|
||||
// unreadable root shouldn't discard the others' results.
|
||||
_ = filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error {
|
||||
if ctx.Err() != nil {
|
||||
return fs.SkipAll
|
||||
}
|
||||
if err != nil {
|
||||
s.logger.Warn("library scan walk error", "path", path, "err", err)
|
||||
errs++
|
||||
if progressCb != nil {
|
||||
progressCb(*stats)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
if d.IsDir() {
|
||||
return nil
|
||||
}
|
||||
if !audioExtensions[strings.ToLower(filepath.Ext(path))] {
|
||||
return nil
|
||||
}
|
||||
paths = append(paths, path)
|
||||
return nil
|
||||
})
|
||||
}
|
||||
return paths, errs
|
||||
}
|
||||
|
||||
// scanFile upserts a single audio file. Returns the album ID the track
|
||||
// belongs to and whether the file was added/updated (false = skipped as
|
||||
// unchanged), so watcher-driven callers can enrich just the changed albums.
|
||||
@@ -254,6 +291,23 @@ func (s *Scanner) scanFile(
|
||||
durationMs = probed
|
||||
}
|
||||
|
||||
// A path we've never seen might not be a new track — it might be one that
|
||||
// moved or was renamed (#2528). Adopting re-points the existing row at this
|
||||
// path and clears its missing mark, so the UpsertTrack below conflicts on
|
||||
// file_path and updates THAT row: same track id, likes and play history
|
||||
// intact. Without this, renumbering an album forks every track on it.
|
||||
//
|
||||
// Runs here rather than earlier because the fingerprint needs the probed
|
||||
// duration, and only for genuinely unknown paths — a known path is already
|
||||
// the row we're going to update.
|
||||
if !knownTrack {
|
||||
if s.adoptMovedTrack(ctx, q, path, info.Size(), durationMs, recordingMBID) {
|
||||
// Count it as an update: the row existed, and reporting it as Added
|
||||
// would overstate library growth on every reorganisation.
|
||||
knownTrack = true
|
||||
}
|
||||
}
|
||||
|
||||
params := dbq.UpsertTrackParams{
|
||||
Title: trackTitle,
|
||||
AlbumID: album.ID,
|
||||
|
||||
Reference in New Issue
Block a user