Nothing in Minstrel ever noticed a deleted file. The walk only visits paths that exist, so a row whose file was gone was never scanned, never errored, never counted — permanently invisible. classifyEvent ignores fsnotify removals by design, and the safety-net scan is the same walk, so it covers additions only. Rows accumulated forever. Found on the operator's library: a completed scan reported skipped=24185 errored=0 while the MBID backfill (which opens files by DB path rather than walking) logged ~40 "no such file or directory" across three reorganised albums. Those rows also kept their pre-#2499 welded genre, which is how this surfaced — the version-stamped tag re-read can only reach files the walk visits. The harm is not cosmetic. tracks is the candidate universe for recommendation.sql / discover.sql / system_mixes.sql and nothing filtered on file existence, so a mix could spend a slot on a track that cannot stream. Marks rather than deletes. A missing file is a claim about the filesystem and the filesystem lies transiently — an unmounted volume, a network blip, a container that started before its media mount attached. Every sweep in internal/gc resolves a truth INSIDE the database and is safe to run blind; this one is not, so no deletion happens here. Three guards refuse to act on ambiguous evidence: every scan root must resolve to a non-empty directory, the walk must have seen at least one file, and one reconcile may newly mark at most 25% of the library. Clearing a mark is never the dangerous direction, so it runs unconditionally — otherwise a library that tripped the cap could never recover once the mount returned. Only a full Scan reconciles. The walk's set of seen paths is the evidence, and ScanFiles has no basis for concluding anything about files it did not look at. Excludes marked tracks from all 13 track-emitting queries (radio x2, system mixes x5, discover x4, most-played x2), the 6 play-history seed picks, and the genre browse axis. Deliberately NOT filtered: the shared ListPlaylistTracks read path, because it also serves user-curated playlists where hiding a track the user added would be wrong — system playlists shed orphans on their next daily rebuild instead. History and the taste profile also keep them: those record the past, and a track you played 200 times still says something about your taste. Reconcile tallies land in scan_runs so a disappearance is visible rather than discovered when a mix comes up short.
327 lines
10 KiB
Go
327 lines
10 KiB
Go
package library
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"fmt"
|
|
"io"
|
|
"log/slog"
|
|
"os"
|
|
"path/filepath"
|
|
"testing"
|
|
|
|
"github.com/jackc/pgx/v5/pgtype"
|
|
|
|
"git.fabledsword.com/bvandeusen/minstrel/internal/db/dbq"
|
|
)
|
|
|
|
// fakeReconciler records what reconcileMissing decided to do, so the guards can
|
|
// be tested without a database. The guards are the whole point of this pass —
|
|
// they are what stands between an unmounted volume and the library disappearing
|
|
// from every mix — so they get tested directly rather than via integration.
|
|
type fakeReconciler struct {
|
|
rows []dbq.ListTrackPathsForReconcileRow
|
|
marked []pgtype.UUID
|
|
cleared []pgtype.UUID
|
|
listErr error
|
|
markErr error
|
|
clearErr error
|
|
}
|
|
|
|
func (f *fakeReconciler) ListTrackPathsForReconcile(context.Context) ([]dbq.ListTrackPathsForReconcileRow, error) {
|
|
return f.rows, f.listErr
|
|
}
|
|
|
|
func (f *fakeReconciler) MarkTracksMissing(_ context.Context, ids []pgtype.UUID) (int64, error) {
|
|
if f.markErr != nil {
|
|
return 0, f.markErr
|
|
}
|
|
f.marked = append(f.marked, ids...)
|
|
return int64(len(ids)), nil
|
|
}
|
|
|
|
func (f *fakeReconciler) ClearTracksMissing(_ context.Context, ids []pgtype.UUID) (int64, error) {
|
|
if f.clearErr != nil {
|
|
return 0, f.clearErr
|
|
}
|
|
f.cleared = append(f.cleared, ids...)
|
|
return int64(len(ids)), nil
|
|
}
|
|
|
|
// Compile-time proof the real queries still satisfy what reconcile needs — the
|
|
// interface exists to narrow dbq.Queries, not to diverge from it.
|
|
var _ trackReconciler = (*dbq.Queries)(nil)
|
|
|
|
func testUUID(n byte) pgtype.UUID {
|
|
var u pgtype.UUID
|
|
u.Bytes[15] = n
|
|
u.Valid = true
|
|
return u
|
|
}
|
|
|
|
func markedAt() pgtype.Timestamptz {
|
|
return pgtype.Timestamptz{Valid: true}
|
|
}
|
|
|
|
func row(n byte, path string, missing bool) dbq.ListTrackPathsForReconcileRow {
|
|
r := dbq.ListTrackPathsForReconcileRow{ID: testUUID(n), FilePath: path}
|
|
if missing {
|
|
r.MissingSince = markedAt()
|
|
}
|
|
return r
|
|
}
|
|
|
|
// populatedRoot returns a directory containing one file, so verifyRootsPresent
|
|
// treats it as a real, mounted library root.
|
|
func populatedRoot(t *testing.T) string {
|
|
t.Helper()
|
|
dir := t.TempDir()
|
|
if err := os.WriteFile(filepath.Join(dir, "a.mp3"), []byte("x"), 0o600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return dir
|
|
}
|
|
|
|
func testScanner(t *testing.T, roots ...string) *Scanner {
|
|
t.Helper()
|
|
return &Scanner{
|
|
logger: slog.New(slog.NewTextHandler(io.Discard, nil)),
|
|
paths: roots,
|
|
}
|
|
}
|
|
|
|
func TestReconcileMissing_MarksRowsTheWalkDidNotSee(t *testing.T) {
|
|
root := populatedRoot(t)
|
|
s := testScanner(t, root)
|
|
|
|
// 10 rows with 2 absent — 20%, deliberately under missingMarkMaxFraction so
|
|
// this exercises marking rather than the cap. (An earlier version of this
|
|
// test used 2-of-4 and was really testing the guard by accident.)
|
|
rows := make([]dbq.ListTrackPathsForReconcileRow, 0, 10)
|
|
seen := map[string]struct{}{}
|
|
for i := 0; i < 10; i++ {
|
|
p := fmt.Sprintf("/music/track-%02d.mp3", i)
|
|
rows = append(rows, row(byte(i), p, false))
|
|
if i >= 2 {
|
|
seen[p] = struct{}{}
|
|
}
|
|
}
|
|
q := &fakeReconciler{rows: rows}
|
|
|
|
var stats Stats
|
|
if err := s.reconcileMissing(context.Background(), q, seen, &stats); err != nil {
|
|
t.Fatalf("reconcile: %v", err)
|
|
}
|
|
if len(q.marked) != 2 {
|
|
t.Fatalf("marked %d rows, want 2", len(q.marked))
|
|
}
|
|
if q.marked[0] != testUUID(0) || q.marked[1] != testUUID(1) {
|
|
t.Errorf("marked the wrong rows: %v", q.marked)
|
|
}
|
|
if stats.Missing != 2 {
|
|
t.Errorf("stats.Missing = %d, want 2", stats.Missing)
|
|
}
|
|
if len(q.cleared) != 0 {
|
|
t.Errorf("cleared %d rows, want 0", len(q.cleared))
|
|
}
|
|
}
|
|
|
|
func TestReconcileMissing_ClearsRowsWhoseFileReturned(t *testing.T) {
|
|
root := populatedRoot(t)
|
|
s := testScanner(t, root)
|
|
q := &fakeReconciler{rows: []dbq.ListTrackPathsForReconcileRow{
|
|
row(1, "/music/back.mp3", true),
|
|
row(2, "/music/still-here.mp3", false),
|
|
}}
|
|
seen := map[string]struct{}{
|
|
"/music/back.mp3": {},
|
|
"/music/still-here.mp3": {},
|
|
}
|
|
|
|
var stats Stats
|
|
if err := s.reconcileMissing(context.Background(), q, seen, &stats); err != nil {
|
|
t.Fatalf("reconcile: %v", err)
|
|
}
|
|
if len(q.cleared) != 1 || q.cleared[0] != testUUID(1) {
|
|
t.Fatalf("cleared = %v, want just row 1", q.cleared)
|
|
}
|
|
if stats.Restored != 1 {
|
|
t.Errorf("stats.Restored = %d, want 1", stats.Restored)
|
|
}
|
|
if len(q.marked) != 0 {
|
|
t.Errorf("marked %d rows, want 0", len(q.marked))
|
|
}
|
|
}
|
|
|
|
// An already-marked row must not be re-marked: the timestamp is the "how long
|
|
// has this been gone" clock that any future cleanup policy depends on.
|
|
func TestReconcileMissing_DoesNotRemarkAlreadyMissingRows(t *testing.T) {
|
|
root := populatedRoot(t)
|
|
s := testScanner(t, root)
|
|
q := &fakeReconciler{rows: []dbq.ListTrackPathsForReconcileRow{
|
|
row(1, "/music/long-gone.mp3", true),
|
|
row(2, "/music/present.mp3", false),
|
|
}}
|
|
seen := map[string]struct{}{"/music/present.mp3": {}}
|
|
|
|
var stats Stats
|
|
if err := s.reconcileMissing(context.Background(), q, seen, &stats); err != nil {
|
|
t.Fatalf("reconcile: %v", err)
|
|
}
|
|
if len(q.marked) != 0 {
|
|
t.Errorf("re-marked an already-missing row: %v", q.marked)
|
|
}
|
|
if len(q.cleared) != 0 {
|
|
t.Errorf("cleared = %v, want none", q.cleared)
|
|
}
|
|
}
|
|
|
|
// The guard that matters most. A half-attached mount makes the walk succeed
|
|
// while seeing only part of the library — evidence indistinguishable from a mass
|
|
// deletion, so reconcile must refuse rather than guess.
|
|
func TestReconcileMissing_RefusesWhenTooMuchWouldBeMarked(t *testing.T) {
|
|
root := populatedRoot(t)
|
|
s := testScanner(t, root)
|
|
rows := make([]dbq.ListTrackPathsForReconcileRow, 0, 100)
|
|
seen := map[string]struct{}{}
|
|
for i := 0; i < 100; i++ {
|
|
p := fmt.Sprintf("/music/track-%03d.mp3", i)
|
|
rows = append(rows, row(byte(i), p, false))
|
|
// Only 60 of 100 present -> 40% would be marked, over the 25% cap.
|
|
if i < 60 {
|
|
seen[p] = struct{}{}
|
|
}
|
|
}
|
|
q := &fakeReconciler{rows: rows}
|
|
|
|
var stats Stats
|
|
err := s.reconcileMissing(context.Background(), q, seen, &stats)
|
|
if err == nil {
|
|
t.Fatal("expected reconcile to refuse, got nil error")
|
|
}
|
|
if len(q.marked) != 0 {
|
|
t.Errorf("marked %d rows despite refusing", len(q.marked))
|
|
}
|
|
if stats.Missing != 0 {
|
|
t.Errorf("stats.Missing = %d, want 0", stats.Missing)
|
|
}
|
|
}
|
|
|
|
// Restoring is never the dangerous direction, so it must survive the cap —
|
|
// otherwise a library that tripped the cap once could never clear its marks
|
|
// even after the volume came back.
|
|
func TestReconcileMissing_ClearsEvenWhenMarkCapTrips(t *testing.T) {
|
|
root := populatedRoot(t)
|
|
s := testScanner(t, root)
|
|
rows := []dbq.ListTrackPathsForReconcileRow{row(1, "/music/back.mp3", true)}
|
|
seen := map[string]struct{}{"/music/back.mp3": {}}
|
|
// Add enough absent rows to blow the cap.
|
|
for i := 2; i < 10; i++ {
|
|
rows = append(rows, row(byte(i), fmt.Sprintf("/music/absent-%02d.mp3", i), false))
|
|
}
|
|
q := &fakeReconciler{rows: rows}
|
|
|
|
var stats Stats
|
|
if err := s.reconcileMissing(context.Background(), q, seen, &stats); err == nil {
|
|
t.Fatal("expected the mark cap to trip")
|
|
}
|
|
if len(q.cleared) != 1 {
|
|
t.Errorf("cleared %d rows, want 1 — restores must not be blocked by the cap", len(q.cleared))
|
|
}
|
|
if stats.Restored != 1 {
|
|
t.Errorf("stats.Restored = %d, want 1", stats.Restored)
|
|
}
|
|
}
|
|
|
|
func TestReconcileMissing_RefusesOnEmptyWalk(t *testing.T) {
|
|
root := populatedRoot(t)
|
|
s := testScanner(t, root)
|
|
q := &fakeReconciler{rows: []dbq.ListTrackPathsForReconcileRow{
|
|
row(1, "/music/a.mp3", false),
|
|
}}
|
|
|
|
var stats Stats
|
|
if err := s.reconcileMissing(context.Background(), q, map[string]struct{}{}, &stats); err == nil {
|
|
t.Fatal("expected refusal when the walk saw no files")
|
|
}
|
|
if len(q.marked) != 0 {
|
|
t.Errorf("marked rows on an empty walk: %v", q.marked)
|
|
}
|
|
}
|
|
|
|
// The unmounted-volume case: the configured root doesn't exist at all.
|
|
func TestReconcileMissing_RefusesWhenRootMissing(t *testing.T) {
|
|
s := testScanner(t, filepath.Join(t.TempDir(), "not-mounted"))
|
|
q := &fakeReconciler{rows: []dbq.ListTrackPathsForReconcileRow{
|
|
row(1, "/music/a.mp3", false),
|
|
}}
|
|
|
|
var stats Stats
|
|
if err := s.reconcileMissing(context.Background(), q, map[string]struct{}{"/x": {}}, &stats); err == nil {
|
|
t.Fatal("expected refusal when a scan root is absent")
|
|
}
|
|
if len(q.marked) != 0 {
|
|
t.Errorf("marked rows with an absent root: %v", q.marked)
|
|
}
|
|
}
|
|
|
|
// A mount point that exists but has nothing mounted on it: os.Stat succeeds on
|
|
// the bare directory, which is why emptiness is checked separately.
|
|
func TestReconcileMissing_RefusesWhenRootEmpty(t *testing.T) {
|
|
s := testScanner(t, t.TempDir())
|
|
q := &fakeReconciler{rows: []dbq.ListTrackPathsForReconcileRow{
|
|
row(1, "/music/a.mp3", false),
|
|
}}
|
|
|
|
var stats Stats
|
|
if err := s.reconcileMissing(context.Background(), q, map[string]struct{}{"/x": {}}, &stats); err == nil {
|
|
t.Fatal("expected refusal when a scan root is empty")
|
|
}
|
|
}
|
|
|
|
// Several roots, one detached. Marking must not proceed on partial evidence just
|
|
// because the other roots looked fine.
|
|
func TestReconcileMissing_RefusesWhenAnyRootMissing(t *testing.T) {
|
|
good := populatedRoot(t)
|
|
s := testScanner(t, good, filepath.Join(t.TempDir(), "detached"))
|
|
q := &fakeReconciler{rows: []dbq.ListTrackPathsForReconcileRow{
|
|
row(1, "/music/a.mp3", false),
|
|
}}
|
|
|
|
var stats Stats
|
|
if err := s.reconcileMissing(context.Background(), q, map[string]struct{}{"/x": {}}, &stats); err == nil {
|
|
t.Fatal("expected refusal when one of several roots is absent")
|
|
}
|
|
}
|
|
|
|
func TestReconcileMissing_NoRowsIsNotAnError(t *testing.T) {
|
|
root := populatedRoot(t)
|
|
s := testScanner(t, root)
|
|
q := &fakeReconciler{}
|
|
|
|
var stats Stats
|
|
if err := s.reconcileMissing(context.Background(), q, map[string]struct{}{"/x": {}}, &stats); err != nil {
|
|
t.Fatalf("empty library should reconcile cleanly, got %v", err)
|
|
}
|
|
}
|
|
|
|
func TestReconcileMissing_PropagatesListError(t *testing.T) {
|
|
root := populatedRoot(t)
|
|
s := testScanner(t, root)
|
|
sentinel := errors.New("boom")
|
|
q := &fakeReconciler{listErr: sentinel}
|
|
|
|
var stats Stats
|
|
err := s.reconcileMissing(context.Background(), q, map[string]struct{}{"/x": {}}, &stats)
|
|
if !errors.Is(err, sentinel) {
|
|
t.Fatalf("err = %v, want it to wrap %v", err, sentinel)
|
|
}
|
|
}
|
|
|
|
func TestVerifyRootsPresent_NoRootsConfigured(t *testing.T) {
|
|
s := testScanner(t)
|
|
if err := s.verifyRootsPresent(); err == nil {
|
|
t.Fatal("expected an error with no scan roots configured")
|
|
}
|
|
}
|