feat: Minstrel can read its irreplaceable data out as JSON Lines and load it back into an empty instance (#6770)

internal/backup names every table a backup holds, in parent-first order,
and every table it leaves out with the reason: what the server computes
from the music files, provider caches it fetches again, sign-ins and
one-time links, and logs of past runs. The library's identity rows travel
because every like, play and playlist entry hangs off a track id, and the
sync change feed travels because the apps' first sync reads it.

Sealed credentials are left out of the file, and so are the markers that
say tag and art enrichment are done, so a restored server asks for the
credentials again and redoes that work rather than believing it has it.

Snapshot reads one REPEATABLE READ, read-only transaction, each table in
primary-key order, keys sorted. Restore refuses an instance with accounts
or tracks and a backup from a newer schema, replaces seeded settings,
inserts by column intersection in batches, skips generated columns, and
carries serial sequences on past the restored ids.

Guards: every schema table is backed up or excluded; every backed-up
table comes after what it references and references nothing excluded;
every omitted column exists; a restore gives back the backup byte for byte.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

internal/backup/backup_test.go

@@ -0,0 +1,238 @@
+package backup_test
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"errors"
+	"io"
+	"log/slog"
+	"os"
+	"slices"
+	"testing"
+
+	"github.com/jackc/pgx/v5"
+	"github.com/jackc/pgx/v5/pgxpool"
+
+	"git.fabledsword.com/bvandeusen/minstrel/internal/backup"
+	"git.fabledsword.com/bvandeusen/minstrel/internal/db"
+	"git.fabledsword.com/bvandeusen/minstrel/internal/dbtest"
+)
+
+func newPool(t *testing.T) *pgxpool.Pool {
+	t.Helper()
+	if testing.Short() {
+		t.Skip("skipping integration test in -short mode")
+	}
+	dsn := os.Getenv("MINSTREL_TEST_DATABASE_URL")
+	if dsn == "" {
+		t.Skip("MINSTREL_TEST_DATABASE_URL not set")
+	}
+	if err := db.Migrate(dsn, slog.New(slog.NewTextHandler(io.Discard, nil))); err != nil {
+		t.Fatalf("migrate: %v", err)
+	}
+	pool, err := pgxpool.New(context.Background(), dsn)
+	if err != nil {
+		t.Fatalf("pool: %v", err)
+	}
+	t.Cleanup(pool.Close)
+	dbtest.ResetDB(t, pool)
+	return pool
+}
+
+func index(name string) int {
+	return slices.IndexFunc(backup.Tables, func(t backup.Table) bool { return t.Name == name })
+}
+
+// Every table in the schema is backed up or excluded with a reason, so a new
+// table fails here until someone decides which.
+func TestEveryTableIsAccountedFor_Integration(t *testing.T) {
+	pool := newPool(t)
+	rows, err := pool.Query(context.Background(), "SELECT tablename FROM pg_tables WHERE schemaname = 'public'")
+	if err != nil {
+		t.Fatal(err)
+	}
+	schema, err := pgx.CollectRows(rows, pgx.RowTo[string])
+	if err != nil {
+		t.Fatal(err)
+	}
+	for _, p := range backup.Coverage(schema) {
+		t.Error(p)
+	}
+}
+
+// A restore loads each table after the ones it references, and never needs a
+// table the backup leaves out.
+func TestTablesComeAfterWhatTheyReference_Integration(t *testing.T) {
+	pool := newPool(t)
+	rows, err := pool.Query(context.Background(), `
+		SELECT cl.relname, pc.relname
+		  FROM pg_constraint co
+		  JOIN pg_class cl ON cl.oid = co.conrelid
+		  JOIN pg_class pc ON pc.oid = co.confrelid
+		  JOIN pg_namespace n ON n.oid = cl.relnamespace
+		 WHERE co.contype = 'f' AND n.nspname = 'public'`)
+	if err != nil {
+		t.Fatal(err)
+	}
+	type fk struct{ child, parent string }
+	fks, err := pgx.CollectRows(rows, func(r pgx.CollectableRow) (fk, error) {
+		var f fk
+		err := r.Scan(&f.child, &f.parent)
+		return f, err
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(fks) == 0 {
+		t.Fatal("found no foreign keys; the query is wrong")
+	}
+	for _, f := range fks {
+		child, parent := index(f.child), index(f.parent)
+		if child < 0 {
+			continue
+		}
+		switch {
+		case parent < 0:
+			t.Errorf("%s is backed up but references %s, which is excluded", f.child, f.parent)
+		case parent >= child && f.parent != f.child:
+			t.Errorf("%s references %s, so it must come after it in Tables", f.child, f.parent)
+		}
+	}
+}
+
+// Every omitted column exists, so a renamed column fails here rather than a
+// sealed credential slipping into the file under its new name.
+func TestOmittedColumnsExist_Integration(t *testing.T) {
+	pool := newPool(t)
+	for _, tbl := range backup.Tables {
+		for _, col := range tbl.Omit {
+			var n int
+			if err := pool.QueryRow(context.Background(), `
+				SELECT count(*) FROM information_schema.columns
+				 WHERE table_schema = 'public' AND table_name = $1 AND column_name = $2`,
+				tbl.Name, col).Scan(&n); err != nil || n != 1 {
+				t.Errorf("%s.%s: found %d columns, err %v", tbl.Name, col, n, err)
+			}
+		}
+	}
+}
+
+func mustExec(t *testing.T, pool *pgxpool.Pool, sql string, args ...any) {
+	t.Helper()
+	if _, err := pool.Exec(context.Background(), sql, args...); err != nil {
+		t.Fatalf("%s: %v", sql, err)
+	}
+}
+
+func mustID(t *testing.T, pool *pgxpool.Pool, sql string, args ...any) string {
+	t.Helper()
+	var id string
+	if err := pool.QueryRow(context.Background(), sql, args...).Scan(&id); err != nil {
+		t.Fatalf("%s: %v", sql, err)
+	}
+	return id
+}
+
+// A backup restored into an emptied instance gives back the same rows, byte
+// for byte, without the credentials and markers it leaves out.
+func TestARestoredInstanceHoldsWhatTheBackupHeld_Integration(t *testing.T) {
+	pool := newPool(t)
+	ctx := context.Background()
+
+	user := mustID(t, pool, `INSERT INTO users (username, password_hash, api_token_hash, listenbrainz_token)
+		VALUES ($1, 'hash', 'backup-token', 'lb-token') RETURNING id::text`, dbtest.TestUserPrefix+"backup")
+	artist := mustID(t, pool, `INSERT INTO artists (name, sort_name, artist_art_source) VALUES ('Artist', 'artist', 'none') RETURNING id::text`)
+	album := mustID(t, pool, `INSERT INTO albums (title, sort_title, artist_id, cover_art_source, cover_art_path)
+		VALUES ('Album', 'album', $1, 'mbcaa', '/data/covers/a.jpg') RETURNING id::text`, artist)
+	track := mustID(t, pool, `INSERT INTO tracks (title, album_id, artist_id, duration_ms, file_path, file_size, file_format, tag_source)
+		VALUES ('Track', $1, $2, 180000, '/music/a.flac', 1000, 'flac', 'musicbrainz') RETURNING id::text`, album, artist)
+	session := mustID(t, pool, `INSERT INTO play_sessions (user_id, started_at, last_event_at) VALUES ($1, now(), now()) RETURNING id::text`, user)
+	mustExec(t, pool, `INSERT INTO play_events (user_id, track_id, session_id, started_at) VALUES ($1, $2, $3, now())`, user, track, session)
+	mustExec(t, pool, `INSERT INTO general_likes (user_id, track_id) VALUES ($1, $2)`, user, track)
+	playlist := mustID(t, pool, `INSERT INTO playlists (user_id, name) VALUES ($1, 'Mine') RETURNING id::text`, user)
+	mustExec(t, pool, `INSERT INTO playlist_tracks (playlist_id, position, track_id, title, artist_name, album_title, duration_sec)
+		VALUES ($1, 0, $2, 'Track', 'Artist', 'Album', 180)`, playlist, track)
+	mustExec(t, pool, `INSERT INTO follows (artist_mbid, artist_name, profile_id, followed_by)
+		SELECT 'mbid-1', 'Artist', id, $1 FROM release_type_profiles WHERE is_default`, user)
+	mustExec(t, pool, `INSERT INTO library_changes (entity_type, entity_id, op) VALUES ('track', $1, 'upsert')`, track)
+
+	first, err := backup.Snapshot(ctx, pool)
+	if err != nil {
+		t.Fatalf("snapshot: %v", err)
+	}
+	for _, tbl := range backup.Tables {
+		for _, line := range first.Tables[tbl.Name] {
+			var row map[string]json.RawMessage
+			if err := json.Unmarshal(line, &row); err != nil {
+				t.Fatalf("%s: %v", tbl.Name, err)
+			}
+			for _, col := range tbl.Omit {
+				if _, ok := row[col]; ok {
+					t.Errorf("%s.%s is in the backup", tbl.Name, col)
+				}
+			}
+		}
+	}
+	for _, tbl := range []string{"users", "tracks", "play_events", "general_likes", "playlist_tracks", "follows", "library_changes"} {
+		if len(first.Tables[tbl]) == 0 {
+			t.Errorf("the backup holds no %s", tbl)
+		}
+	}
+
+	if err := backup.Restore(ctx, pool, first); !errors.Is(err, backup.ErrNotEmpty) {
+		t.Fatalf("restore over data: err = %v, want ErrNotEmpty", err)
+	}
+	newer := first
+	newer.Schema++
+	if err := backup.Restore(ctx, pool, newer); !errors.Is(err, backup.ErrNewerSchema) {
+		t.Fatalf("restore from a newer schema: err = %v, want ErrNewerSchema", err)
+	}
+
+	mustExec(t, pool, "TRUNCATE users, artists, library_changes CASCADE")
+	if err := backup.Restore(ctx, pool, first); err != nil {
+		t.Fatalf("restore: %v", err)
+	}
+
+	second, err := backup.Snapshot(ctx, pool)
+	if err != nil {
+		t.Fatalf("snapshot after restore: %v", err)
+	}
+	for _, tbl := range backup.Tables {
+		a, b := first.Tables[tbl.Name], second.Tables[tbl.Name]
+		if len(a) != len(b) {
+			t.Errorf("%s: %d rows restored, the backup held %d", tbl.Name, len(b), len(a))
+			continue
+		}
+		for i := range a {
+			if !bytes.Equal(a[i], b[i]) {
+				t.Errorf("%s row %d:\nbacked up %s\nrestored  %s", tbl.Name, i+1, a[i], b[i])
+			}
+		}
+	}
+
+	// What the backup leaves out comes back unset, so the server asks for the
+	// credential again and redoes the art and tags.
+	var lbToken, coverSource, tagSource *string
+	if err := pool.QueryRow(ctx, `SELECT u.listenbrainz_token, al.cover_art_source, tr.tag_source
+		FROM users u, albums al, tracks tr WHERE u.id = $1 AND al.id = $2 AND tr.id = $3`,
+		user, album, track).Scan(&lbToken, &coverSource, &tagSource); err != nil {
+		t.Fatal(err)
+	}
+	if lbToken != nil || coverSource != nil || tagSource != nil {
+		t.Errorf("token %v, cover source %v, tag source %v; want all unset", lbToken, coverSource, tagSource)
+	}
+
+	// The change feed carries on after the restored ids.
+	var restoredMax, next int64
+	if err := pool.QueryRow(ctx, "SELECT max(id) FROM library_changes").Scan(&restoredMax); err != nil {
+		t.Fatal(err)
+	}
+	if err := pool.QueryRow(ctx, `INSERT INTO library_changes (entity_type, entity_id, op)
+		VALUES ('track', $1, 'upsert') RETURNING id`, track).Scan(&next); err != nil {
+		t.Fatal(err)
+	}
+	if next <= restoredMax {
+		t.Errorf("next change id %d, want it after the restored %d", next, restoredMax)
+	}
+}