package backup_test import ( "bytes" "context" "encoding/json" "errors" "io" "log/slog" "os" "slices" "testing" "github.com/jackc/pgx/v5" "github.com/jackc/pgx/v5/pgxpool" "git.fabledsword.com/bvandeusen/minstrel/internal/backup" "git.fabledsword.com/bvandeusen/minstrel/internal/db" "git.fabledsword.com/bvandeusen/minstrel/internal/dbtest" ) func newPool(t *testing.T) *pgxpool.Pool { t.Helper() if testing.Short() { t.Skip("skipping integration test in -short mode") } dsn := os.Getenv("MINSTREL_TEST_DATABASE_URL") if dsn == "" { t.Skip("MINSTREL_TEST_DATABASE_URL not set") } if err := db.Migrate(dsn, slog.New(slog.NewTextHandler(io.Discard, nil))); err != nil { t.Fatalf("migrate: %v", err) } pool, err := pgxpool.New(context.Background(), dsn) if err != nil { t.Fatalf("pool: %v", err) } t.Cleanup(pool.Close) dbtest.ResetDB(t, pool) return pool } func index(name string) int { return slices.IndexFunc(backup.Tables, func(t backup.Table) bool { return t.Name == name }) } // Every table in the schema is backed up or excluded with a reason, so a new // table fails here until someone decides which. func TestEveryTableIsAccountedFor_Integration(t *testing.T) { pool := newPool(t) rows, err := pool.Query(context.Background(), "SELECT tablename FROM pg_tables WHERE schemaname = 'public'") if err != nil { t.Fatal(err) } schema, err := pgx.CollectRows(rows, pgx.RowTo[string]) if err != nil { t.Fatal(err) } for _, p := range backup.Coverage(schema) { t.Error(p) } } // A restore loads each table after the ones it references, and never needs a // table the backup leaves out. func TestTablesComeAfterWhatTheyReference_Integration(t *testing.T) { pool := newPool(t) rows, err := pool.Query(context.Background(), ` SELECT cl.relname, pc.relname FROM pg_constraint co JOIN pg_class cl ON cl.oid = co.conrelid JOIN pg_class pc ON pc.oid = co.confrelid JOIN pg_namespace n ON n.oid = cl.relnamespace WHERE co.contype = 'f' AND n.nspname = 'public'`) if err != nil { t.Fatal(err) } type fk struct{ child, parent string } fks, err := pgx.CollectRows(rows, func(r pgx.CollectableRow) (fk, error) { var f fk err := r.Scan(&f.child, &f.parent) return f, err }) if err != nil { t.Fatal(err) } if len(fks) == 0 { t.Fatal("found no foreign keys; the query is wrong") } for _, f := range fks { child, parent := index(f.child), index(f.parent) if child < 0 { continue } switch { case parent < 0: t.Errorf("%s is backed up but references %s, which is excluded", f.child, f.parent) case parent >= child && f.parent != f.child: t.Errorf("%s references %s, so it must come after it in Tables", f.child, f.parent) } } } // Every omitted column exists, so a renamed column fails here rather than a // sealed credential slipping into the file under its new name. func TestOmittedColumnsExist_Integration(t *testing.T) { pool := newPool(t) for _, tbl := range backup.Tables { for _, col := range tbl.Omit { var n int if err := pool.QueryRow(context.Background(), ` SELECT count(*) FROM information_schema.columns WHERE table_schema = 'public' AND table_name = $1 AND column_name = $2`, tbl.Name, col).Scan(&n); err != nil || n != 1 { t.Errorf("%s.%s: found %d columns, err %v", tbl.Name, col, n, err) } } } } func mustExec(t *testing.T, pool *pgxpool.Pool, sql string, args ...any) { t.Helper() if _, err := pool.Exec(context.Background(), sql, args...); err != nil { t.Fatalf("%s: %v", sql, err) } } func mustID(t *testing.T, pool *pgxpool.Pool, sql string, args ...any) string { t.Helper() var id string if err := pool.QueryRow(context.Background(), sql, args...).Scan(&id); err != nil { t.Fatalf("%s: %v", sql, err) } return id } // A backup restored into an emptied instance gives back the same rows, byte // for byte, without the credentials and markers it leaves out. func TestARestoredInstanceHoldsWhatTheBackupHeld_Integration(t *testing.T) { pool := newPool(t) ctx := context.Background() user := mustID(t, pool, `INSERT INTO users (username, password_hash, api_token_hash, listenbrainz_token) VALUES ($1, 'hash', 'backup-token', 'lb-token') RETURNING id::text`, dbtest.TestUserPrefix+"backup") artist := mustID(t, pool, `INSERT INTO artists (name, sort_name, artist_art_source) VALUES ('Artist', 'artist', 'none') RETURNING id::text`) album := mustID(t, pool, `INSERT INTO albums (title, sort_title, artist_id, cover_art_source, cover_art_path) VALUES ('Album', 'album', $1, 'mbcaa', '/data/covers/a.jpg') RETURNING id::text`, artist) track := mustID(t, pool, `INSERT INTO tracks (title, album_id, artist_id, duration_ms, file_path, file_size, file_format, tag_source) VALUES ('Track', $1, $2, 180000, '/music/a.flac', 1000, 'flac', 'musicbrainz') RETURNING id::text`, album, artist) session := mustID(t, pool, `INSERT INTO play_sessions (user_id, started_at, last_event_at) VALUES ($1, now(), now()) RETURNING id::text`, user) mustExec(t, pool, `INSERT INTO play_events (user_id, track_id, session_id, started_at) VALUES ($1, $2, $3, now())`, user, track, session) mustExec(t, pool, `INSERT INTO general_likes (user_id, track_id) VALUES ($1, $2)`, user, track) playlist := mustID(t, pool, `INSERT INTO playlists (user_id, name) VALUES ($1, 'Mine') RETURNING id::text`, user) mustExec(t, pool, `INSERT INTO playlist_tracks (playlist_id, position, track_id, title, artist_name, album_title, duration_sec) VALUES ($1, 0, $2, 'Track', 'Artist', 'Album', 180)`, playlist, track) mustExec(t, pool, `INSERT INTO follows (artist_mbid, artist_name, profile_id, followed_by) SELECT 'mbid-1', 'Artist', id, $1 FROM release_type_profiles WHERE is_default`, user) mustExec(t, pool, `INSERT INTO library_changes (entity_type, entity_id, op) VALUES ('track', $1, 'upsert')`, track) first, err := backup.Snapshot(ctx, pool) if err != nil { t.Fatalf("snapshot: %v", err) } for _, tbl := range backup.Tables { for _, line := range first.Tables[tbl.Name] { var row map[string]json.RawMessage if err := json.Unmarshal(line, &row); err != nil { t.Fatalf("%s: %v", tbl.Name, err) } for _, col := range tbl.Omit { if _, ok := row[col]; ok { t.Errorf("%s.%s is in the backup", tbl.Name, col) } } } } for _, tbl := range []string{"users", "tracks", "play_events", "general_likes", "playlist_tracks", "follows", "library_changes"} { if len(first.Tables[tbl]) == 0 { t.Errorf("the backup holds no %s", tbl) } } if err := backup.Restore(ctx, pool, first); !errors.Is(err, backup.ErrNotEmpty) { t.Fatalf("restore over data: err = %v, want ErrNotEmpty", err) } newer := first newer.Schema++ if err := backup.Restore(ctx, pool, newer); !errors.Is(err, backup.ErrNewerSchema) { t.Fatalf("restore from a newer schema: err = %v, want ErrNewerSchema", err) } // A new server scans its music before anyone signs up, so it already has // a library, here with a track the backup doesn't know. It still takes // the backup: the backup's library replaces what the scan found, which the // next scan finds again under the backup's ids. mustExec(t, pool, "TRUNCATE users CASCADE") scannedArtist := mustID(t, pool, `INSERT INTO artists (name, sort_name) VALUES ('Scanned', 'scanned') RETURNING id::text`) scannedAlbum := mustID(t, pool, `INSERT INTO albums (title, sort_title, artist_id) VALUES ('Scanned', 'scanned', $1) RETURNING id::text`, scannedArtist) scanned := mustID(t, pool, `INSERT INTO tracks (title, album_id, artist_id, duration_ms, file_path, file_size, file_format) VALUES ('Scanned', $1, $2, 1000, '/music/b.flac', 10, 'flac') RETURNING id::text`, scannedAlbum, scannedArtist) mustExec(t, pool, `INSERT INTO library_changes (entity_type, entity_id, op) VALUES ('track', $1, 'upsert')`, scanned) if err := backup.Restore(ctx, pool, first); err != nil { t.Fatalf("restore: %v", err) } second, err := backup.Snapshot(ctx, pool) if err != nil { t.Fatalf("snapshot after restore: %v", err) } for _, tbl := range backup.Tables { a, b := first.Tables[tbl.Name], second.Tables[tbl.Name] if len(a) != len(b) { t.Errorf("%s: %d rows restored, the backup held %d", tbl.Name, len(b), len(a)) continue } for i := range a { if !bytes.Equal(a[i], b[i]) { t.Errorf("%s row %d:\nbacked up %s\nrestored %s", tbl.Name, i+1, a[i], b[i]) } } } // What the backup leaves out comes back unset, so the server asks for the // credential again and redoes the art and tags. var lbToken, coverSource, tagSource *string if err := pool.QueryRow(ctx, `SELECT u.listenbrainz_token, al.cover_art_source, tr.tag_source FROM users u, albums al, tracks tr WHERE u.id = $1 AND al.id = $2 AND tr.id = $3`, user, album, track).Scan(&lbToken, &coverSource, &tagSource); err != nil { t.Fatal(err) } if lbToken != nil || coverSource != nil || tagSource != nil { t.Errorf("token %v, cover source %v, tag source %v; want all unset", lbToken, coverSource, tagSource) } // The change feed carries on after the restored ids. var restoredMax, issued, next int64 if err := pool.QueryRow(ctx, "SELECT max(id) FROM library_changes").Scan(&restoredMax); err != nil { t.Fatal(err) } // The sequence says the restored history ends there, which is what the // library sync compares an app's cursor with. if err := pool.QueryRow(ctx, `SELECT COALESCE(pg_sequence_last_value( pg_get_serial_sequence('library_changes', 'id')::regclass), 0)`).Scan(&issued); err != nil { t.Fatal(err) } if issued != restoredMax { t.Errorf("the change sequence stands at %d, want the restored %d", issued, restoredMax) } if err := pool.QueryRow(ctx, `INSERT INTO library_changes (entity_type, entity_id, op) VALUES ('track', $1, 'upsert') RETURNING id`, track).Scan(&next); err != nil { t.Fatal(err) } if next <= restoredMax { t.Errorf("next change id %d, want it after the restored %d", next, restoredMax) } }