minstrel/internal/backup/backup_test.go

package backup_test

import (
	"bytes"
	"context"
	"encoding/json"
	"errors"
	"io"
	"log/slog"
	"os"
	"slices"
	"testing"

	"github.com/jackc/pgx/v5"
	"github.com/jackc/pgx/v5/pgxpool"

	"git.fabledsword.com/bvandeusen/minstrel/internal/backup"
	"git.fabledsword.com/bvandeusen/minstrel/internal/db"
	"git.fabledsword.com/bvandeusen/minstrel/internal/dbtest"
)

func newPool(t *testing.T) *pgxpool.Pool {
	t.Helper()
	if testing.Short() {
		t.Skip("skipping integration test in -short mode")
	}
	dsn := os.Getenv("MINSTREL_TEST_DATABASE_URL")
	if dsn == "" {
		t.Skip("MINSTREL_TEST_DATABASE_URL not set")
	}
	if err := db.Migrate(dsn, slog.New(slog.NewTextHandler(io.Discard, nil))); err != nil {
		t.Fatalf("migrate: %v", err)
	}
	pool, err := pgxpool.New(context.Background(), dsn)
	if err != nil {
		t.Fatalf("pool: %v", err)
	}
	t.Cleanup(pool.Close)
	dbtest.ResetDB(t, pool)
	return pool
}

func index(name string) int {
	return slices.IndexFunc(backup.Tables, func(t backup.Table) bool { return t.Name == name })
}

// Every table in the schema is backed up or excluded with a reason, so a new
// table fails here until someone decides which.
func TestEveryTableIsAccountedFor_Integration(t *testing.T) {
	pool := newPool(t)
	rows, err := pool.Query(context.Background(), "SELECT tablename FROM pg_tables WHERE schemaname = 'public'")
	if err != nil {
		t.Fatal(err)
	}
	schema, err := pgx.CollectRows(rows, pgx.RowTo[string])
	if err != nil {
		t.Fatal(err)
	}
	for _, p := range backup.Coverage(schema) {
		t.Error(p)
	}
}

// A restore loads each table after the ones it references, and never needs a
// table the backup leaves out.
func TestTablesComeAfterWhatTheyReference_Integration(t *testing.T) {
	pool := newPool(t)
	rows, err := pool.Query(context.Background(), `
		SELECT cl.relname, pc.relname
		  FROM pg_constraint co
		  JOIN pg_class cl ON cl.oid = co.conrelid
		  JOIN pg_class pc ON pc.oid = co.confrelid
		  JOIN pg_namespace n ON n.oid = cl.relnamespace
		 WHERE co.contype = 'f' AND n.nspname = 'public'`)
	if err != nil {
		t.Fatal(err)
	}
	type fk struct{ child, parent string }
	fks, err := pgx.CollectRows(rows, func(r pgx.CollectableRow) (fk, error) {
		var f fk
		err := r.Scan(&f.child, &f.parent)
		return f, err
	})
	if err != nil {
		t.Fatal(err)
	}
	if len(fks) == 0 {
		t.Fatal("found no foreign keys; the query is wrong")
	}
	for _, f := range fks {
		child, parent := index(f.child), index(f.parent)
		if child < 0 {
			continue
		}
		switch {
		case parent < 0:
			t.Errorf("%s is backed up but references %s, which is excluded", f.child, f.parent)
		case parent >= child && f.parent != f.child:
			t.Errorf("%s references %s, so it must come after it in Tables", f.child, f.parent)
		}
	}
}

// Every omitted column exists, so a renamed column fails here rather than a
// sealed credential slipping into the file under its new name.
func TestOmittedColumnsExist_Integration(t *testing.T) {
	pool := newPool(t)
	for _, tbl := range backup.Tables {
		for _, col := range tbl.Omit {
			var n int
			if err := pool.QueryRow(context.Background(), `
				SELECT count(*) FROM information_schema.columns
				 WHERE table_schema = 'public' AND table_name = $1 AND column_name = $2`,
				tbl.Name, col).Scan(&n); err != nil || n != 1 {
				t.Errorf("%s.%s: found %d columns, err %v", tbl.Name, col, n, err)
			}
		}
	}
}

func mustExec(t *testing.T, pool *pgxpool.Pool, sql string, args ...any) {
	t.Helper()
	if _, err := pool.Exec(context.Background(), sql, args...); err != nil {
		t.Fatalf("%s: %v", sql, err)
	}
}

func mustID(t *testing.T, pool *pgxpool.Pool, sql string, args ...any) string {
	t.Helper()
	var id string
	if err := pool.QueryRow(context.Background(), sql, args...).Scan(&id); err != nil {
		t.Fatalf("%s: %v", sql, err)
	}
	return id
}

// A backup restored into an emptied instance gives back the same rows, byte
// for byte, without the credentials and markers it leaves out.
func TestARestoredInstanceHoldsWhatTheBackupHeld_Integration(t *testing.T) {
	pool := newPool(t)
	ctx := context.Background()

	user := mustID(t, pool, `INSERT INTO users (username, password_hash, api_token_hash, listenbrainz_token)
		VALUES ($1, 'hash', 'backup-token', 'lb-token') RETURNING id::text`, dbtest.TestUserPrefix+"backup")
	artist := mustID(t, pool, `INSERT INTO artists (name, sort_name, artist_art_source) VALUES ('Artist', 'artist', 'none') RETURNING id::text`)
	album := mustID(t, pool, `INSERT INTO albums (title, sort_title, artist_id, cover_art_source, cover_art_path)
		VALUES ('Album', 'album', $1, 'mbcaa', '/data/covers/a.jpg') RETURNING id::text`, artist)
	track := mustID(t, pool, `INSERT INTO tracks (title, album_id, artist_id, duration_ms, file_path, file_size, file_format, tag_source)
		VALUES ('Track', $1, $2, 180000, '/music/a.flac', 1000, 'flac', 'musicbrainz') RETURNING id::text`, album, artist)
	session := mustID(t, pool, `INSERT INTO play_sessions (user_id, started_at, last_event_at) VALUES ($1, now(), now()) RETURNING id::text`, user)
	mustExec(t, pool, `INSERT INTO play_events (user_id, track_id, session_id, started_at) VALUES ($1, $2, $3, now())`, user, track, session)
	mustExec(t, pool, `INSERT INTO general_likes (user_id, track_id) VALUES ($1, $2)`, user, track)
	playlist := mustID(t, pool, `INSERT INTO playlists (user_id, name) VALUES ($1, 'Mine') RETURNING id::text`, user)
	mustExec(t, pool, `INSERT INTO playlist_tracks (playlist_id, position, track_id, title, artist_name, album_title, duration_sec)
		VALUES ($1, 0, $2, 'Track', 'Artist', 'Album', 180)`, playlist, track)
	mustExec(t, pool, `INSERT INTO follows (artist_mbid, artist_name, profile_id, followed_by)
		SELECT 'mbid-1', 'Artist', id, $1 FROM release_type_profiles WHERE is_default`, user)
	mustExec(t, pool, `INSERT INTO library_changes (entity_type, entity_id, op) VALUES ('track', $1, 'upsert')`, track)

	first, err := backup.Snapshot(ctx, pool)
	if err != nil {
		t.Fatalf("snapshot: %v", err)
	}
	for _, tbl := range backup.Tables {
		for _, line := range first.Tables[tbl.Name] {
			var row map[string]json.RawMessage
			if err := json.Unmarshal(line, &row); err != nil {
				t.Fatalf("%s: %v", tbl.Name, err)
			}
			for _, col := range tbl.Omit {
				if _, ok := row[col]; ok {
					t.Errorf("%s.%s is in the backup", tbl.Name, col)
				}
			}
		}
	}
	for _, tbl := range []string{"users", "tracks", "play_events", "general_likes", "playlist_tracks", "follows", "library_changes"} {
		if len(first.Tables[tbl]) == 0 {
			t.Errorf("the backup holds no %s", tbl)
		}
	}

	if err := backup.Restore(ctx, pool, first); !errors.Is(err, backup.ErrNotEmpty) {
		t.Fatalf("restore over data: err = %v, want ErrNotEmpty", err)
	}
	newer := first
	newer.Schema++
	if err := backup.Restore(ctx, pool, newer); !errors.Is(err, backup.ErrNewerSchema) {
		t.Fatalf("restore from a newer schema: err = %v, want ErrNewerSchema", err)
	}

	// A new server scans its music before anyone signs up, so it already has
	// a library, here with a track the backup doesn't know. It still takes
	// the backup: the backup's library replaces what the scan found, which the
	// next scan finds again under the backup's ids.
	mustExec(t, pool, "TRUNCATE users CASCADE")
	scannedArtist := mustID(t, pool, `INSERT INTO artists (name, sort_name) VALUES ('Scanned', 'scanned') RETURNING id::text`)
	scannedAlbum := mustID(t, pool, `INSERT INTO albums (title, sort_title, artist_id) VALUES ('Scanned', 'scanned', $1) RETURNING id::text`, scannedArtist)
	scanned := mustID(t, pool, `INSERT INTO tracks (title, album_id, artist_id, duration_ms, file_path, file_size, file_format)
		VALUES ('Scanned', $1, $2, 1000, '/music/b.flac', 10, 'flac') RETURNING id::text`, scannedAlbum, scannedArtist)
	mustExec(t, pool, `INSERT INTO library_changes (entity_type, entity_id, op) VALUES ('track', $1, 'upsert')`, scanned)
	if err := backup.Restore(ctx, pool, first); err != nil {
		t.Fatalf("restore: %v", err)
	}

	second, err := backup.Snapshot(ctx, pool)
	if err != nil {
		t.Fatalf("snapshot after restore: %v", err)
	}
	for _, tbl := range backup.Tables {
		a, b := first.Tables[tbl.Name], second.Tables[tbl.Name]
		if len(a) != len(b) {
			t.Errorf("%s: %d rows restored, the backup held %d", tbl.Name, len(b), len(a))
			continue
		}
		for i := range a {
			if !bytes.Equal(a[i], b[i]) {
				t.Errorf("%s row %d:\nbacked up %s\nrestored  %s", tbl.Name, i+1, a[i], b[i])
			}
		}
	}

	// What the backup leaves out comes back unset, so the server asks for the
	// credential again and redoes the art and tags.
	var lbToken, coverSource, tagSource *string
	if err := pool.QueryRow(ctx, `SELECT u.listenbrainz_token, al.cover_art_source, tr.tag_source
		FROM users u, albums al, tracks tr WHERE u.id = $1 AND al.id = $2 AND tr.id = $3`,
		user, album, track).Scan(&lbToken, &coverSource, &tagSource); err != nil {
		t.Fatal(err)
	}
	if lbToken != nil || coverSource != nil || tagSource != nil {
		t.Errorf("token %v, cover source %v, tag source %v; want all unset", lbToken, coverSource, tagSource)
	}

	// The change feed carries on after the restored ids.
	var restoredMax, issued, next int64
	if err := pool.QueryRow(ctx, "SELECT max(id) FROM library_changes").Scan(&restoredMax); err != nil {
		t.Fatal(err)
	}
	// The sequence says the restored history ends there, which is what the
	// library sync compares an app's cursor with.
	if err := pool.QueryRow(ctx, `SELECT COALESCE(pg_sequence_last_value(
		pg_get_serial_sequence('library_changes', 'id')::regclass), 0)`).Scan(&issued); err != nil {
		t.Fatal(err)
	}
	if issued != restoredMax {
		t.Errorf("the change sequence stands at %d, want the restored %d", issued, restoredMax)
	}
	if err := pool.QueryRow(ctx, `INSERT INTO library_changes (entity_type, entity_id, op)
		VALUES ('track', $1, 'upsert') RETURNING id`, track).Scan(&next); err != nil {
		t.Fatal(err)
	}
	if next <= restoredMax {
		t.Errorf("next change id %d, want it after the restored %d", next, restoredMax)
	}
}