diff --git a/internal/backup/backup_test.go b/internal/backup/backup_test.go new file mode 100644 index 00000000..2043553c --- /dev/null +++ b/internal/backup/backup_test.go @@ -0,0 +1,238 @@ +package backup_test + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "io" + "log/slog" + "os" + "slices" + "testing" + + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgxpool" + + "git.fabledsword.com/bvandeusen/minstrel/internal/backup" + "git.fabledsword.com/bvandeusen/minstrel/internal/db" + "git.fabledsword.com/bvandeusen/minstrel/internal/dbtest" +) + +func newPool(t *testing.T) *pgxpool.Pool { + t.Helper() + if testing.Short() { + t.Skip("skipping integration test in -short mode") + } + dsn := os.Getenv("MINSTREL_TEST_DATABASE_URL") + if dsn == "" { + t.Skip("MINSTREL_TEST_DATABASE_URL not set") + } + if err := db.Migrate(dsn, slog.New(slog.NewTextHandler(io.Discard, nil))); err != nil { + t.Fatalf("migrate: %v", err) + } + pool, err := pgxpool.New(context.Background(), dsn) + if err != nil { + t.Fatalf("pool: %v", err) + } + t.Cleanup(pool.Close) + dbtest.ResetDB(t, pool) + return pool +} + +func index(name string) int { + return slices.IndexFunc(backup.Tables, func(t backup.Table) bool { return t.Name == name }) +} + +// Every table in the schema is backed up or excluded with a reason, so a new +// table fails here until someone decides which. +func TestEveryTableIsAccountedFor_Integration(t *testing.T) { + pool := newPool(t) + rows, err := pool.Query(context.Background(), "SELECT tablename FROM pg_tables WHERE schemaname = 'public'") + if err != nil { + t.Fatal(err) + } + schema, err := pgx.CollectRows(rows, pgx.RowTo[string]) + if err != nil { + t.Fatal(err) + } + for _, p := range backup.Coverage(schema) { + t.Error(p) + } +} + +// A restore loads each table after the ones it references, and never needs a +// table the backup leaves out. +func TestTablesComeAfterWhatTheyReference_Integration(t *testing.T) { + pool := newPool(t) + rows, err := pool.Query(context.Background(), ` + SELECT cl.relname, pc.relname + FROM pg_constraint co + JOIN pg_class cl ON cl.oid = co.conrelid + JOIN pg_class pc ON pc.oid = co.confrelid + JOIN pg_namespace n ON n.oid = cl.relnamespace + WHERE co.contype = 'f' AND n.nspname = 'public'`) + if err != nil { + t.Fatal(err) + } + type fk struct{ child, parent string } + fks, err := pgx.CollectRows(rows, func(r pgx.CollectableRow) (fk, error) { + var f fk + err := r.Scan(&f.child, &f.parent) + return f, err + }) + if err != nil { + t.Fatal(err) + } + if len(fks) == 0 { + t.Fatal("found no foreign keys; the query is wrong") + } + for _, f := range fks { + child, parent := index(f.child), index(f.parent) + if child < 0 { + continue + } + switch { + case parent < 0: + t.Errorf("%s is backed up but references %s, which is excluded", f.child, f.parent) + case parent >= child && f.parent != f.child: + t.Errorf("%s references %s, so it must come after it in Tables", f.child, f.parent) + } + } +} + +// Every omitted column exists, so a renamed column fails here rather than a +// sealed credential slipping into the file under its new name. +func TestOmittedColumnsExist_Integration(t *testing.T) { + pool := newPool(t) + for _, tbl := range backup.Tables { + for _, col := range tbl.Omit { + var n int + if err := pool.QueryRow(context.Background(), ` + SELECT count(*) FROM information_schema.columns + WHERE table_schema = 'public' AND table_name = $1 AND column_name = $2`, + tbl.Name, col).Scan(&n); err != nil || n != 1 { + t.Errorf("%s.%s: found %d columns, err %v", tbl.Name, col, n, err) + } + } + } +} + +func mustExec(t *testing.T, pool *pgxpool.Pool, sql string, args ...any) { + t.Helper() + if _, err := pool.Exec(context.Background(), sql, args...); err != nil { + t.Fatalf("%s: %v", sql, err) + } +} + +func mustID(t *testing.T, pool *pgxpool.Pool, sql string, args ...any) string { + t.Helper() + var id string + if err := pool.QueryRow(context.Background(), sql, args...).Scan(&id); err != nil { + t.Fatalf("%s: %v", sql, err) + } + return id +} + +// A backup restored into an emptied instance gives back the same rows, byte +// for byte, without the credentials and markers it leaves out. +func TestARestoredInstanceHoldsWhatTheBackupHeld_Integration(t *testing.T) { + pool := newPool(t) + ctx := context.Background() + + user := mustID(t, pool, `INSERT INTO users (username, password_hash, api_token_hash, listenbrainz_token) + VALUES ($1, 'hash', 'backup-token', 'lb-token') RETURNING id::text`, dbtest.TestUserPrefix+"backup") + artist := mustID(t, pool, `INSERT INTO artists (name, sort_name, artist_art_source) VALUES ('Artist', 'artist', 'none') RETURNING id::text`) + album := mustID(t, pool, `INSERT INTO albums (title, sort_title, artist_id, cover_art_source, cover_art_path) + VALUES ('Album', 'album', $1, 'mbcaa', '/data/covers/a.jpg') RETURNING id::text`, artist) + track := mustID(t, pool, `INSERT INTO tracks (title, album_id, artist_id, duration_ms, file_path, file_size, file_format, tag_source) + VALUES ('Track', $1, $2, 180000, '/music/a.flac', 1000, 'flac', 'musicbrainz') RETURNING id::text`, album, artist) + session := mustID(t, pool, `INSERT INTO play_sessions (user_id, started_at, last_event_at) VALUES ($1, now(), now()) RETURNING id::text`, user) + mustExec(t, pool, `INSERT INTO play_events (user_id, track_id, session_id, started_at) VALUES ($1, $2, $3, now())`, user, track, session) + mustExec(t, pool, `INSERT INTO general_likes (user_id, track_id) VALUES ($1, $2)`, user, track) + playlist := mustID(t, pool, `INSERT INTO playlists (user_id, name) VALUES ($1, 'Mine') RETURNING id::text`, user) + mustExec(t, pool, `INSERT INTO playlist_tracks (playlist_id, position, track_id, title, artist_name, album_title, duration_sec) + VALUES ($1, 0, $2, 'Track', 'Artist', 'Album', 180)`, playlist, track) + mustExec(t, pool, `INSERT INTO follows (artist_mbid, artist_name, profile_id, followed_by) + SELECT 'mbid-1', 'Artist', id, $1 FROM release_type_profiles WHERE is_default`, user) + mustExec(t, pool, `INSERT INTO library_changes (entity_type, entity_id, op) VALUES ('track', $1, 'upsert')`, track) + + first, err := backup.Snapshot(ctx, pool) + if err != nil { + t.Fatalf("snapshot: %v", err) + } + for _, tbl := range backup.Tables { + for _, line := range first.Tables[tbl.Name] { + var row map[string]json.RawMessage + if err := json.Unmarshal(line, &row); err != nil { + t.Fatalf("%s: %v", tbl.Name, err) + } + for _, col := range tbl.Omit { + if _, ok := row[col]; ok { + t.Errorf("%s.%s is in the backup", tbl.Name, col) + } + } + } + } + for _, tbl := range []string{"users", "tracks", "play_events", "general_likes", "playlist_tracks", "follows", "library_changes"} { + if len(first.Tables[tbl]) == 0 { + t.Errorf("the backup holds no %s", tbl) + } + } + + if err := backup.Restore(ctx, pool, first); !errors.Is(err, backup.ErrNotEmpty) { + t.Fatalf("restore over data: err = %v, want ErrNotEmpty", err) + } + newer := first + newer.Schema++ + if err := backup.Restore(ctx, pool, newer); !errors.Is(err, backup.ErrNewerSchema) { + t.Fatalf("restore from a newer schema: err = %v, want ErrNewerSchema", err) + } + + mustExec(t, pool, "TRUNCATE users, artists, library_changes CASCADE") + if err := backup.Restore(ctx, pool, first); err != nil { + t.Fatalf("restore: %v", err) + } + + second, err := backup.Snapshot(ctx, pool) + if err != nil { + t.Fatalf("snapshot after restore: %v", err) + } + for _, tbl := range backup.Tables { + a, b := first.Tables[tbl.Name], second.Tables[tbl.Name] + if len(a) != len(b) { + t.Errorf("%s: %d rows restored, the backup held %d", tbl.Name, len(b), len(a)) + continue + } + for i := range a { + if !bytes.Equal(a[i], b[i]) { + t.Errorf("%s row %d:\nbacked up %s\nrestored %s", tbl.Name, i+1, a[i], b[i]) + } + } + } + + // What the backup leaves out comes back unset, so the server asks for the + // credential again and redoes the art and tags. + var lbToken, coverSource, tagSource *string + if err := pool.QueryRow(ctx, `SELECT u.listenbrainz_token, al.cover_art_source, tr.tag_source + FROM users u, albums al, tracks tr WHERE u.id = $1 AND al.id = $2 AND tr.id = $3`, + user, album, track).Scan(&lbToken, &coverSource, &tagSource); err != nil { + t.Fatal(err) + } + if lbToken != nil || coverSource != nil || tagSource != nil { + t.Errorf("token %v, cover source %v, tag source %v; want all unset", lbToken, coverSource, tagSource) + } + + // The change feed carries on after the restored ids. + var restoredMax, next int64 + if err := pool.QueryRow(ctx, "SELECT max(id) FROM library_changes").Scan(&restoredMax); err != nil { + t.Fatal(err) + } + if err := pool.QueryRow(ctx, `INSERT INTO library_changes (entity_type, entity_id, op) + VALUES ('track', $1, 'upsert') RETURNING id`, track).Scan(&next); err != nil { + t.Fatal(err) + } + if next <= restoredMax { + t.Errorf("next change id %d, want it after the restored %d", next, restoredMax) + } +} diff --git a/internal/backup/restore.go b/internal/backup/restore.go new file mode 100644 index 00000000..2020b7fa --- /dev/null +++ b/internal/backup/restore.go @@ -0,0 +1,196 @@ +package backup + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "slices" + "strings" + + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgxpool" +) + +var ( + // ErrNotEmpty is a restore into an instance that already has accounts or + // a library. + ErrNotEmpty = errors.New("backup: this Minstrel already has accounts or a library; a backup restores into an empty one") + + // ErrNewerSchema is a backup written by a newer Minstrel than this one. + ErrNewerSchema = errors.New("backup: the backup was written by a newer version of Minstrel; upgrade this one first") + + // ErrNotABackup is contents that aren't rows a backup writes. + ErrNotABackup = errors.New("backup: not a Minstrel backup") +) + +// batchRows is how many rows one INSERT carries. +const batchRows = 500 + +// Restore loads a backup into an empty instance, in one transaction: either +// all of it lands or none of it does. +// +// The instance must have no accounts and no tracks. Its other rows in the +// tables the backup holds are replaced, which is what a fresh instance's +// seeded settings need. A table the backup doesn't hold, because it was +// written before that table existed, keeps what the instance has. +// +// A row's columns are those the backup and the table both have: an older +// backup's rows take the defaults of columns added since, and a column dropped +// since is left out. +func Restore(ctx context.Context, pool *pgxpool.Pool, c Contents) error { + tx, err := pool.Begin(ctx) + if err != nil { + return fmt.Errorf("backup: begin: %w", err) + } + defer func() { _ = tx.Rollback(ctx) }() + + schema, err := schemaVersion(ctx, tx) + if err != nil { + return err + } + if c.Schema > schema { + return ErrNewerSchema + } + // Taken before the emptiness check, so a first account or a scan starting + // at the same moment waits for the restore, then finds the data. + if _, err := tx.Exec(ctx, "LOCK TABLE users, tracks IN EXCLUSIVE MODE"); err != nil { + return fmt.Errorf("backup: lock: %w", err) + } + var occupied bool + if err := tx.QueryRow(ctx, + "SELECT EXISTS (SELECT 1 FROM users) OR EXISTS (SELECT 1 FROM tracks)").Scan(&occupied); err != nil { + return fmt.Errorf("backup: check the instance is empty: %w", err) + } + if occupied { + return ErrNotEmpty + } + + // Children first, so no row is left pointing at one already gone. + for _, t := range slices.Backward(Tables) { + if _, ok := c.Tables[t.Name]; !ok { + continue + } + if _, err := tx.Exec(ctx, "DELETE FROM "+pgx.Identifier{t.Name}.Sanitize()); err != nil { + return fmt.Errorf("backup: clear %s: %w", t.Name, err) + } + } + for _, t := range Tables { + rows, ok := c.Tables[t.Name] + if !ok { + continue + } + if err := restoreTable(ctx, tx, t.Name, rows); err != nil { + return err + } + } + if err := tx.Commit(ctx); err != nil { + return fmt.Errorf("backup: commit: %w", err) + } + return nil +} + +type column struct { + name string + identity bool + sequence bool +} + +func restoreTable(ctx context.Context, tx pgx.Tx, table string, rows [][]byte) error { + cols, err := writableColumns(ctx, tx, table) + if err != nil { + return err + } + ident := pgx.Identifier{table}.Sanitize() + overriding := "" + if slices.ContainsFunc(cols, func(c column) bool { return c.identity }) { + overriding = " OVERRIDING SYSTEM VALUE" + } + + // Rows go in batches that share one key set; a backup's rows for a table + // all do, since one read wrote them. + var ( + batch []json.RawMessage + list string + ) + flush := func() error { + if len(batch) == 0 { + return nil + } + payload, err := json.Marshal(batch) + if err != nil { + return err + } + sql := fmt.Sprintf("INSERT INTO %s (%s)%s SELECT %s FROM jsonb_populate_recordset(NULL::%s, $1::jsonb)", + ident, list, overriding, list, ident) + if _, err := tx.Exec(ctx, sql, string(payload)); err != nil { + return fmt.Errorf("backup: restore %s: %w", table, err) + } + batch = batch[:0] + return nil + } + for i, raw := range rows { + var keys map[string]json.RawMessage + if err := json.Unmarshal(raw, &keys); err != nil { + return fmt.Errorf("%w: row %d of %s is not an object", ErrNotABackup, i+1, table) + } + var names []string + for _, c := range cols { + if _, ok := keys[c.name]; ok { + names = append(names, pgx.Identifier{c.name}.Sanitize()) + } + } + if len(names) == 0 { + continue + } + slices.Sort(names) + rowList := strings.Join(names, ", ") + if rowList != list || len(batch) == batchRows { + if err := flush(); err != nil { + return err + } + list = rowList + } + batch = append(batch, raw) + } + if err := flush(); err != nil { + return err + } + + // A sequence carries on after the highest restored value, so the next row + // doesn't collide with one from the backup. + for _, c := range cols { + if !c.sequence { + continue + } + col := pgx.Identifier{c.name}.Sanitize() + if _, err := tx.Exec(ctx, fmt.Sprintf( + "SELECT setval(pg_get_serial_sequence($1, $2), COALESCE((SELECT max(%s) FROM %s), 0) + 1, false)", col, ident), + table, c.name); err != nil { + return fmt.Errorf("backup: carry on %s's sequence: %w", table, err) + } + } + return nil +} + +// writableColumns returns the columns of a table an INSERT can set: all but +// generated ones, which Postgres computes. +func writableColumns(ctx context.Context, tx pgx.Tx, table string) ([]column, error) { + rows, err := tx.Query(ctx, ` + SELECT column_name, is_identity = 'YES', pg_get_serial_sequence($1::text, column_name) IS NOT NULL + FROM information_schema.columns + WHERE table_schema = current_schema() AND table_name = $1::text AND is_generated = 'NEVER' + ORDER BY ordinal_position`, table) + if err != nil { + return nil, fmt.Errorf("backup: read %s's columns: %w", table, err) + } + cols, err := pgx.CollectRows(rows, func(row pgx.CollectableRow) (column, error) { + var c column + err := row.Scan(&c.name, &c.identity, &c.sequence) + return c, err + }) + if err != nil { + return nil, fmt.Errorf("backup: read %s's columns: %w", table, err) + } + return cols, nil +} diff --git a/internal/backup/snapshot.go b/internal/backup/snapshot.go new file mode 100644 index 00000000..9fa716dd --- /dev/null +++ b/internal/backup/snapshot.go @@ -0,0 +1,114 @@ +package backup + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "strings" + + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgxpool" +) + +// Contents is a backup's data: the schema version that wrote it, and each +// table's rows as JSON objects with their keys sorted, in primary-key order. +// An unchanged row is the same bytes in every backup. +type Contents struct { + Schema int64 + Tables map[string][][]byte +} + +// Snapshot reads every table in Tables as of one moment. +func Snapshot(ctx context.Context, pool *pgxpool.Pool) (Contents, error) { + c := Contents{Tables: make(map[string][][]byte, len(Tables))} + tx, err := pool.BeginTx(ctx, pgx.TxOptions{IsoLevel: pgx.RepeatableRead, AccessMode: pgx.ReadOnly}) + if err != nil { + return c, fmt.Errorf("backup: begin: %w", err) + } + defer func() { _ = tx.Rollback(ctx) }() + + if c.Schema, err = schemaVersion(ctx, tx); err != nil { + return c, err + } + for _, t := range Tables { + lines, err := readTable(ctx, tx, t) + if err != nil { + return c, err + } + c.Tables[t.Name] = lines + } + return c, nil +} + +func readTable(ctx context.Context, tx pgx.Tx, t Table) ([][]byte, error) { + pk, err := primaryKey(ctx, tx, t.Name) + if err != nil { + return nil, err + } + order := make([]string, len(pk)) + for i, col := range pk { + order[i] = "t." + pgx.Identifier{col}.Sanitize() + } + omit := t.Omit + if omit == nil { + omit = []string{} + } + rows, err := tx.Query(ctx, fmt.Sprintf("SELECT to_jsonb(t) - $1::text[] FROM %s t ORDER BY %s", + pgx.Identifier{t.Name}.Sanitize(), strings.Join(order, ", ")), omit) + if err != nil { + return nil, fmt.Errorf("backup: read %s: %w", t.Name, err) + } + lines, err := pgx.CollectRows(rows, func(row pgx.CollectableRow) ([]byte, error) { + var raw []byte + if err := row.Scan(&raw); err != nil { + return nil, err + } + return canonical(raw) + }) + if err != nil { + return nil, fmt.Errorf("backup: read %s: %w", t.Name, err) + } + return lines, nil +} + +// canonical re-encodes a row with its keys sorted. Numbers pass through as +// written, so a bigint or a float stays exact. +func canonical(raw []byte) ([]byte, error) { + dec := json.NewDecoder(bytes.NewReader(raw)) + dec.UseNumber() + var row map[string]any + if err := dec.Decode(&row); err != nil { + return nil, err + } + return json.Marshal(row) +} + +// primaryKey returns a table's primary-key columns in key order. +func primaryKey(ctx context.Context, tx pgx.Tx, table string) ([]string, error) { + rows, err := tx.Query(ctx, ` + SELECT a.attname + FROM pg_index i + JOIN pg_attribute a ON a.attrelid = i.indrelid AND a.attnum = ANY (i.indkey) + WHERE i.indrelid = to_regclass($1) AND i.indisprimary + ORDER BY array_position(i.indkey, a.attnum)`, table) + if err != nil { + return nil, fmt.Errorf("backup: read %s's primary key: %w", table, err) + } + cols, err := pgx.CollectRows(rows, pgx.RowTo[string]) + if err != nil { + return nil, fmt.Errorf("backup: read %s's primary key: %w", table, err) + } + if len(cols) == 0 { + return nil, fmt.Errorf("backup: %s has no primary key to order its rows by", table) + } + return cols, nil +} + +func schemaVersion(ctx context.Context, tx pgx.Tx) (int64, error) { + var v int64 + if err := tx.QueryRow(ctx, "SELECT version FROM schema_migrations").Scan(&v); err != nil { + return 0, fmt.Errorf("backup: read the schema version: %w", err) + } + return v, nil +} diff --git a/internal/backup/tables.go b/internal/backup/tables.go new file mode 100644 index 00000000..81ad6dc7 --- /dev/null +++ b/internal/backup/tables.go @@ -0,0 +1,193 @@ +// Package backup reads Minstrel's irreplaceable data out of Postgres as JSON +// Lines, one table at a time, and loads it back into an empty instance. +// +// What goes in is everything a person made or decided: accounts, likes, play +// and skip history, playlists, follows, wants and requests, download history, +// admin decisions, and settings. What stays out is everything the restored +// server rebuilds by itself: what it computes from the music files, what it +// fetches again from MusicBrainz, ListenBrainz and the other providers, and +// logs of past runs. Excluded says why for each, and the coverage test fails +// on a table that is in neither list. +package backup + +// Table is one table a backup holds. +// +// Omit names columns left out of the file. They restore to their defaults: +// - a credential Minstrel stores sealed, which only the server holding the +// stream secret could open, so the restored server asks for it again; +// - a marker saying work on an excluded table is done (a tag source, an art +// source), so the restored server does that work again rather than +// believing it has the result. +type Table struct { + Name string + Omit []string +} + +// Tables is every table a backup holds, in the order a restore loads them: +// each after every table it references. There are no self-referencing +// tables, so no table needs its rows loaded in depth order. +var Tables = []Table{ + {Name: "users", Omit: []string{"subsonic_password", "listenbrainz_token"}}, + + // The library's identity. Every like, play and playlist entry hangs off a + // track id, so the rows travel even though a scan could rebuild most of + // their columns; the scan after a restore finds each file at its path and + // keeps its id. + {Name: "artists", Omit: []string{"artist_thumb_path", "artist_fanart_path", "artist_art_source", "artist_art_sources_version"}}, + {Name: "albums", Omit: []string{"cover_art_path", "cover_art_source", "cover_art_sources_version"}}, + {Name: "tracks", Omit: []string{"tag_source", "tag_sources_version"}}, + + // Settings. + {Name: "library_settings"}, + {Name: "network_settings"}, + {Name: "registration_settings"}, + {Name: "smtp_config", Omit: []string{"password"}}, + {Name: "notification_email_settings"}, + {Name: "acoustid_settings", Omit: []string{"api_key"}}, + {Name: "fingerprint_settings"}, + {Name: "loudness_settings"}, + {Name: "cover_art_provider_settings", Omit: []string{"api_key"}}, + {Name: "cover_art_sources_meta"}, + {Name: "tag_provider_settings", Omit: []string{"api_key"}}, + {Name: "tag_sources_meta"}, + {Name: "acquisition_services", Omit: []string{"api_key", "password"}}, + {Name: "acquisition_settings"}, + {Name: "reacquisition_settings"}, + {Name: "discover_tuning"}, + {Name: "taste_tuning"}, + {Name: "recommendation_weight_profiles"}, + {Name: "recommendation_tuning_audit"}, + + // Follows, wants, requests and downloads. + {Name: "release_type_profiles"}, + {Name: "follows"}, + {Name: "follow_seen_groups"}, + {Name: "wants"}, + {Name: "grabs"}, + {Name: "download_interventions"}, + {Name: "release_blocks"}, + {Name: "requests"}, + + // Listening history and likes. + {Name: "play_sessions"}, + {Name: "play_events"}, + {Name: "skip_events"}, + {Name: "scrobble_queue"}, + {Name: "contextual_likes"}, + {Name: "general_likes"}, + {Name: "general_likes_albums"}, + {Name: "general_likes_artists"}, + + // Playlists and what is built from history. The taste profile and the + // "you might like" rows are rebuilt nightly from the history above; they + // travel so a restored Home is full before that first nightly pass. + {Name: "playlists"}, + {Name: "playlist_tracks"}, + {Name: "system_playlist_runs"}, + {Name: "system_playlist_rotation_state"}, + {Name: "taste_profile_artists"}, + {Name: "taste_profile_tags"}, + {Name: "taste_profile_eras"}, + {Name: "taste_profile_moods"}, + {Name: "you_might_like_albums"}, + {Name: "you_might_like_artists"}, + + // Per-user preferences and notifications. + {Name: "user_normalization_prefs"}, + {Name: "user_notification_prefs"}, + {Name: "user_notification_email_state"}, + {Name: "user_notifications"}, + {Name: "suggestion_snoozes"}, + {Name: "quarantine"}, + {Name: "quarantine_actions"}, + + // Admin decisions: an album taken off the wanted listing stays off, and a + // dismissed duplicate group stays dismissed when the sweep finds it again. + {Name: "library_album_matches"}, + {Name: "duplicate_sweeps"}, + {Name: "duplicate_groups"}, + {Name: "duplicate_group_members"}, + + // The library change feed is how the apps sync, including a first sync, + // so a restored server carries it on from where the backup left it. + {Name: "library_changes"}, + {Name: "audit_log"}, + {Name: "playback_errors"}, +} + +// Excluded is every table a backup leaves out, with the reason. +var Excluded = map[string]string{ + "schema_migrations": "the restored server's own migrations write it; the backup's schema version is in its manifest", + "sessions": "sign-ins; everyone signs in again on the restored server", + "password_resets": "one-time reset links, which expire within hours", + "user_invites": "open sign-up links; an admin sends an invite again from the restored server", + + "track_fingerprints": "computed from the music files", + "track_loudness": "computed from the music files", + "album_loudness": "computed from the music files", + "track_acoustid_lookups": "looked up again from each track's fingerprint", + "track_tags": "fetched again from the tag providers", + "candidate_artist_tags": "fetched again from the tag providers", + "candidate_artist_tag_state": "fetched again from the tag providers", + + "mb_artists": "MusicBrainz cache, fetched again", + "mb_release_groups": "MusicBrainz cache, fetched again", + "mb_releases": "MusicBrainz cache, fetched again", + "mb_release_tracks": "MusicBrainz cache, fetched again", + "mb_artist_release_groups": "MusicBrainz cache, fetched again", + + "listenbrainz_similar_recordings": "similarity, fetched again from ListenBrainz", + "track_similarity": "similarity, fetched again from the similarity providers", + "track_similarity_fetches": "similarity, fetched again from the similarity providers", + "artist_similarity": "similarity, fetched again from the similarity providers", + "artist_similarity_fetches": "similarity, fetched again from the similarity providers", + "artist_similarity_unmatched": "similarity, fetched again from the similarity providers", + + "want_candidates": "the last search's results; the next search finds them again", + "scan_runs": "a log of past library scans", + "diagnostic_events": "debug-mode diagnostics", +} + +// carried reports whether a table is in Tables. +func carried(name string) bool { + for _, t := range Tables { + if t.Name == name { + return true + } + } + return false +} + +// Coverage checks a schema's tables against Tables and Excluded. It returns +// one line per problem: a table in neither list, in both, or listed twice, +// and a listed table the schema doesn't have. +func Coverage(schema []string) []string { + var problems []string + inSchema := map[string]bool{} + for _, name := range schema { + inSchema[name] = true + _, excluded := Excluded[name] + switch { + case carried(name) && excluded: + problems = append(problems, name+": both backed up and excluded") + case !carried(name) && !excluded: + problems = append(problems, name+": neither backed up nor excluded; add it to Tables, or to Excluded with the reason") + } + } + seen := map[string]bool{} + for _, t := range Tables { + if seen[t.Name] { + problems = append(problems, t.Name+": listed twice in Tables") + } + seen[t.Name] = true + if !inSchema[t.Name] { + problems = append(problems, t.Name+": in Tables but not in the schema") + } + } + for name := range Excluded { + if !inSchema[name] { + problems = append(problems, name+": in Excluded but not in the schema") + } + } + return problems +} diff --git a/internal/backup/tables_test.go b/internal/backup/tables_test.go new file mode 100644 index 00000000..591db231 --- /dev/null +++ b/internal/backup/tables_test.go @@ -0,0 +1,50 @@ +package backup + +import ( + "slices" + "strings" + "testing" +) + +func everyListedTable() []string { + var names []string + for _, t := range Tables { + names = append(names, t.Name) + } + for name := range Excluded { + names = append(names, name) + } + return names +} + +func TestCoverageAcceptsASchemaThatMatchesTheLists(t *testing.T) { + if problems := Coverage(everyListedTable()); len(problems) != 0 { + t.Fatalf("problems = %v, want none", problems) + } +} + +func TestCoverageNamesATableInNeitherList(t *testing.T) { + problems := Coverage(append(everyListedTable(), "new_table")) + if len(problems) != 1 || !strings.HasPrefix(problems[0], "new_table: neither") { + t.Fatalf("problems = %v, want one naming new_table", problems) + } +} + +func TestCoverageNamesAListedTableTheSchemaLacks(t *testing.T) { + schema := slices.DeleteFunc(everyListedTable(), func(n string) bool { return n == "playlists" || n == "scan_runs" }) + problems := Coverage(schema) + if len(problems) != 2 { + t.Fatalf("problems = %v, want playlists and scan_runs named", problems) + } +} + +func TestEveryExclusionHasAReason(t *testing.T) { + for name, why := range Excluded { + if strings.TrimSpace(why) == "" { + t.Errorf("%s is excluded without a reason", name) + } + if carried(name) { + t.Errorf("%s is both backed up and excluded", name) + } + } +}