From a67897a6cdd4c62a7ed26a00496acd5636e331c8 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Mon, 2 Nov 2026 13:43:16 -0500 Subject: [PATCH] feat: Minstrel can read its irreplaceable data out as JSON Lines and load it back into an empty instance (#6770) internal/backup names every table a backup holds, in parent-first order, and every table it leaves out with the reason: what the server computes from the music files, provider caches it fetches again, sign-ins and one-time links, and logs of past runs. The library's identity rows travel because every like, play and playlist entry hangs off a track id, and the sync change feed travels because the apps' first sync reads it. Sealed credentials are left out of the file, and so are the markers that say tag and art enrichment are done, so a restored server asks for the credentials again and redoes that work rather than believing it has it. Snapshot reads one REPEATABLE READ, read-only transaction, each table in primary-key order, keys sorted. Restore refuses an instance with accounts or tracks and a backup from a newer schema, replaces seeded settings, inserts by column intersection in batches, skips generated columns, and carries serial sequences on past the restored ids. Guards: every schema table is backed up or excluded; every backed-up table comes after what it references and references nothing excluded; every omitted column exists; a restore gives back the backup byte for byte. Co-Authored-By: Claude Opus 5.5 --- internal/backup/backup_test.go | 238 +++++++++++++++++++++++++++++++++ internal/backup/restore.go | 196 +++++++++++++++++++++++++++ internal/backup/snapshot.go | 114 ++++++++++++++++ internal/backup/tables.go | 193 ++++++++++++++++++++++++++ internal/backup/tables_test.go | 50 +++++++ 5 files changed, 791 insertions(+) create mode 100644 internal/backup/backup_test.go create mode 100644 internal/backup/restore.go create mode 100644 internal/backup/snapshot.go create mode 100644 internal/backup/tables.go create mode 100644 internal/backup/tables_test.go diff --git a/internal/backup/backup_test.go b/internal/backup/backup_test.go new file mode 100644 index 00000000..2043553c --- /dev/null +++ b/internal/backup/backup_test.go @@ -0,0 +1,238 @@ +package backup_test + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "io" + "log/slog" + "os" + "slices" + "testing" + + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgxpool" + + "git.fabledsword.com/bvandeusen/minstrel/internal/backup" + "git.fabledsword.com/bvandeusen/minstrel/internal/db" + "git.fabledsword.com/bvandeusen/minstrel/internal/dbtest" +) + +func newPool(t *testing.T) *pgxpool.Pool { + t.Helper() + if testing.Short() { + t.Skip("skipping integration test in -short mode") + } + dsn := os.Getenv("MINSTREL_TEST_DATABASE_URL") + if dsn == "" { + t.Skip("MINSTREL_TEST_DATABASE_URL not set") + } + if err := db.Migrate(dsn, slog.New(slog.NewTextHandler(io.Discard, nil))); err != nil { + t.Fatalf("migrate: %v", err) + } + pool, err := pgxpool.New(context.Background(), dsn) + if err != nil { + t.Fatalf("pool: %v", err) + } + t.Cleanup(pool.Close) + dbtest.ResetDB(t, pool) + return pool +} + +func index(name string) int { + return slices.IndexFunc(backup.Tables, func(t backup.Table) bool { return t.Name == name }) +} + +// Every table in the schema is backed up or excluded with a reason, so a new +// table fails here until someone decides which. +func TestEveryTableIsAccountedFor_Integration(t *testing.T) { + pool := newPool(t) + rows, err := pool.Query(context.Background(), "SELECT tablename FROM pg_tables WHERE schemaname = 'public'") + if err != nil { + t.Fatal(err) + } + schema, err := pgx.CollectRows(rows, pgx.RowTo[string]) + if err != nil { + t.Fatal(err) + } + for _, p := range backup.Coverage(schema) { + t.Error(p) + } +} + +// A restore loads each table after the ones it references, and never needs a +// table the backup leaves out. +func TestTablesComeAfterWhatTheyReference_Integration(t *testing.T) { + pool := newPool(t) + rows, err := pool.Query(context.Background(), ` + SELECT cl.relname, pc.relname + FROM pg_constraint co + JOIN pg_class cl ON cl.oid = co.conrelid + JOIN pg_class pc ON pc.oid = co.confrelid + JOIN pg_namespace n ON n.oid = cl.relnamespace + WHERE co.contype = 'f' AND n.nspname = 'public'`) + if err != nil { + t.Fatal(err) + } + type fk struct{ child, parent string } + fks, err := pgx.CollectRows(rows, func(r pgx.CollectableRow) (fk, error) { + var f fk + err := r.Scan(&f.child, &f.parent) + return f, err + }) + if err != nil { + t.Fatal(err) + } + if len(fks) == 0 { + t.Fatal("found no foreign keys; the query is wrong") + } + for _, f := range fks { + child, parent := index(f.child), index(f.parent) + if child < 0 { + continue + } + switch { + case parent < 0: + t.Errorf("%s is backed up but references %s, which is excluded", f.child, f.parent) + case parent >= child && f.parent != f.child: + t.Errorf("%s references %s, so it must come after it in Tables", f.child, f.parent) + } + } +} + +// Every omitted column exists, so a renamed column fails here rather than a +// sealed credential slipping into the file under its new name. +func TestOmittedColumnsExist_Integration(t *testing.T) { + pool := newPool(t) + for _, tbl := range backup.Tables { + for _, col := range tbl.Omit { + var n int + if err := pool.QueryRow(context.Background(), ` + SELECT count(*) FROM information_schema.columns + WHERE table_schema = 'public' AND table_name = $1 AND column_name = $2`, + tbl.Name, col).Scan(&n); err != nil || n != 1 { + t.Errorf("%s.%s: found %d columns, err %v", tbl.Name, col, n, err) + } + } + } +} + +func mustExec(t *testing.T, pool *pgxpool.Pool, sql string, args ...any) { + t.Helper() + if _, err := pool.Exec(context.Background(), sql, args...); err != nil { + t.Fatalf("%s: %v", sql, err) + } +} + +func mustID(t *testing.T, pool *pgxpool.Pool, sql string, args ...any) string { + t.Helper() + var id string + if err := pool.QueryRow(context.Background(), sql, args...).Scan(&id); err != nil { + t.Fatalf("%s: %v", sql, err) + } + return id +} + +// A backup restored into an emptied instance gives back the same rows, byte +// for byte, without the credentials and markers it leaves out. +func TestARestoredInstanceHoldsWhatTheBackupHeld_Integration(t *testing.T) { + pool := newPool(t) + ctx := context.Background() + + user := mustID(t, pool, `INSERT INTO users (username, password_hash, api_token_hash, listenbrainz_token) + VALUES ($1, 'hash', 'backup-token', 'lb-token') RETURNING id::text`, dbtest.TestUserPrefix+"backup") + artist := mustID(t, pool, `INSERT INTO artists (name, sort_name, artist_art_source) VALUES ('Artist', 'artist', 'none') RETURNING id::text`) + album := mustID(t, pool, `INSERT INTO albums (title, sort_title, artist_id, cover_art_source, cover_art_path) + VALUES ('Album', 'album', $1, 'mbcaa', '/data/covers/a.jpg') RETURNING id::text`, artist) + track := mustID(t, pool, `INSERT INTO tracks (title, album_id, artist_id, duration_ms, file_path, file_size, file_format, tag_source) + VALUES ('Track', $1, $2, 180000, '/music/a.flac', 1000, 'flac', 'musicbrainz') RETURNING id::text`, album, artist) + session := mustID(t, pool, `INSERT INTO play_sessions (user_id, started_at, last_event_at) VALUES ($1, now(), now()) RETURNING id::text`, user) + mustExec(t, pool, `INSERT INTO play_events (user_id, track_id, session_id, started_at) VALUES ($1, $2, $3, now())`, user, track, session) + mustExec(t, pool, `INSERT INTO general_likes (user_id, track_id) VALUES ($1, $2)`, user, track) + playlist := mustID(t, pool, `INSERT INTO playlists (user_id, name) VALUES ($1, 'Mine') RETURNING id::text`, user) + mustExec(t, pool, `INSERT INTO playlist_tracks (playlist_id, position, track_id, title, artist_name, album_title, duration_sec) + VALUES ($1, 0, $2, 'Track', 'Artist', 'Album', 180)`, playlist, track) + mustExec(t, pool, `INSERT INTO follows (artist_mbid, artist_name, profile_id, followed_by) + SELECT 'mbid-1', 'Artist', id, $1 FROM release_type_profiles WHERE is_default`, user) + mustExec(t, pool, `INSERT INTO library_changes (entity_type, entity_id, op) VALUES ('track', $1, 'upsert')`, track) + + first, err := backup.Snapshot(ctx, pool) + if err != nil { + t.Fatalf("snapshot: %v", err) + } + for _, tbl := range backup.Tables { + for _, line := range first.Tables[tbl.Name] { + var row map[string]json.RawMessage + if err := json.Unmarshal(line, &row); err != nil { + t.Fatalf("%s: %v", tbl.Name, err) + } + for _, col := range tbl.Omit { + if _, ok := row[col]; ok { + t.Errorf("%s.%s is in the backup", tbl.Name, col) + } + } + } + } + for _, tbl := range []string{"users", "tracks", "play_events", "general_likes", "playlist_tracks", "follows", "library_changes"} { + if len(first.Tables[tbl]) == 0 { + t.Errorf("the backup holds no %s", tbl) + } + } + + if err := backup.Restore(ctx, pool, first); !errors.Is(err, backup.ErrNotEmpty) { + t.Fatalf("restore over data: err = %v, want ErrNotEmpty", err) + } + newer := first + newer.Schema++ + if err := backup.Restore(ctx, pool, newer); !errors.Is(err, backup.ErrNewerSchema) { + t.Fatalf("restore from a newer schema: err = %v, want ErrNewerSchema", err) + } + + mustExec(t, pool, "TRUNCATE users, artists, library_changes CASCADE") + if err := backup.Restore(ctx, pool, first); err != nil { + t.Fatalf("restore: %v", err) + } + + second, err := backup.Snapshot(ctx, pool) + if err != nil { + t.Fatalf("snapshot after restore: %v", err) + } + for _, tbl := range backup.Tables { + a, b := first.Tables[tbl.Name], second.Tables[tbl.Name] + if len(a) != len(b) { + t.Errorf("%s: %d rows restored, the backup held %d", tbl.Name, len(b), len(a)) + continue + } + for i := range a { + if !bytes.Equal(a[i], b[i]) { + t.Errorf("%s row %d:\nbacked up %s\nrestored %s", tbl.Name, i+1, a[i], b[i]) + } + } + } + + // What the backup leaves out comes back unset, so the server asks for the + // credential again and redoes the art and tags. + var lbToken, coverSource, tagSource *string + if err := pool.QueryRow(ctx, `SELECT u.listenbrainz_token, al.cover_art_source, tr.tag_source + FROM users u, albums al, tracks tr WHERE u.id = $1 AND al.id = $2 AND tr.id = $3`, + user, album, track).Scan(&lbToken, &coverSource, &tagSource); err != nil { + t.Fatal(err) + } + if lbToken != nil || coverSource != nil || tagSource != nil { + t.Errorf("token %v, cover source %v, tag source %v; want all unset", lbToken, coverSource, tagSource) + } + + // The change feed carries on after the restored ids. + var restoredMax, next int64 + if err := pool.QueryRow(ctx, "SELECT max(id) FROM library_changes").Scan(&restoredMax); err != nil { + t.Fatal(err) + } + if err := pool.QueryRow(ctx, `INSERT INTO library_changes (entity_type, entity_id, op) + VALUES ('track', $1, 'upsert') RETURNING id`, track).Scan(&next); err != nil { + t.Fatal(err) + } + if next <= restoredMax { + t.Errorf("next change id %d, want it after the restored %d", next, restoredMax) + } +} diff --git a/internal/backup/restore.go b/internal/backup/restore.go new file mode 100644 index 00000000..2020b7fa --- /dev/null +++ b/internal/backup/restore.go @@ -0,0 +1,196 @@ +package backup + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "slices" + "strings" + + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgxpool" +) + +var ( + // ErrNotEmpty is a restore into an instance that already has accounts or + // a library. + ErrNotEmpty = errors.New("backup: this Minstrel already has accounts or a library; a backup restores into an empty one") + + // ErrNewerSchema is a backup written by a newer Minstrel than this one. + ErrNewerSchema = errors.New("backup: the backup was written by a newer version of Minstrel; upgrade this one first") + + // ErrNotABackup is contents that aren't rows a backup writes. + ErrNotABackup = errors.New("backup: not a Minstrel backup") +) + +// batchRows is how many rows one INSERT carries. +const batchRows = 500 + +// Restore loads a backup into an empty instance, in one transaction: either +// all of it lands or none of it does. +// +// The instance must have no accounts and no tracks. Its other rows in the +// tables the backup holds are replaced, which is what a fresh instance's +// seeded settings need. A table the backup doesn't hold, because it was +// written before that table existed, keeps what the instance has. +// +// A row's columns are those the backup and the table both have: an older +// backup's rows take the defaults of columns added since, and a column dropped +// since is left out. +func Restore(ctx context.Context, pool *pgxpool.Pool, c Contents) error { + tx, err := pool.Begin(ctx) + if err != nil { + return fmt.Errorf("backup: begin: %w", err) + } + defer func() { _ = tx.Rollback(ctx) }() + + schema, err := schemaVersion(ctx, tx) + if err != nil { + return err + } + if c.Schema > schema { + return ErrNewerSchema + } + // Taken before the emptiness check, so a first account or a scan starting + // at the same moment waits for the restore, then finds the data. + if _, err := tx.Exec(ctx, "LOCK TABLE users, tracks IN EXCLUSIVE MODE"); err != nil { + return fmt.Errorf("backup: lock: %w", err) + } + var occupied bool + if err := tx.QueryRow(ctx, + "SELECT EXISTS (SELECT 1 FROM users) OR EXISTS (SELECT 1 FROM tracks)").Scan(&occupied); err != nil { + return fmt.Errorf("backup: check the instance is empty: %w", err) + } + if occupied { + return ErrNotEmpty + } + + // Children first, so no row is left pointing at one already gone. + for _, t := range slices.Backward(Tables) { + if _, ok := c.Tables[t.Name]; !ok { + continue + } + if _, err := tx.Exec(ctx, "DELETE FROM "+pgx.Identifier{t.Name}.Sanitize()); err != nil { + return fmt.Errorf("backup: clear %s: %w", t.Name, err) + } + } + for _, t := range Tables { + rows, ok := c.Tables[t.Name] + if !ok { + continue + } + if err := restoreTable(ctx, tx, t.Name, rows); err != nil { + return err + } + } + if err := tx.Commit(ctx); err != nil { + return fmt.Errorf("backup: commit: %w", err) + } + return nil +} + +type column struct { + name string + identity bool + sequence bool +} + +func restoreTable(ctx context.Context, tx pgx.Tx, table string, rows [][]byte) error { + cols, err := writableColumns(ctx, tx, table) + if err != nil { + return err + } + ident := pgx.Identifier{table}.Sanitize() + overriding := "" + if slices.ContainsFunc(cols, func(c column) bool { return c.identity }) { + overriding = " OVERRIDING SYSTEM VALUE" + } + + // Rows go in batches that share one key set; a backup's rows for a table + // all do, since one read wrote them. + var ( + batch []json.RawMessage + list string + ) + flush := func() error { + if len(batch) == 0 { + return nil + } + payload, err := json.Marshal(batch) + if err != nil { + return err + } + sql := fmt.Sprintf("INSERT INTO %s (%s)%s SELECT %s FROM jsonb_populate_recordset(NULL::%s, $1::jsonb)", + ident, list, overriding, list, ident) + if _, err := tx.Exec(ctx, sql, string(payload)); err != nil { + return fmt.Errorf("backup: restore %s: %w", table, err) + } + batch = batch[:0] + return nil + } + for i, raw := range rows { + var keys map[string]json.RawMessage + if err := json.Unmarshal(raw, &keys); err != nil { + return fmt.Errorf("%w: row %d of %s is not an object", ErrNotABackup, i+1, table) + } + var names []string + for _, c := range cols { + if _, ok := keys[c.name]; ok { + names = append(names, pgx.Identifier{c.name}.Sanitize()) + } + } + if len(names) == 0 { + continue + } + slices.Sort(names) + rowList := strings.Join(names, ", ") + if rowList != list || len(batch) == batchRows { + if err := flush(); err != nil { + return err + } + list = rowList + } + batch = append(batch, raw) + } + if err := flush(); err != nil { + return err + } + + // A sequence carries on after the highest restored value, so the next row + // doesn't collide with one from the backup. + for _, c := range cols { + if !c.sequence { + continue + } + col := pgx.Identifier{c.name}.Sanitize() + if _, err := tx.Exec(ctx, fmt.Sprintf( + "SELECT setval(pg_get_serial_sequence($1, $2), COALESCE((SELECT max(%s) FROM %s), 0) + 1, false)", col, ident), + table, c.name); err != nil { + return fmt.Errorf("backup: carry on %s's sequence: %w", table, err) + } + } + return nil +} + +// writableColumns returns the columns of a table an INSERT can set: all but +// generated ones, which Postgres computes. +func writableColumns(ctx context.Context, tx pgx.Tx, table string) ([]column, error) { + rows, err := tx.Query(ctx, ` + SELECT column_name, is_identity = 'YES', pg_get_serial_sequence($1::text, column_name) IS NOT NULL + FROM information_schema.columns + WHERE table_schema = current_schema() AND table_name = $1::text AND is_generated = 'NEVER' + ORDER BY ordinal_position`, table) + if err != nil { + return nil, fmt.Errorf("backup: read %s's columns: %w", table, err) + } + cols, err := pgx.CollectRows(rows, func(row pgx.CollectableRow) (column, error) { + var c column + err := row.Scan(&c.name, &c.identity, &c.sequence) + return c, err + }) + if err != nil { + return nil, fmt.Errorf("backup: read %s's columns: %w", table, err) + } + return cols, nil +} diff --git a/internal/backup/snapshot.go b/internal/backup/snapshot.go new file mode 100644 index 00000000..9fa716dd --- /dev/null +++ b/internal/backup/snapshot.go @@ -0,0 +1,114 @@ +package backup + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "strings" + + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgxpool" +) + +// Contents is a backup's data: the schema version that wrote it, and each +// table's rows as JSON objects with their keys sorted, in primary-key order. +// An unchanged row is the same bytes in every backup. +type Contents struct { + Schema int64 + Tables map[string][][]byte +} + +// Snapshot reads every table in Tables as of one moment. +func Snapshot(ctx context.Context, pool *pgxpool.Pool) (Contents, error) { + c := Contents{Tables: make(map[string][][]byte, len(Tables))} + tx, err := pool.BeginTx(ctx, pgx.TxOptions{IsoLevel: pgx.RepeatableRead, AccessMode: pgx.ReadOnly}) + if err != nil { + return c, fmt.Errorf("backup: begin: %w", err) + } + defer func() { _ = tx.Rollback(ctx) }() + + if c.Schema, err = schemaVersion(ctx, tx); err != nil { + return c, err + } + for _, t := range Tables { + lines, err := readTable(ctx, tx, t) + if err != nil { + return c, err + } + c.Tables[t.Name] = lines + } + return c, nil +} + +func readTable(ctx context.Context, tx pgx.Tx, t Table) ([][]byte, error) { + pk, err := primaryKey(ctx, tx, t.Name) + if err != nil { + return nil, err + } + order := make([]string, len(pk)) + for i, col := range pk { + order[i] = "t." + pgx.Identifier{col}.Sanitize() + } + omit := t.Omit + if omit == nil { + omit = []string{} + } + rows, err := tx.Query(ctx, fmt.Sprintf("SELECT to_jsonb(t) - $1::text[] FROM %s t ORDER BY %s", + pgx.Identifier{t.Name}.Sanitize(), strings.Join(order, ", ")), omit) + if err != nil { + return nil, fmt.Errorf("backup: read %s: %w", t.Name, err) + } + lines, err := pgx.CollectRows(rows, func(row pgx.CollectableRow) ([]byte, error) { + var raw []byte + if err := row.Scan(&raw); err != nil { + return nil, err + } + return canonical(raw) + }) + if err != nil { + return nil, fmt.Errorf("backup: read %s: %w", t.Name, err) + } + return lines, nil +} + +// canonical re-encodes a row with its keys sorted. Numbers pass through as +// written, so a bigint or a float stays exact. +func canonical(raw []byte) ([]byte, error) { + dec := json.NewDecoder(bytes.NewReader(raw)) + dec.UseNumber() + var row map[string]any + if err := dec.Decode(&row); err != nil { + return nil, err + } + return json.Marshal(row) +} + +// primaryKey returns a table's primary-key columns in key order. +func primaryKey(ctx context.Context, tx pgx.Tx, table string) ([]string, error) { + rows, err := tx.Query(ctx, ` + SELECT a.attname + FROM pg_index i + JOIN pg_attribute a ON a.attrelid = i.indrelid AND a.attnum = ANY (i.indkey) + WHERE i.indrelid = to_regclass($1) AND i.indisprimary + ORDER BY array_position(i.indkey, a.attnum)`, table) + if err != nil { + return nil, fmt.Errorf("backup: read %s's primary key: %w", table, err) + } + cols, err := pgx.CollectRows(rows, pgx.RowTo[string]) + if err != nil { + return nil, fmt.Errorf("backup: read %s's primary key: %w", table, err) + } + if len(cols) == 0 { + return nil, fmt.Errorf("backup: %s has no primary key to order its rows by", table) + } + return cols, nil +} + +func schemaVersion(ctx context.Context, tx pgx.Tx) (int64, error) { + var v int64 + if err := tx.QueryRow(ctx, "SELECT version FROM schema_migrations").Scan(&v); err != nil { + return 0, fmt.Errorf("backup: read the schema version: %w", err) + } + return v, nil +} diff --git a/internal/backup/tables.go b/internal/backup/tables.go new file mode 100644 index 00000000..81ad6dc7 --- /dev/null +++ b/internal/backup/tables.go @@ -0,0 +1,193 @@ +// Package backup reads Minstrel's irreplaceable data out of Postgres as JSON +// Lines, one table at a time, and loads it back into an empty instance. +// +// What goes in is everything a person made or decided: accounts, likes, play +// and skip history, playlists, follows, wants and requests, download history, +// admin decisions, and settings. What stays out is everything the restored +// server rebuilds by itself: what it computes from the music files, what it +// fetches again from MusicBrainz, ListenBrainz and the other providers, and +// logs of past runs. Excluded says why for each, and the coverage test fails +// on a table that is in neither list. +package backup + +// Table is one table a backup holds. +// +// Omit names columns left out of the file. They restore to their defaults: +// - a credential Minstrel stores sealed, which only the server holding the +// stream secret could open, so the restored server asks for it again; +// - a marker saying work on an excluded table is done (a tag source, an art +// source), so the restored server does that work again rather than +// believing it has the result. +type Table struct { + Name string + Omit []string +} + +// Tables is every table a backup holds, in the order a restore loads them: +// each after every table it references. There are no self-referencing +// tables, so no table needs its rows loaded in depth order. +var Tables = []Table{ + {Name: "users", Omit: []string{"subsonic_password", "listenbrainz_token"}}, + + // The library's identity. Every like, play and playlist entry hangs off a + // track id, so the rows travel even though a scan could rebuild most of + // their columns; the scan after a restore finds each file at its path and + // keeps its id. + {Name: "artists", Omit: []string{"artist_thumb_path", "artist_fanart_path", "artist_art_source", "artist_art_sources_version"}}, + {Name: "albums", Omit: []string{"cover_art_path", "cover_art_source", "cover_art_sources_version"}}, + {Name: "tracks", Omit: []string{"tag_source", "tag_sources_version"}}, + + // Settings. + {Name: "library_settings"}, + {Name: "network_settings"}, + {Name: "registration_settings"}, + {Name: "smtp_config", Omit: []string{"password"}}, + {Name: "notification_email_settings"}, + {Name: "acoustid_settings", Omit: []string{"api_key"}}, + {Name: "fingerprint_settings"}, + {Name: "loudness_settings"}, + {Name: "cover_art_provider_settings", Omit: []string{"api_key"}}, + {Name: "cover_art_sources_meta"}, + {Name: "tag_provider_settings", Omit: []string{"api_key"}}, + {Name: "tag_sources_meta"}, + {Name: "acquisition_services", Omit: []string{"api_key", "password"}}, + {Name: "acquisition_settings"}, + {Name: "reacquisition_settings"}, + {Name: "discover_tuning"}, + {Name: "taste_tuning"}, + {Name: "recommendation_weight_profiles"}, + {Name: "recommendation_tuning_audit"}, + + // Follows, wants, requests and downloads. + {Name: "release_type_profiles"}, + {Name: "follows"}, + {Name: "follow_seen_groups"}, + {Name: "wants"}, + {Name: "grabs"}, + {Name: "download_interventions"}, + {Name: "release_blocks"}, + {Name: "requests"}, + + // Listening history and likes. + {Name: "play_sessions"}, + {Name: "play_events"}, + {Name: "skip_events"}, + {Name: "scrobble_queue"}, + {Name: "contextual_likes"}, + {Name: "general_likes"}, + {Name: "general_likes_albums"}, + {Name: "general_likes_artists"}, + + // Playlists and what is built from history. The taste profile and the + // "you might like" rows are rebuilt nightly from the history above; they + // travel so a restored Home is full before that first nightly pass. + {Name: "playlists"}, + {Name: "playlist_tracks"}, + {Name: "system_playlist_runs"}, + {Name: "system_playlist_rotation_state"}, + {Name: "taste_profile_artists"}, + {Name: "taste_profile_tags"}, + {Name: "taste_profile_eras"}, + {Name: "taste_profile_moods"}, + {Name: "you_might_like_albums"}, + {Name: "you_might_like_artists"}, + + // Per-user preferences and notifications. + {Name: "user_normalization_prefs"}, + {Name: "user_notification_prefs"}, + {Name: "user_notification_email_state"}, + {Name: "user_notifications"}, + {Name: "suggestion_snoozes"}, + {Name: "quarantine"}, + {Name: "quarantine_actions"}, + + // Admin decisions: an album taken off the wanted listing stays off, and a + // dismissed duplicate group stays dismissed when the sweep finds it again. + {Name: "library_album_matches"}, + {Name: "duplicate_sweeps"}, + {Name: "duplicate_groups"}, + {Name: "duplicate_group_members"}, + + // The library change feed is how the apps sync, including a first sync, + // so a restored server carries it on from where the backup left it. + {Name: "library_changes"}, + {Name: "audit_log"}, + {Name: "playback_errors"}, +} + +// Excluded is every table a backup leaves out, with the reason. +var Excluded = map[string]string{ + "schema_migrations": "the restored server's own migrations write it; the backup's schema version is in its manifest", + "sessions": "sign-ins; everyone signs in again on the restored server", + "password_resets": "one-time reset links, which expire within hours", + "user_invites": "open sign-up links; an admin sends an invite again from the restored server", + + "track_fingerprints": "computed from the music files", + "track_loudness": "computed from the music files", + "album_loudness": "computed from the music files", + "track_acoustid_lookups": "looked up again from each track's fingerprint", + "track_tags": "fetched again from the tag providers", + "candidate_artist_tags": "fetched again from the tag providers", + "candidate_artist_tag_state": "fetched again from the tag providers", + + "mb_artists": "MusicBrainz cache, fetched again", + "mb_release_groups": "MusicBrainz cache, fetched again", + "mb_releases": "MusicBrainz cache, fetched again", + "mb_release_tracks": "MusicBrainz cache, fetched again", + "mb_artist_release_groups": "MusicBrainz cache, fetched again", + + "listenbrainz_similar_recordings": "similarity, fetched again from ListenBrainz", + "track_similarity": "similarity, fetched again from the similarity providers", + "track_similarity_fetches": "similarity, fetched again from the similarity providers", + "artist_similarity": "similarity, fetched again from the similarity providers", + "artist_similarity_fetches": "similarity, fetched again from the similarity providers", + "artist_similarity_unmatched": "similarity, fetched again from the similarity providers", + + "want_candidates": "the last search's results; the next search finds them again", + "scan_runs": "a log of past library scans", + "diagnostic_events": "debug-mode diagnostics", +} + +// carried reports whether a table is in Tables. +func carried(name string) bool { + for _, t := range Tables { + if t.Name == name { + return true + } + } + return false +} + +// Coverage checks a schema's tables against Tables and Excluded. It returns +// one line per problem: a table in neither list, in both, or listed twice, +// and a listed table the schema doesn't have. +func Coverage(schema []string) []string { + var problems []string + inSchema := map[string]bool{} + for _, name := range schema { + inSchema[name] = true + _, excluded := Excluded[name] + switch { + case carried(name) && excluded: + problems = append(problems, name+": both backed up and excluded") + case !carried(name) && !excluded: + problems = append(problems, name+": neither backed up nor excluded; add it to Tables, or to Excluded with the reason") + } + } + seen := map[string]bool{} + for _, t := range Tables { + if seen[t.Name] { + problems = append(problems, t.Name+": listed twice in Tables") + } + seen[t.Name] = true + if !inSchema[t.Name] { + problems = append(problems, t.Name+": in Tables but not in the schema") + } + } + for name := range Excluded { + if !inSchema[name] { + problems = append(problems, name+": in Excluded but not in the schema") + } + } + return problems +} diff --git a/internal/backup/tables_test.go b/internal/backup/tables_test.go new file mode 100644 index 00000000..591db231 --- /dev/null +++ b/internal/backup/tables_test.go @@ -0,0 +1,50 @@ +package backup + +import ( + "slices" + "strings" + "testing" +) + +func everyListedTable() []string { + var names []string + for _, t := range Tables { + names = append(names, t.Name) + } + for name := range Excluded { + names = append(names, name) + } + return names +} + +func TestCoverageAcceptsASchemaThatMatchesTheLists(t *testing.T) { + if problems := Coverage(everyListedTable()); len(problems) != 0 { + t.Fatalf("problems = %v, want none", problems) + } +} + +func TestCoverageNamesATableInNeitherList(t *testing.T) { + problems := Coverage(append(everyListedTable(), "new_table")) + if len(problems) != 1 || !strings.HasPrefix(problems[0], "new_table: neither") { + t.Fatalf("problems = %v, want one naming new_table", problems) + } +} + +func TestCoverageNamesAListedTableTheSchemaLacks(t *testing.T) { + schema := slices.DeleteFunc(everyListedTable(), func(n string) bool { return n == "playlists" || n == "scan_runs" }) + problems := Coverage(schema) + if len(problems) != 2 { + t.Fatalf("problems = %v, want playlists and scan_runs named", problems) + } +} + +func TestEveryExclusionHasAReason(t *testing.T) { + for name, why := range Excluded { + if strings.TrimSpace(why) == "" { + t.Errorf("%s is excluded without a reason", name) + } + if carried(name) { + t.Errorf("%s is both backed up and excluded", name) + } + } +}