feat: Minstrel can read its irreplaceable data out as JSON Lines and load it back into an empty instance (#6770)

internal/backup names every table a backup holds, in parent-first order,
and every table it leaves out with the reason: what the server computes
from the music files, provider caches it fetches again, sign-ins and
one-time links, and logs of past runs. The library's identity rows travel
because every like, play and playlist entry hangs off a track id, and the
sync change feed travels because the apps' first sync reads it.

Sealed credentials are left out of the file, and so are the markers that
say tag and art enrichment are done, so a restored server asks for the
credentials again and redoes that work rather than believing it has it.

Snapshot reads one REPEATABLE READ, read-only transaction, each table in
primary-key order, keys sorted. Restore refuses an instance with accounts
or tracks and a backup from a newer schema, replaces seeded settings,
inserts by column intersection in batches, skips generated columns, and
carries serial sequences on past the restored ids.

Guards: every schema table is backed up or excluded; every backed-up
table comes after what it references and references nothing excluded;
every omitted column exists; a restore gives back the backup byte for byte.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

internal/backup/tables.go

@@ -0,0 +1,193 @@
+// Package backup reads Minstrel's irreplaceable data out of Postgres as JSON
+// Lines, one table at a time, and loads it back into an empty instance.
+//
+// What goes in is everything a person made or decided: accounts, likes, play
+// and skip history, playlists, follows, wants and requests, download history,
+// admin decisions, and settings. What stays out is everything the restored
+// server rebuilds by itself: what it computes from the music files, what it
+// fetches again from MusicBrainz, ListenBrainz and the other providers, and
+// logs of past runs. Excluded says why for each, and the coverage test fails
+// on a table that is in neither list.
+package backup
+
+// Table is one table a backup holds.
+//
+// Omit names columns left out of the file. They restore to their defaults:
+//   - a credential Minstrel stores sealed, which only the server holding the
+//     stream secret could open, so the restored server asks for it again;
+//   - a marker saying work on an excluded table is done (a tag source, an art
+//     source), so the restored server does that work again rather than
+//     believing it has the result.
+type Table struct {
+	Name string
+	Omit []string
+}
+
+// Tables is every table a backup holds, in the order a restore loads them:
+// each after every table it references. There are no self-referencing
+// tables, so no table needs its rows loaded in depth order.
+var Tables = []Table{
+	{Name: "users", Omit: []string{"subsonic_password", "listenbrainz_token"}},
+
+	// The library's identity. Every like, play and playlist entry hangs off a
+	// track id, so the rows travel even though a scan could rebuild most of
+	// their columns; the scan after a restore finds each file at its path and
+	// keeps its id.
+	{Name: "artists", Omit: []string{"artist_thumb_path", "artist_fanart_path", "artist_art_source", "artist_art_sources_version"}},
+	{Name: "albums", Omit: []string{"cover_art_path", "cover_art_source", "cover_art_sources_version"}},
+	{Name: "tracks", Omit: []string{"tag_source", "tag_sources_version"}},
+
+	// Settings.
+	{Name: "library_settings"},
+	{Name: "network_settings"},
+	{Name: "registration_settings"},
+	{Name: "smtp_config", Omit: []string{"password"}},
+	{Name: "notification_email_settings"},
+	{Name: "acoustid_settings", Omit: []string{"api_key"}},
+	{Name: "fingerprint_settings"},
+	{Name: "loudness_settings"},
+	{Name: "cover_art_provider_settings", Omit: []string{"api_key"}},
+	{Name: "cover_art_sources_meta"},
+	{Name: "tag_provider_settings", Omit: []string{"api_key"}},
+	{Name: "tag_sources_meta"},
+	{Name: "acquisition_services", Omit: []string{"api_key", "password"}},
+	{Name: "acquisition_settings"},
+	{Name: "reacquisition_settings"},
+	{Name: "discover_tuning"},
+	{Name: "taste_tuning"},
+	{Name: "recommendation_weight_profiles"},
+	{Name: "recommendation_tuning_audit"},
+
+	// Follows, wants, requests and downloads.
+	{Name: "release_type_profiles"},
+	{Name: "follows"},
+	{Name: "follow_seen_groups"},
+	{Name: "wants"},
+	{Name: "grabs"},
+	{Name: "download_interventions"},
+	{Name: "release_blocks"},
+	{Name: "requests"},
+
+	// Listening history and likes.
+	{Name: "play_sessions"},
+	{Name: "play_events"},
+	{Name: "skip_events"},
+	{Name: "scrobble_queue"},
+	{Name: "contextual_likes"},
+	{Name: "general_likes"},
+	{Name: "general_likes_albums"},
+	{Name: "general_likes_artists"},
+
+	// Playlists and what is built from history. The taste profile and the
+	// "you might like" rows are rebuilt nightly from the history above; they
+	// travel so a restored Home is full before that first nightly pass.
+	{Name: "playlists"},
+	{Name: "playlist_tracks"},
+	{Name: "system_playlist_runs"},
+	{Name: "system_playlist_rotation_state"},
+	{Name: "taste_profile_artists"},
+	{Name: "taste_profile_tags"},
+	{Name: "taste_profile_eras"},
+	{Name: "taste_profile_moods"},
+	{Name: "you_might_like_albums"},
+	{Name: "you_might_like_artists"},
+
+	// Per-user preferences and notifications.
+	{Name: "user_normalization_prefs"},
+	{Name: "user_notification_prefs"},
+	{Name: "user_notification_email_state"},
+	{Name: "user_notifications"},
+	{Name: "suggestion_snoozes"},
+	{Name: "quarantine"},
+	{Name: "quarantine_actions"},
+
+	// Admin decisions: an album taken off the wanted listing stays off, and a
+	// dismissed duplicate group stays dismissed when the sweep finds it again.
+	{Name: "library_album_matches"},
+	{Name: "duplicate_sweeps"},
+	{Name: "duplicate_groups"},
+	{Name: "duplicate_group_members"},
+
+	// The library change feed is how the apps sync, including a first sync,
+	// so a restored server carries it on from where the backup left it.
+	{Name: "library_changes"},
+	{Name: "audit_log"},
+	{Name: "playback_errors"},
+}
+
+// Excluded is every table a backup leaves out, with the reason.
+var Excluded = map[string]string{
+	"schema_migrations": "the restored server's own migrations write it; the backup's schema version is in its manifest",
+	"sessions":          "sign-ins; everyone signs in again on the restored server",
+	"password_resets":   "one-time reset links, which expire within hours",
+	"user_invites":      "open sign-up links; an admin sends an invite again from the restored server",
+
+	"track_fingerprints":         "computed from the music files",
+	"track_loudness":             "computed from the music files",
+	"album_loudness":             "computed from the music files",
+	"track_acoustid_lookups":     "looked up again from each track's fingerprint",
+	"track_tags":                 "fetched again from the tag providers",
+	"candidate_artist_tags":      "fetched again from the tag providers",
+	"candidate_artist_tag_state": "fetched again from the tag providers",
+
+	"mb_artists":               "MusicBrainz cache, fetched again",
+	"mb_release_groups":        "MusicBrainz cache, fetched again",
+	"mb_releases":              "MusicBrainz cache, fetched again",
+	"mb_release_tracks":        "MusicBrainz cache, fetched again",
+	"mb_artist_release_groups": "MusicBrainz cache, fetched again",
+
+	"listenbrainz_similar_recordings": "similarity, fetched again from ListenBrainz",
+	"track_similarity":                "similarity, fetched again from the similarity providers",
+	"track_similarity_fetches":        "similarity, fetched again from the similarity providers",
+	"artist_similarity":               "similarity, fetched again from the similarity providers",
+	"artist_similarity_fetches":       "similarity, fetched again from the similarity providers",
+	"artist_similarity_unmatched":     "similarity, fetched again from the similarity providers",
+
+	"want_candidates":   "the last search's results; the next search finds them again",
+	"scan_runs":         "a log of past library scans",
+	"diagnostic_events": "debug-mode diagnostics",
+}
+
+// carried reports whether a table is in Tables.
+func carried(name string) bool {
+	for _, t := range Tables {
+		if t.Name == name {
+			return true
+		}
+	}
+	return false
+}
+
+// Coverage checks a schema's tables against Tables and Excluded. It returns
+// one line per problem: a table in neither list, in both, or listed twice,
+// and a listed table the schema doesn't have.
+func Coverage(schema []string) []string {
+	var problems []string
+	inSchema := map[string]bool{}
+	for _, name := range schema {
+		inSchema[name] = true
+		_, excluded := Excluded[name]
+		switch {
+		case carried(name) && excluded:
+			problems = append(problems, name+": both backed up and excluded")
+		case !carried(name) && !excluded:
+			problems = append(problems, name+": neither backed up nor excluded; add it to Tables, or to Excluded with the reason")
+		}
+	}
+	seen := map[string]bool{}
+	for _, t := range Tables {
+		if seen[t.Name] {
+			problems = append(problems, t.Name+": listed twice in Tables")
+		}
+		seen[t.Name] = true
+		if !inSchema[t.Name] {
+			problems = append(problems, t.Name+": in Tables but not in the schema")
+		}
+	}
+	for name := range Excluded {
+		if !inSchema[name] {
+			problems = append(problems, name+": in Excluded but not in the schema")
+		}
+	}
+	return problems
+}