// Package backup reads Minstrel's irreplaceable data out of Postgres as JSON // Lines, one table at a time, and loads it back into an empty instance. // // What goes in is everything a person made or decided: accounts, likes, play // and skip history, playlists, follows, wants and requests, download history, // admin decisions, and settings. What stays out is everything the restored // server rebuilds by itself: what it computes from the music files, what it // fetches again from MusicBrainz, ListenBrainz and the other providers, and // logs of past runs. Excluded says why for each, and the coverage test fails // on a table that is in neither list. package backup // Table is one table a backup holds. // // Omit names columns left out of the file. They restore to their defaults: // - a credential Minstrel stores sealed, which only the server holding the // stream secret could open, so the restored server asks for it again; // - a marker saying work on an excluded table is done (a tag source, an art // source), so the restored server does that work again rather than // believing it has the result. type Table struct { Name string Omit []string } // Tables is every table a backup holds, in the order a restore loads them: // each after every table it references. There are no self-referencing // tables, so no table needs its rows loaded in depth order. var Tables = []Table{ {Name: "users", Omit: []string{"subsonic_password", "listenbrainz_token"}}, // The library's identity. Every like, play and playlist entry hangs off a // track id, so the rows travel even though a scan could rebuild most of // their columns; the scan after a restore finds each file at its path and // keeps its id. {Name: "artists", Omit: []string{"artist_thumb_path", "artist_fanart_path", "artist_art_source", "artist_art_sources_version"}}, {Name: "albums", Omit: []string{"cover_art_path", "cover_art_source", "cover_art_sources_version"}}, {Name: "tracks", Omit: []string{"tag_source", "tag_sources_version"}}, // Settings. {Name: "library_settings"}, {Name: "network_settings"}, {Name: "registration_settings"}, {Name: "smtp_config", Omit: []string{"password"}}, {Name: "notification_email_settings"}, {Name: "acoustid_settings", Omit: []string{"api_key"}}, {Name: "fingerprint_settings"}, {Name: "loudness_settings"}, {Name: "cover_art_provider_settings", Omit: []string{"api_key"}}, {Name: "cover_art_sources_meta"}, {Name: "tag_provider_settings", Omit: []string{"api_key"}}, {Name: "tag_sources_meta"}, {Name: "acquisition_services", Omit: []string{"api_key", "password"}}, {Name: "acquisition_settings"}, {Name: "reacquisition_settings"}, {Name: "discover_tuning"}, {Name: "taste_tuning"}, {Name: "recommendation_weight_profiles"}, {Name: "recommendation_tuning_audit"}, // Follows, wants, requests and downloads. {Name: "release_type_profiles"}, {Name: "follows"}, {Name: "follow_seen_groups"}, {Name: "wants"}, {Name: "grabs"}, {Name: "download_interventions"}, {Name: "release_blocks"}, {Name: "requests"}, // Listening history and likes. {Name: "play_sessions"}, {Name: "play_events"}, {Name: "skip_events"}, {Name: "scrobble_queue"}, {Name: "contextual_likes"}, {Name: "general_likes"}, {Name: "general_likes_albums"}, {Name: "general_likes_artists"}, // Playlists and what is built from history. The taste profile and the // "you might like" rows are rebuilt nightly from the history above; they // travel so a restored Home is full before that first nightly pass. {Name: "playlists"}, {Name: "playlist_tracks"}, {Name: "system_playlist_runs"}, {Name: "system_playlist_rotation_state"}, {Name: "taste_profile_artists"}, {Name: "taste_profile_tags"}, {Name: "taste_profile_eras"}, {Name: "taste_profile_moods"}, {Name: "you_might_like_albums"}, {Name: "you_might_like_artists"}, // Per-user preferences and notifications. {Name: "user_normalization_prefs"}, {Name: "user_notification_prefs"}, {Name: "user_notification_email_state"}, {Name: "user_notifications"}, {Name: "suggestion_snoozes"}, {Name: "quarantine"}, {Name: "quarantine_actions"}, // Admin decisions: an album taken off the wanted listing stays off, and a // dismissed duplicate group stays dismissed when the sweep finds it again. {Name: "library_album_matches"}, {Name: "duplicate_sweeps"}, {Name: "duplicate_groups"}, {Name: "duplicate_group_members"}, // The library change feed is how the apps sync, including a first sync, // so a restored server carries it on from where the backup left it. {Name: "library_changes"}, {Name: "audit_log"}, {Name: "playback_errors"}, } // Excluded is every table a backup leaves out, with the reason. var Excluded = map[string]string{ "schema_migrations": "the restored server's own migrations write it; the backup's schema version is in its manifest", "sessions": "sign-ins; everyone signs in again on the restored server", "password_resets": "one-time reset links, which expire within hours", "user_invites": "open sign-up links; an admin sends an invite again from the restored server", "track_fingerprints": "computed from the music files", "track_loudness": "computed from the music files", "album_loudness": "computed from the music files", "track_acoustid_lookups": "looked up again from each track's fingerprint", "track_tags": "fetched again from the tag providers", "candidate_artist_tags": "fetched again from the tag providers", "candidate_artist_tag_state": "fetched again from the tag providers", "mb_artists": "MusicBrainz cache, fetched again", "mb_release_groups": "MusicBrainz cache, fetched again", "mb_releases": "MusicBrainz cache, fetched again", "mb_release_tracks": "MusicBrainz cache, fetched again", "mb_artist_release_groups": "MusicBrainz cache, fetched again", "listenbrainz_similar_recordings": "similarity, fetched again from ListenBrainz", "track_similarity": "similarity, fetched again from the similarity providers", "track_similarity_fetches": "similarity, fetched again from the similarity providers", "artist_similarity": "similarity, fetched again from the similarity providers", "artist_similarity_fetches": "similarity, fetched again from the similarity providers", "artist_similarity_unmatched": "similarity, fetched again from the similarity providers", "want_candidates": "the last search's results; the next search finds them again", "scan_runs": "a log of past library scans", "diagnostic_events": "debug-mode diagnostics", } // carried reports whether a table is in Tables. func carried(name string) bool { for _, t := range Tables { if t.Name == name { return true } } return false } // Coverage checks a schema's tables against Tables and Excluded. It returns // one line per problem: a table in neither list, in both, or listed twice, // and a listed table the schema doesn't have. func Coverage(schema []string) []string { var problems []string inSchema := map[string]bool{} for _, name := range schema { inSchema[name] = true _, excluded := Excluded[name] switch { case carried(name) && excluded: problems = append(problems, name+": both backed up and excluded") case !carried(name) && !excluded: problems = append(problems, name+": neither backed up nor excluded; add it to Tables, or to Excluded with the reason") } } seen := map[string]bool{} for _, t := range Tables { if seen[t.Name] { problems = append(problems, t.Name+": listed twice in Tables") } seen[t.Name] = true if !inSchema[t.Name] { problems = append(problems, t.Name+": in Tables but not in the schema") } } for name := range Excluded { if !inSchema[name] { problems = append(problems, name+": in Excluded but not in the schema") } } return problems }