minstrel/internal/backup/tables.go

// Package backup reads Minstrel's irreplaceable data out of Postgres as JSON
// Lines, one table at a time, and loads it back into an empty instance.
//
// What goes in is everything a person made or decided: accounts, likes, play
// and skip history, playlists, follows, wants and requests, download history,
// admin decisions, and settings. What stays out is everything the restored
// server rebuilds by itself: what it computes from the music files, what it
// fetches again from MusicBrainz, ListenBrainz and the other providers, and
// logs of past runs. Excluded says why for each, and the coverage test fails
// on a table that is in neither list.
package backup

// Table is one table a backup holds.
//
// Omit names columns left out of the file. They restore to their defaults:
//   - a credential Minstrel stores sealed, which only the server holding the
//     stream secret could open, so the restored server asks for it again;
//   - a marker saying work on an excluded table is done (a tag source, an art
//     source), so the restored server does that work again rather than
//     believing it has the result.
type Table struct {
	Name string
	Omit []string
}

// Tables is every table a backup holds, in the order a restore loads them:
// each after every table it references. There are no self-referencing
// tables, so no table needs its rows loaded in depth order.
var Tables = []Table{
	{Name: "users", Omit: []string{"subsonic_password", "listenbrainz_token"}},

	// The library's identity. Every like, play and playlist entry hangs off a
	// track id, so the rows travel even though a scan could rebuild most of
	// their columns; the scan after a restore finds each file at its path and
	// keeps its id.
	{Name: "artists", Omit: []string{"artist_thumb_path", "artist_fanart_path", "artist_art_source", "artist_art_sources_version"}},
	{Name: "albums", Omit: []string{"cover_art_path", "cover_art_source", "cover_art_sources_version"}},
	{Name: "tracks", Omit: []string{"tag_source", "tag_sources_version"}},

	// Settings.
	{Name: "library_settings"},
	{Name: "network_settings"},
	{Name: "registration_settings"},
	{Name: "smtp_config", Omit: []string{"password"}},
	{Name: "notification_email_settings"},
	{Name: "acoustid_settings", Omit: []string{"api_key"}},
	{Name: "fingerprint_settings"},
	{Name: "loudness_settings"},
	{Name: "cover_art_provider_settings", Omit: []string{"api_key"}},
	{Name: "cover_art_sources_meta"},
	{Name: "tag_provider_settings", Omit: []string{"api_key"}},
	{Name: "tag_sources_meta"},
	{Name: "acquisition_services", Omit: []string{"api_key", "password"}},
	{Name: "acquisition_settings"},
	{Name: "reacquisition_settings"},
	{Name: "discover_tuning"},
	{Name: "taste_tuning"},
	{Name: "recommendation_weight_profiles"},
	{Name: "recommendation_tuning_audit"},

	// Follows, wants, requests and downloads.
	{Name: "release_type_profiles"},
	{Name: "follows"},
	{Name: "follow_seen_groups"},
	{Name: "wants"},
	{Name: "grabs"},
	{Name: "download_interventions"},
	{Name: "release_blocks"},
	{Name: "requests"},

	// Listening history and likes.
	{Name: "play_sessions"},
	{Name: "play_events"},
	{Name: "skip_events"},
	{Name: "scrobble_queue"},
	{Name: "contextual_likes"},
	{Name: "general_likes"},
	{Name: "general_likes_albums"},
	{Name: "general_likes_artists"},

	// Playlists and what is built from history. The taste profile and the
	// "you might like" rows are rebuilt nightly from the history above; they
	// travel so a restored Home is full before that first nightly pass.
	{Name: "playlists"},
	{Name: "playlist_tracks"},
	{Name: "system_playlist_runs"},
	{Name: "system_playlist_rotation_state"},
	{Name: "taste_profile_artists"},
	{Name: "taste_profile_tags"},
	{Name: "taste_profile_eras"},
	{Name: "taste_profile_moods"},
	{Name: "you_might_like_albums"},
	{Name: "you_might_like_artists"},

	// Per-user preferences and notifications.
	{Name: "user_normalization_prefs"},
	{Name: "user_notification_prefs"},
	{Name: "user_notification_email_state"},
	{Name: "user_notifications"},
	{Name: "suggestion_snoozes"},
	{Name: "quarantine"},
	{Name: "quarantine_actions"},

	// Admin decisions: an album taken off the wanted listing stays off, and a
	// dismissed duplicate group stays dismissed when the sweep finds it again.
	{Name: "library_album_matches"},
	{Name: "duplicate_sweeps"},
	{Name: "duplicate_groups"},
	{Name: "duplicate_group_members"},

	// The library change feed is how the apps sync, including a first sync,
	// so a restored server carries it on from where the backup left it.
	{Name: "library_changes"},
	{Name: "audit_log"},
	{Name: "playback_errors"},
}

// Excluded is every table a backup leaves out, with the reason.
var Excluded = map[string]string{
	"schema_migrations": "the restored server's own migrations write it; the backup's schema version is in its manifest",
	"sessions":          "sign-ins; everyone signs in again on the restored server",
	"password_resets":   "one-time reset links, which expire within hours",
	"user_invites":      "open sign-up links; an admin sends an invite again from the restored server",

	"track_fingerprints":         "computed from the music files",
	"track_loudness":             "computed from the music files",
	"album_loudness":             "computed from the music files",
	"track_acoustid_lookups":     "looked up again from each track's fingerprint",
	"track_tags":                 "fetched again from the tag providers",
	"candidate_artist_tags":      "fetched again from the tag providers",
	"candidate_artist_tag_state": "fetched again from the tag providers",

	"mb_artists":               "MusicBrainz cache, fetched again",
	"mb_release_groups":        "MusicBrainz cache, fetched again",
	"mb_releases":              "MusicBrainz cache, fetched again",
	"mb_release_tracks":        "MusicBrainz cache, fetched again",
	"mb_artist_release_groups": "MusicBrainz cache, fetched again",

	"listenbrainz_similar_recordings": "similarity, fetched again from ListenBrainz",
	"track_similarity":                "similarity, fetched again from the similarity providers",
	"track_similarity_fetches":        "similarity, fetched again from the similarity providers",
	"artist_similarity":               "similarity, fetched again from the similarity providers",
	"artist_similarity_fetches":       "similarity, fetched again from the similarity providers",
	"artist_similarity_unmatched":     "similarity, fetched again from the similarity providers",

	"want_candidates":   "the last search's results; the next search finds them again",
	"scan_runs":         "a log of past library scans",
	"diagnostic_events": "debug-mode diagnostics",

	"backup_settings":    "holds the bucket's secret key; point the restored server at its destination again",
	"backup_runs":        "this server's own backup history",
	"backup_music_files": "which music files this server has sent; the files themselves are beside the database in the destination",
}

// carried reports whether a table is in Tables.
func carried(name string) bool {
	for _, t := range Tables {
		if t.Name == name {
			return true
		}
	}
	return false
}

// Coverage checks a schema's tables against Tables and Excluded. It returns
// one line per problem: a table in neither list, in both, or listed twice,
// and a listed table the schema doesn't have.
func Coverage(schema []string) []string {
	var problems []string
	inSchema := map[string]bool{}
	for _, name := range schema {
		inSchema[name] = true
		_, excluded := Excluded[name]
		switch {
		case carried(name) && excluded:
			problems = append(problems, name+": both backed up and excluded")
		case !carried(name) && !excluded:
			problems = append(problems, name+": neither backed up nor excluded; add it to Tables, or to Excluded with the reason")
		}
	}
	seen := map[string]bool{}
	for _, t := range Tables {
		if seen[t.Name] {
			problems = append(problems, t.Name+": listed twice in Tables")
		}
		seen[t.Name] = true
		if !inSchema[t.Name] {
			problems = append(problems, t.Name+": in Tables but not in the schema")
		}
	}
	for name := range Excluded {
		if !inSchema[name] {
			problems = append(problems, name+": in Excluded but not in the schema")
		}
	}
	return problems
}