feat: Minstrel backs itself up nightly to a folder or an S3 bucket, off until an admin switches it on (#6773)

Admin › Backups shows a verdict first (off, not set up, waiting, backed up,
overdue, failing since when), Back up now, Test destination and the latest
runs, with the settings beside them.

- backup.Service: one run at a time, bounded at 30 minutes; the schedule
  ticks every 5 minutes, catches up a slot missed while the server was down,
  and retries a failure after an hour and then after as long as it has been
  failing. Runs are recorded in backup_runs, pruned after 90 days; a run cut
  off by a shutdown is marked failed at the next start.
- A folder destination writes database/minstrel-<UTC stamp>.tar and refuses
  a folder that isn't there. An S3 destination stages the file, hashes it,
  and uploads it under database/ through a throttled reader (4 MiB/s by
  default, 0 for no limit).
- Test destination checks the form as typed and only lists.
- The bucket secret is sealed like every stored credential; backup_settings
  and backup_runs are left out of the backup itself.
- The panel polls only while a run is going, capped at the run limit plus
  five minutes.

The music switch named in the task is left to #6774, so no control ships
that does nothing.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

internal/api/admin_backups.go

@@ -0,0 +1,94 @@
+package api
+
+import (
+	"encoding/json"
+	"errors"
+	"net/http"
+	"strings"
+
+	"git.fabledsword.com/bvandeusen/minstrel/internal/apierror"
+	"git.fabledsword.com/bvandeusen/minstrel/internal/backup"
+)
+
+// GET /api/admin/backups — the backup's verdict, settings and recent runs.
+func (h *handlers) handleGetBackups(w http.ResponseWriter, r *http.Request) {
+	st, err := h.backups.Status(r.Context())
+	if err != nil {
+		writeErrWithLog(w, h.logger, "admin backups: status", apierror.Internal(err))
+		return
+	}
+	writeJSON(w, http.StatusOK, st)
+}
+
+func decodeBackupUpdate(w http.ResponseWriter, r *http.Request) (backup.Update, bool) {
+	var u backup.Update
+	if err := json.NewDecoder(r.Body).Decode(&u); err != nil {
+		writeErr(w, apierror.BadRequest("invalid_body", ""))
+		return u, false
+	}
+	return u, true
+}
+
+// fieldErr writes a setting that can't be kept as a 400 whose message says
+// which setting and why, and reports whether err was one.
+func fieldErr(w http.ResponseWriter, err error) bool {
+	var fe *backup.FieldError
+	if !errors.As(err, &fe) {
+		return false
+	}
+	writeErr(w, apierror.BadRequest("invalid_backup_setting", fe.Message))
+	return true
+}
+
+// PUT /api/admin/backups — save the settings. An empty secret_key keeps the
+// stored one. Returns the new status.
+func (h *handlers) handleUpdateBackups(w http.ResponseWriter, r *http.Request) {
+	u, ok := decodeBackupUpdate(w, r)
+	if !ok {
+		return
+	}
+	if err := h.backups.Save(r.Context(), u); err != nil {
+		if !fieldErr(w, err) {
+			writeErrWithLog(w, h.logger, "admin backups: save", apierror.Internal(err))
+		}
+		return
+	}
+	h.handleGetBackups(w, r)
+}
+
+// POST /api/admin/backups/test — check the destination in the body without
+// saving it, by listing the backups already there. Nothing is written.
+func (h *handlers) handleTestBackups(w http.ResponseWriter, r *http.Request) {
+	u, ok := decodeBackupUpdate(w, r)
+	if !ok {
+		return
+	}
+	res, err := h.backups.Test(r.Context(), u)
+	if err != nil {
+		if fieldErr(w, err) {
+			return
+		}
+		msg := strings.TrimPrefix(err.Error(), "backup: ")
+		writeErr(w, &apierror.Error{Status: http.StatusBadGateway, Code: "destination_failed", Message: msg})
+		return
+	}
+	writeJSON(w, http.StatusOK, res)
+}
+
+// POST /api/admin/backups/run — start a backup now. 202 once it has started;
+// 409 while one runs or before the destination is set up.
+func (h *handlers) handleRunBackup(w http.ResponseWriter, r *http.Request) {
+	err := h.backups.Start(r.Context(), backup.TriggerManual)
+	switch {
+	case err == nil:
+		w.WriteHeader(http.StatusAccepted)
+	case errors.Is(err, backup.ErrRunning):
+		writeErr(w, apierror.Conflict("backup_running", "A backup is already running."))
+	case errors.Is(err, backup.ErrNotReady):
+		writeErr(w, apierror.Conflict("backup_not_ready", "Set up where backups go first."))
+	case errors.Is(err, backup.ErrClosed):
+		writeErr(w, apierror.Conflict("shutting_down", "The server is shutting down."))
+	default:
+		writeErrWithLog(w, h.logger, "admin backups: start", apierror.Internal(err))
+	}
+}

internal/api/api.go

@@ -15,6 +15,7 @@ import (
 
 	"git.fabledsword.com/bvandeusen/minstrel/internal/acquire"
 	"git.fabledsword.com/bvandeusen/minstrel/internal/auth"
+	"git.fabledsword.com/bvandeusen/minstrel/internal/backup"
 	"git.fabledsword.com/bvandeusen/minstrel/internal/catalogue"
 	"git.fabledsword.com/bvandeusen/minstrel/internal/config"
 	"git.fabledsword.com/bvandeusen/minstrel/internal/coverart"
@@ -40,7 +41,7 @@ import (
 // Mount attaches /api/* handlers to r. Public endpoints (login) are outside
 // RequireUser; everything else is gated by the middleware. The events writer
 // is shared with the Subsonic mount so /rest/scrobble feeds the same store.
-func Mount(r chi.Router, pool *pgxpool.Pool, logger *slog.Logger, events *playevents.Writer, recCfg config.RecommendationConfig, recSettings *recsettings.Service, requestSvc *requests.Service, quarantineSvc *quarantine.Service, tracksSvc *tracks.Service, playlistsSvc *playlists.Service, coverEnricher *coverart.Enricher, coverSettings *coverart.SettingsService, tagSettings *tags.SettingsService, scanner *library.Scanner, scanCfg library.RunScanConfig, dataDir string, sender mailer.Sender, bus *eventbus.Bus, playlistScheduler *playlists.Scheduler, streamSecret []byte, netSettings *netsettings.Service, reacqSettings *reacquisition.SettingsService, fpSettings *library.FingerprintSettingsService, loudSettings *library.LoudnessSettingsService, acoustIDLookup *library.AcoustIDLookupWorker) {
+func Mount(r chi.Router, pool *pgxpool.Pool, logger *slog.Logger, events *playevents.Writer, recCfg config.RecommendationConfig, recSettings *recsettings.Service, requestSvc *requests.Service, quarantineSvc *quarantine.Service, tracksSvc *tracks.Service, playlistsSvc *playlists.Service, coverEnricher *coverart.Enricher, coverSettings *coverart.SettingsService, tagSettings *tags.SettingsService, scanner *library.Scanner, scanCfg library.RunScanConfig, dataDir string, sender mailer.Sender, bus *eventbus.Bus, playlistScheduler *playlists.Scheduler, streamSecret []byte, netSettings *netsettings.Service, reacqSettings *reacquisition.SettingsService, fpSettings *library.FingerprintSettingsService, loudSettings *library.LoudnessSettingsService, acoustIDLookup *library.AcoustIDLookupWorker, backups *backup.Service) {
 	rng := rand.New(rand.NewSource(rand.Int63()))
 	setupToken, err := auth.NewSetupToken()
 	if err != nil {
@@ -74,6 +75,7 @@ func Mount(r chi.Router, pool *pgxpool.Pool, logger *slog.Logger, events *playev
 		fingerprintSettings: fpSettings,
 		loudnessSettings:    loudSettings,
 		acoustIDLookup:      acoustIDLookup,
+		backups:             backups,
 		acquisition:         acquire.NewStore(pool),
 		catalogue:           cat,
 		wanted:              wanted.New(pool, cat, logger),
@@ -358,6 +360,11 @@ func Mount(r chi.Router, pool *pgxpool.Pool, logger *slog.Logger, events *playev
 				admin.Post("/tag-sources/{provider_id}/test", h.handleTestTagSource)
 				admin.Post("/tag-sources/research", h.handleResearchTags)
 
+				admin.Get("/backups", h.handleGetBackups)
+				admin.Put("/backups", h.handleUpdateBackups)
+				admin.Post("/backups/test", h.handleTestBackups)
+				admin.Post("/backups/run", h.handleRunBackup)
+
 				admin.Get("/smtp-config", h.handleGetSMTPConfig)
 				admin.Put("/smtp-config", h.handleUpdateSMTPConfig)
 				admin.Post("/smtp-config/test", h.handleTestSMTPConfig)
@@ -429,6 +436,8 @@ type handlers struct {
 	// acoustIDLookup is the AcoustID lookup worker: its settings, its
 	// status and "look up now". Nil serves the defaults (off).
 	acoustIDLookup *library.AcoustIDLookupWorker
+	// backups is the server's own backup: settings, status, test and run.
+	backups *backup.Service
 	// acquisition is the configured indexers and download clients.
 	acquisition *acquire.Store
 	// catalogue is the release catalogue read from MusicBrainz, and